From a754fe0845f0e28ae0398625d8132a6047aa991e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 15:07:38 +0200 Subject: [PATCH 01/63] add schema checking for draft 2020-12 --- nf_core/pipelines/lint/schema_lint.py | 32 ++++++++++++++++++++++----- nf_core/pipelines/schema.py | 21 +++++++++++++----- requirements.txt | 2 +- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/nf_core/pipelines/lint/schema_lint.py b/nf_core/pipelines/lint/schema_lint.py index 6786c5012d..4007bf8fe5 100644 --- a/nf_core/pipelines/lint/schema_lint.py +++ b/nf_core/pipelines/lint/schema_lint.py @@ -16,26 +16,26 @@ def schema_lint(self): The lint test checks the schema for the following: * Schema should be a valid JSON file - * Schema should adhere to `JSONSchema `_, Draft 7. + * Schema should adhere to `JSONSchema `_, Draft 7 or Draft 2020-12. * Parameters can be described in two places: * As ``properties`` in the top-level schema object - * As ``properties`` within subschemas listed in a top-level ``definitions`` objects + * As ``properties`` within subschemas listed in a top-level ``definitions``(draft 7) or ``$defs``(draft 2020-12) objects * The schema must describe at least one parameter * There must be no duplicate parameter IDs across the schema and definition subschema - * All subschema in ``definitions`` must be referenced in the top-level ``allOf`` key + * All subschema in ``definitions`` or ``$defs`` must be referenced in the top-level ``allOf`` key * The top-level ``allOf`` key must not describe any non-existent definitions * Default parameters in the schema must be valid * Core top-level schema attributes should exist and be set as follows: - * ``$schema``: ``https://json-schema.org/draft-07/schema`` + * ``$schema``: ``https://json-schema.org/draft-07/schema`` or ``https://json-schema.org/draft/2020-12/schema`` * ``$id``: URL to the raw schema file, eg. ``https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json`` * ``title``: ``YOURPIPELINE pipeline parameters`` * ``description``: The pipeline config ``manifest.description`` * That the ``input`` property is defined and has a mimetype. A list of common mimetypes can be found `here `_. - For example, an *extremely* minimal schema could look like this: + For example, an *extremely* minimal schema could look like this (draft 7): .. code-block:: json @@ -57,6 +57,28 @@ def schema_lint(self): "allOf": [{"$ref": "#/definitions/my_first_group"}] } + Or this (draft 2020-12): + + .. code-block:: json + + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "$defs": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{"$ref": "#/$defs/my_first_group"}] + } + .. tip:: You can check your pipeline schema without having to run the entire pipeline lint by running ``nf-core pipelines schema lint`` instead of ``nf-core pipelines lint`` """ diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 7f562bff38..854bddccc0 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -359,11 +359,22 @@ def validate_schema(self, schema=None): """ if schema is None: schema = self.schema - try: - jsonschema.Draft7Validator.check_schema(schema) - log.debug("JSON Schema Draft7 validated") - except jsonschema.exceptions.SchemaError as e: - raise AssertionError(f"Schema does not validate as Draft 7 JSON Schema:\n {e}") + + schema_draft = schema["$schema"] + if schema_draft == "https://json-schema.org/draft-07/schema": + try: + jsonschema.Draft7Validator.check_schema(schema) + log.debug("JSON Schema Draft7 validated") + except jsonschema.exceptions.SchemaError as e: + raise AssertionError(f"Schema does not validate as Draft 7 JSON Schema:\n {e}") + elif schema_draft == "https://json-schema.org/draft/2020-12/schema": + try: + jsonschema.Draft202012Validator.check_schema(schema) + log.debug("JSON Schema Draft2020-12 validated") + except jsonschema.exceptions.SchemaError as e: + raise AssertionError(f"Schema does not validate as Draft 2020-12 JSON Schema:\n {e}") + else: + raise AssertionError(f"Unsupported JSON schema draft detected: ${schema_draft}. Use draft 7 or 2020-12 instead.") param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) diff --git a/requirements.txt b/requirements.txt index fb658be2fb..eba6460f03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ filetype GitPython PyGithub jinja2 -jsonschema>=3.0 +jsonschema>=4.0 markdown>=3.3 packaging pillow From 464f16e982337699382e66e3b1da21af4f01dd7a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 15:26:52 +0200 Subject: [PATCH 02/63] update validate schema --- nf_core/pipelines/schema.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 854bddccc0..abdad719f6 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -378,17 +378,32 @@ def validate_schema(self, schema=None): param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) - for d_key, d_schema in schema.get("definitions", {}).items(): + schema_defs = dict() + defs_notation = "" + if "$defs" in schema: + schema_defs = schema.get("$defs", {}).items() + defs_notation = "$defs" + elif "definitions" in schema: + schema_defs = schema.get("definitions", {}).items() + defs_notation = "definitions" + elif "defs" in schema: + # nf-schema v2.0.0 only supported defs. this has been changed to $defs in nf-schema v2.1.0 + # this line prevents the breakage of schemas created for v2.0.0 + schema_defs = schema.get("defs", {}).items() + defs_notation = "defs" + + for d_key, d_schema in schema_defs: # Check that this definition is mentioned in allOf if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") in_allOf = False for allOf in schema.get("allOf", []): - if allOf["$ref"] == f"#/definitions/{d_key}": + if allOf["$ref"] == f"#/{defs_notation}/{d_key}": in_allOf = True if not in_allOf: - raise AssertionError(f"Definition subschema `{d_key}` not included in schema `allOf`") + raise AssertionError(f"Definition subschema `#{defs_notation}/{d_key}` not included in schema `allOf`") + # TODO add support for nested parameters for d_param_id in d_schema.get("properties", {}): # Check that we don't have any duplicate parameter IDs in different definitions if d_param_id in param_keys: @@ -398,11 +413,11 @@ def validate_schema(self, schema=None): # Check that everything in allOf exists for allOf in schema.get("allOf", []): - if "definitions" not in schema: - raise AssertionError("Schema has allOf, but no definitions") - def_key = allOf["$ref"][14:] - if def_key not in schema.get("definitions", {}): - raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `definitions`") + _, allOf_defs_notation, def_key = allOf["$ref"].split("/") # "#//" + if allOf_defs_notation not in schema: + raise AssertionError(f"Schema has allOf, but no {allOf_defs_notation}") + if def_key not in schema.get(allOf_defs_notation, {}): + raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `{allOf_defs_notation}`") # Check that the schema describes at least one parameter if num_params == 0: From 6658449cc892021782a22885c3db63c0215254b6 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 15:51:27 +0200 Subject: [PATCH 03/63] update more code to support draft 2020-12 --- nf_core/pipelines/schema.py | 55 ++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index abdad719f6..398783d5a8 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -116,8 +116,26 @@ def load_schema(self): self.schema = json.load(fh) self.schema_defaults = {} self.schema_params = {} + if "$schema" not in self.schema: + raise AssertionError("Schema missing top-level `$schema` attribute") + self.schema_draft = self.schema["$schema"] + self.defs_notation = self.get_defs_notation() log.debug(f"JSON file loaded: {self.schema_filename}") + def get_defs_notation(self, schema=None): + if schema is None: + schema = self.schema + + if "$defs" in schema: + defs_notation = "$defs" + elif "definitions" in schema: + defs_notation = "definitions" + elif "defs" in schema: + # nf-schema v2.0.0 only supported defs. this has been changed to $defs in nf-schema v2.1.0 + # this line prevents the breakage of schemas created for v2.0.0 + defs_notation = "defs" + return defs_notation + def sanitise_param_default(self, param): """ Given a param, ensure that the default value is the correct variable type @@ -168,10 +186,11 @@ def get_schema_defaults(self) -> None: if param["default"] is not None: self.schema_defaults[p_key] = param["default"] + # TODO add support for nested parameters # Grouped schema properties in subschema definitions - for defn_name, definition in self.schema.get("definitions", {}).items(): + for defn_name, definition in self.schema.get(self.defs_notation, {}).items(): for p_key, param in definition.get("properties", {}).items(): - self.schema_params[p_key] = ("definitions", defn_name, "properties", p_key) + self.schema_params[p_key] = (self.defs_notation, defn_name, "properties", p_key) if "default" in param: param = self.sanitise_param_default(param) if param["default"] is not None: @@ -248,13 +267,14 @@ def validate_default_params(self): if self.schema is None: log.error("[red][✗] Pipeline schema not found") try: + # TODO add support for nested parameters # Make copy of schema and remove required flags schema_no_required = copy.deepcopy(self.schema) if "required" in schema_no_required: schema_no_required.pop("required") - for group_key, group in schema_no_required.get("definitions", {}).items(): + for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): if "required" in group: - schema_no_required["definitions"][group_key].pop("required") + schema_no_required[self.defs_notation][group_key].pop("required") jsonschema.validate(self.schema_defaults, schema_no_required) except jsonschema.exceptions.ValidationError as e: raise AssertionError(f"Default parameters are invalid: {e.message}") @@ -360,6 +380,8 @@ def validate_schema(self, schema=None): if schema is None: schema = self.schema + if "$schema" not in schema: + raise AssertionError("Schema missing top-level `$schema` attribute") schema_draft = schema["$schema"] if schema_draft == "https://json-schema.org/draft-07/schema": try: @@ -374,25 +396,14 @@ def validate_schema(self, schema=None): except jsonschema.exceptions.SchemaError as e: raise AssertionError(f"Schema does not validate as Draft 2020-12 JSON Schema:\n {e}") else: - raise AssertionError(f"Unsupported JSON schema draft detected: ${schema_draft}. Use draft 7 or 2020-12 instead.") + raise AssertionError(f"Schema `$schema` should be `https://json-schema.org/draft/2020-12/schema` or `https://json-schema.org/draft-07/schema` \n Found `{schema_draft}`") param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) schema_defs = dict() - defs_notation = "" - if "$defs" in schema: - schema_defs = schema.get("$defs", {}).items() - defs_notation = "$defs" - elif "definitions" in schema: - schema_defs = schema.get("definitions", {}).items() - defs_notation = "definitions" - elif "defs" in schema: - # nf-schema v2.0.0 only supported defs. this has been changed to $defs in nf-schema v2.1.0 - # this line prevents the breakage of schemas created for v2.0.0 - schema_defs = schema.get("defs", {}).items() - defs_notation = "defs" + defs_notation = self.get_defs_notation(schema) - for d_key, d_schema in schema_defs: + for d_key, d_schema in schema.get(defs_notation, {}).items(): # Check that this definition is mentioned in allOf if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") @@ -428,7 +439,7 @@ def validate_schema(self, schema=None): def validate_schema_title_description(self, schema=None): """ Extra validation command for linting. - Checks that the schema "$id", "title" and "description" attributes match the piipeline config. + Checks that the schema "$id", "title" and "description" attributes match the pipeline config. """ if schema is None: schema = self.schema @@ -436,12 +447,6 @@ def validate_schema_title_description(self, schema=None): log.debug("Pipeline schema not set - skipping validation of top-level attributes") return None - if "$schema" not in self.schema: - raise AssertionError("Schema missing top-level `$schema` attribute") - schema_attr = "http://json-schema.org/draft-07/schema" - if self.schema["$schema"] != schema_attr: - raise AssertionError(f"Schema `$schema` should be `{schema_attr}`\n Found `{self.schema['$schema']}`") - if self.pipeline_manifest == {}: self.get_wf_params() From 6ecde202d05baa2944f582f0b0835ea73765006b Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:11:47 +0200 Subject: [PATCH 04/63] fix draft 7 schema using http --- nf_core/pipelines/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 398783d5a8..e1931b9007 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -383,7 +383,7 @@ def validate_schema(self, schema=None): if "$schema" not in schema: raise AssertionError("Schema missing top-level `$schema` attribute") schema_draft = schema["$schema"] - if schema_draft == "https://json-schema.org/draft-07/schema": + if schema_draft == "https://json-schema.org/draft-07/schema" or schema_draft == "http://json-schema.org/draft-07/schema": try: jsonschema.Draft7Validator.check_schema(schema) log.debug("JSON Schema Draft7 validated") From 36fad24c7eda672fce4a885fe2e7d5610b022a9c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:18:08 +0200 Subject: [PATCH 05/63] ruff --- nf_core/pipelines/schema.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index e1931b9007..188861fff1 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -400,7 +400,6 @@ def validate_schema(self, schema=None): param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) - schema_defs = dict() defs_notation = self.get_defs_notation(schema) for d_key, d_schema in schema.get(defs_notation, {}).items(): @@ -424,11 +423,11 @@ def validate_schema(self, schema=None): # Check that everything in allOf exists for allOf in schema.get("allOf", []): - _, allOf_defs_notation, def_key = allOf["$ref"].split("/") # "#//" - if allOf_defs_notation not in schema: - raise AssertionError(f"Schema has allOf, but no {allOf_defs_notation}") - if def_key not in schema.get(allOf_defs_notation, {}): - raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `{allOf_defs_notation}`") + _, allof_defs_notation, def_key = allOf["$ref"].split("/") # "#//" + if allof_defs_notation not in schema: + raise AssertionError(f"Schema has allOf, but no {allof_defs_notation}") + if def_key not in schema.get(allof_defs_notation, {}): + raise AssertionError(f"Subschema `{def_key}` found in `allOf` but not `{allof_defs_notation}`") # Check that the schema describes at least one parameter if num_params == 0: From 6777694b20726f333dd8eae416917c3324c3e158 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:33:00 +0200 Subject: [PATCH 06/63] fix tests + add draft 2020-12 as default draft --- nf_core/pipelines/schema.py | 2 ++ tests/pipelines/test_schema.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 188861fff1..1555a4198d 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -46,6 +46,8 @@ def __init__(self): self.web_schema_build_url = "https://nf-co.re/pipeline_schema_builder" self.web_schema_build_web_url = None self.web_schema_build_api_url = None + self.schema_draft = "https://json-schema.org/draft/2020-12/schema" + self.defs_notation = "$defs" def get_schema_path( self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None diff --git a/tests/pipelines/test_schema.py b/tests/pipelines/test_schema.py index 633de3db69..777d959e66 100644 --- a/tests/pipelines/test_schema.py +++ b/tests/pipelines/test_schema.py @@ -175,6 +175,7 @@ def test_validate_schema_fail_duplicate_ids(self): Check that the schema validation fails when we have duplicate IDs in definition subschema """ self.schema_obj.schema = { + "$schema": "http://json-schema.org/draft-07/schema", "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"foo": {}}}}, "allOf": [{"$ref": "#/definitions/groupOne"}, {"$ref": "#/definitions/groupTwo"}], } @@ -187,18 +188,20 @@ def test_validate_schema_fail_missing_def(self): Check that the schema validation fails when we a definition in allOf is not in definitions """ self.schema_obj.schema = { + "$schema": "http://json-schema.org/draft-07/schema", "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, "allOf": [{"$ref": "#/definitions/groupOne"}], } with pytest.raises(AssertionError) as exc_info: self.schema_obj.validate_schema(self.schema_obj.schema) - assert exc_info.value.args[0] == "Definition subschema `groupTwo` not included in schema `allOf`" + assert exc_info.value.args[0] == "Definition subschema `#/definitions/groupTwo` not included in schema `allOf`" def test_validate_schema_fail_unexpected_allof(self): """ Check that the schema validation fails when we an unrecognised definition is in allOf """ self.schema_obj.schema = { + "$schema": "http://json-schema.org/draft-07/schema", "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, "allOf": [ {"$ref": "#/definitions/groupOne"}, From 0dfd15ded935054fc8d185b5fd36223505441ff7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:34:01 +0200 Subject: [PATCH 07/63] fix typo --- nf_core/pipelines/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 1555a4198d..ba5af51f96 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -413,7 +413,7 @@ def validate_schema(self, schema=None): if allOf["$ref"] == f"#/{defs_notation}/{d_key}": in_allOf = True if not in_allOf: - raise AssertionError(f"Definition subschema `#{defs_notation}/{d_key}` not included in schema `allOf`") + raise AssertionError(f"Definition subschema `#/{defs_notation}/{d_key}` not included in schema `allOf`") # TODO add support for nested parameters for d_param_id in d_schema.get("properties", {}): From 7f29f373a82ed1ada16b6b493f10d724a4b879cc Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 16:37:06 +0200 Subject: [PATCH 08/63] ruff --- nf_core/pipelines/schema.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index ba5af51f96..43058d4e6f 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -119,7 +119,7 @@ def load_schema(self): self.schema_defaults = {} self.schema_params = {} if "$schema" not in self.schema: - raise AssertionError("Schema missing top-level `$schema` attribute") + raise AssertionError("Schema missing top-level `$schema` attribute") self.schema_draft = self.schema["$schema"] self.defs_notation = self.get_defs_notation() log.debug(f"JSON file loaded: {self.schema_filename}") @@ -385,7 +385,10 @@ def validate_schema(self, schema=None): if "$schema" not in schema: raise AssertionError("Schema missing top-level `$schema` attribute") schema_draft = schema["$schema"] - if schema_draft == "https://json-schema.org/draft-07/schema" or schema_draft == "http://json-schema.org/draft-07/schema": + if ( + schema_draft == "https://json-schema.org/draft-07/schema" + or schema_draft == "http://json-schema.org/draft-07/schema" + ): try: jsonschema.Draft7Validator.check_schema(schema) log.debug("JSON Schema Draft7 validated") @@ -398,7 +401,9 @@ def validate_schema(self, schema=None): except jsonschema.exceptions.SchemaError as e: raise AssertionError(f"Schema does not validate as Draft 2020-12 JSON Schema:\n {e}") else: - raise AssertionError(f"Schema `$schema` should be `https://json-schema.org/draft/2020-12/schema` or `https://json-schema.org/draft-07/schema` \n Found `{schema_draft}`") + raise AssertionError( + f"Schema `$schema` should be `https://json-schema.org/draft/2020-12/schema` or `https://json-schema.org/draft-07/schema` \n Found `{schema_draft}`" + ) param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) @@ -425,7 +430,7 @@ def validate_schema(self, schema=None): # Check that everything in allOf exists for allOf in schema.get("allOf", []): - _, allof_defs_notation, def_key = allOf["$ref"].split("/") # "#//" + _, allof_defs_notation, def_key = allOf["$ref"].split("/") # "#//" if allof_defs_notation not in schema: raise AssertionError(f"Schema has allOf, but no {allof_defs_notation}") if def_key not in schema.get(allof_defs_notation, {}): From 730c6779779d4e14512df25c043eb02efe6d1659 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 17:34:53 +0200 Subject: [PATCH 09/63] add checks for nf-validation and nf-schema plugin config --- nf_core/pipelines/lint/nextflow_config.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 96323af94d..8fe933d102 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -336,6 +336,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) # Check for the availability of the "test" configuration profile by parsing nextflow.config + # Also check for the presence of nf-validation/nf-schema and check if they have pinned versions with open(Path(self.wf_path, "nextflow.config")) as f: content = f.read() @@ -363,6 +364,38 @@ def nextflow_config(self) -> Dict[str, List[str]]: else: failed.append("nextflow.config does not contain configuration profile `test`") + match_plugins = re.search(r"\bplugins\s*\{([^}]+)}", cleaned_content, re.MULTILINE) + if not match_plugins: + failed.append( + "nextflow.config does not contain `plugins` scope, but `nf-validation` or `nf-schema` plugins are required" + ) + else: + found_plugins = {} + for line in match_plugins.group(1).split("\n"): + cleaned_line = line.split("//")[0].strip().replace("\"", "'") + if "id" not in line: continue + match_line = re.search(r"\bid\s'([^']+)'", cleaned_line) + if not match_line: + failed.append(f"nextflow.config contains an invalid plugins identifier: {cleaned_line}") + continue + plugin = match_line.group(1) + name = plugin.split("@")[0] + version = "" + if "@" in plugin: + version = plugin.split("@")[1] + found_plugins[name] = version + + if len(found_plugins) == 0: + failed.append("nextflow.config contains an empty plugins scope") + elif "nf-validation" in found_plugins and "nf-schema" in found_plugins: + failed.append("nextflow.config contains both nf-validation and nf-schema") + elif "nf-validation" in found_plugins and found_plugins["nf-validation"] == "": + failed.append("nextflow.config contains an unpinned version of nf-validation") + elif "nf-schema" in found_plugins and found_plugins["nf-schema"] == "": + failed.append("nextflow.config contains an unpinned version of nf-schema") + elif "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: + failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json ignore_defaults = [] for item in ignore_configs: From 7fbbff7a412c64ef67417dd60a722d67f6d37647 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 14 Aug 2024 17:39:10 +0200 Subject: [PATCH 10/63] linting fully works now! --- nf_core/pipelines/lint/schema_description.py | 5 +++-- nf_core/pipelines/schema.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/pipelines/lint/schema_description.py b/nf_core/pipelines/lint/schema_description.py index d617e40949..b586cc5242 100644 --- a/nf_core/pipelines/lint/schema_description.py +++ b/nf_core/pipelines/lint/schema_description.py @@ -36,8 +36,9 @@ def schema_description(self): warned.append(f"Ungrouped param in schema: `{up}`") # Iterate over groups and add warning for parameters without a description - for group_key in self.schema_obj.schema["definitions"].keys(): - group = self.schema_obj.schema["definitions"][group_key] + defs_notation = self.schema_obj.defs_notation + for group_key in self.schema_obj.schema[defs_notation].keys(): + group = self.schema_obj.schema[defs_notation][group_key] for param_key, param in group["properties"].items(): if param_key in ignore_params: ignored.append(f"Ignoring description check for param in schema: `{param_key}`") diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 43058d4e6f..c70332beb2 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -203,7 +203,7 @@ def get_schema_types(self) -> None: for name, param in self.schema.get("properties", {}).items(): if "type" in param: self.schema_types[name] = param["type"] - for _, definition in self.schema.get("definitions", {}).items(): + for _, definition in self.schema.get(self.defs_notation, {}).items(): for name, param in definition.get("properties", {}).items(): if "type" in param: self.schema_types[name] = param["type"] From 0838d6bb040b8218493fad13c5fc33fa355b84af Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 14:58:27 +0200 Subject: [PATCH 11/63] warning for nf-validation --- nf_core/pipelines/lint/nextflow_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 8fe933d102..84bb492cb0 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -396,6 +396,9 @@ def nextflow_config(self) -> Dict[str, List[str]]: elif "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + if "nf-validation" in found_plugins: + warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") + # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json ignore_defaults = [] for item in ignore_configs: From 8559c2d6a18624c5096957e6a941011c8725e580 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 16:54:38 +0200 Subject: [PATCH 12/63] fetch plugin on schema filename setting --- nf_core/pipelines/lint/nextflow_config.py | 1 + nf_core/pipelines/schema.py | 81 +++++++++++++++-------- 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 84bb492cb0..a5b7cf325f 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -364,6 +364,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: else: failed.append("nextflow.config does not contain configuration profile `test`") + # Lint for nf-validation and nf-schema match_plugins = re.search(r"\bplugins\s*\{([^}]+)}", cleaned_content, re.MULTILINE) if not match_plugins: failed.append( diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index c70332beb2..77f421724e 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -5,6 +5,7 @@ import logging import tempfile import webbrowser +import re from pathlib import Path from typing import Union @@ -32,7 +33,7 @@ def __init__(self): self.schema = {} self.pipeline_dir = "" - self.schema_filename = "" + self._schema_filename = "" self.schema_defaults = {} self.schema_types = {} self.schema_params = {} @@ -46,8 +47,46 @@ def __init__(self): self.web_schema_build_url = "https://nf-co.re/pipeline_schema_builder" self.web_schema_build_web_url = None self.web_schema_build_api_url = None - self.schema_draft = "https://json-schema.org/draft/2020-12/schema" - self.defs_notation = "$defs" + self.validation_plugin = None + self.schema_draft = None + self.defs_notation = None + + # Update the validation plugin code everytime the schema gets changed + def set_schema_filename(self, schema: str) -> None: + self._schema_filename = schema + basepath = "/".join(str(schema).split("/")[:-1]) + config = f"{basepath}/nextflow.config" if basepath != "" else "nextflow.config" + self._update_validation_plugin_from_config(config) + + def get_schema_filename(self) -> str: + return self._schema_filename + + def del_schema_filename(self) -> None: + del self._schema_filename + + schema_filename = property(get_schema_filename, set_schema_filename, del_schema_filename) + + def _update_validation_plugin_from_config(self, config: str) -> None: + plugin = "nf-schema" + with open(Path(config)) as conf: + nf_schema_pattern = re.compile("id\s*[\"']nf-schema", re.MULTILINE) + nf_validation_pattern = re.compile("id\s*[\"']nf-validation", re.MULTILINE) + config_content = conf.read() + if re.search(nf_validation_pattern, config_content): + plugin = "nf-validation" + elif re.search(nf_schema_pattern, config_content): + plugin = "nf-schema" + else: + log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") + + self.validation_plugin = plugin + # Previous versions of nf-schema used "defs", but it's advised to use "$defs" + if plugin == "nf-schema": + self.defs_notation = "$defs" + self.schema_draft = "https://json-schema.org/draft/2020-12/schema" + else: + self.defs_notation = "definitions" + self.schema_draft = "https://json-schema.org/draft-07/schema" def get_schema_path( self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None @@ -120,24 +159,8 @@ def load_schema(self): self.schema_params = {} if "$schema" not in self.schema: raise AssertionError("Schema missing top-level `$schema` attribute") - self.schema_draft = self.schema["$schema"] - self.defs_notation = self.get_defs_notation() log.debug(f"JSON file loaded: {self.schema_filename}") - def get_defs_notation(self, schema=None): - if schema is None: - schema = self.schema - - if "$defs" in schema: - defs_notation = "$defs" - elif "definitions" in schema: - defs_notation = "definitions" - elif "defs" in schema: - # nf-schema v2.0.0 only supported defs. this has been changed to $defs in nf-schema v2.1.0 - # this line prevents the breakage of schemas created for v2.0.0 - defs_notation = "defs" - return defs_notation - def sanitise_param_default(self, param): """ Given a param, ensure that the default value is the correct variable type @@ -385,16 +408,15 @@ def validate_schema(self, schema=None): if "$schema" not in schema: raise AssertionError("Schema missing top-level `$schema` attribute") schema_draft = schema["$schema"] - if ( - schema_draft == "https://json-schema.org/draft-07/schema" - or schema_draft == "http://json-schema.org/draft-07/schema" - ): + if self.schema_draft != schema_draft: + raise AssertionError(f"Schema is using the wrong draft: {schema_draft}, should be {self.schema_draft}") + if self.schema_draft == "https://json-schema.org/draft-07/schema": try: jsonschema.Draft7Validator.check_schema(schema) log.debug("JSON Schema Draft7 validated") except jsonschema.exceptions.SchemaError as e: raise AssertionError(f"Schema does not validate as Draft 7 JSON Schema:\n {e}") - elif schema_draft == "https://json-schema.org/draft/2020-12/schema": + elif self.schema_draft == "https://json-schema.org/draft/2020-12/schema": try: jsonschema.Draft202012Validator.check_schema(schema) log.debug("JSON Schema Draft2020-12 validated") @@ -407,18 +429,21 @@ def validate_schema(self, schema=None): param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) - defs_notation = self.get_defs_notation(schema) - for d_key, d_schema in schema.get(defs_notation, {}).items(): + print(self.defs_notation) + + for d_key, d_schema in schema.get(self.defs_notation, {}).items(): + print(d_key) + print(d_schema) # Check that this definition is mentioned in allOf if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") in_allOf = False for allOf in schema.get("allOf", []): - if allOf["$ref"] == f"#/{defs_notation}/{d_key}": + if allOf["$ref"] == f"#/{self.defs_notation}/{d_key}": in_allOf = True if not in_allOf: - raise AssertionError(f"Definition subschema `#/{defs_notation}/{d_key}` not included in schema `allOf`") + raise AssertionError(f"Definition subschema `#/{self.defs_notation}/{d_key}` not included in schema `allOf`") # TODO add support for nested parameters for d_param_id in d_schema.get("properties", {}): From 0296f7d8fa6c5cc7a8dac62b450a8babcd77bab7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 17:00:33 +0200 Subject: [PATCH 13/63] add a check for defs usage instead of $defs --- nf_core/pipelines/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 77f421724e..9c65f34fde 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -430,11 +430,11 @@ def validate_schema(self, schema=None): param_keys = list(schema.get("properties", {}).keys()) num_params = len(param_keys) - print(self.defs_notation) + # Add a small check for older nf-schema JSON schemas + if "defs" in schema: + raise AssertionError(f'Using "defs" for schema definitions is not supported. Please use {self.defs_notation} instead') for d_key, d_schema in schema.get(self.defs_notation, {}).items(): - print(d_key) - print(d_schema) # Check that this definition is mentioned in allOf if "allOf" not in schema: raise AssertionError("Schema has definitions, but no allOf key") From 0b8d04f89eb7f348bce87cc93f04d983e48e218f Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 17:16:31 +0200 Subject: [PATCH 14/63] change all definitions to self.defs_notation --- nf_core/pipelines/schema.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 9c65f34fde..9db9457516 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -235,7 +235,7 @@ def save_schema(self, suppress_logging=False): """Save a pipeline schema to a file""" # Write results to a JSON file num_params = len(self.schema.get("properties", {})) - num_params += sum(len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()) + num_params += sum(len(d.get("properties", {})) for d in self.schema.get(self.defs_notation, {}).values()) if not suppress_logging: log.info(f"Writing schema with {num_params} params: '{self.schema_filename}'") dump_json_with_prettier(self.schema_filename, self.schema) @@ -321,7 +321,7 @@ def validate_default_params(self): params_ignore = [] # Go over group keys - for group_key, group in schema_no_required.get("definitions", {}).items(): + for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): group_properties = group.get("properties") for param in group_properties: if param in params_ignore: @@ -527,9 +527,9 @@ def check_for_input_mimetype(self): if "input" not in self.schema_params: raise LookupError("Parameter `input` not found in schema") # Check that the input parameter is defined in the right place - if "input" not in self.schema.get("definitions", {}).get("input_output_options", {}).get("properties", {}): + if "input" not in self.schema.get(self.defs_notation, {}).get("input_output_options", {}).get("properties", {}): raise LookupError("Parameter `input` is not defined in the correct subschema (input_output_options)") - input_entry = self.schema["definitions"]["input_output_options"]["properties"]["input"] + input_entry = self.schema[self.defs_notation]["input_output_options"]["properties"]["input"] if "mimetype" not in input_entry: return None mimetype = input_entry["mimetype"] @@ -581,7 +581,7 @@ def schema_to_markdown(self, columns): out = f"# {self.schema['title']}\n\n" out += f"{self.schema['description']}\n" # Grouped parameters - for definition in self.schema.get("definitions", {}).values(): + for definition in self.schema.get(self.defs_notation, {}).values(): out += f"\n## {definition.get('title', {})}\n\n" out += f"{definition.get('description', '')}\n\n" required = definition.get("required", []) @@ -763,15 +763,15 @@ def remove_schema_empty_definitions(self): """ # Identify and remove empty definitions from the schema empty_definitions = [] - for d_key, d_schema in list(self.schema.get("definitions", {}).items()): + for d_key, d_schema in list(self.schema.get(self.defs_notation, {}).items()): if not d_schema.get("properties"): - del self.schema["definitions"][d_key] + del self.schema[self.defs_notation][d_key] empty_definitions.append(d_key) log.warning(f"Removing empty group: '{d_key}'") # Remove "allOf" group with empty definitions from the schema for d_key in empty_definitions: - allOf = {"$ref": f"#/definitions/{d_key}"} + allOf = {"$ref": f"#/{self.defs_notation}/{d_key}"} if allOf in self.schema.get("allOf", []): self.schema["allOf"].remove(allOf) @@ -780,8 +780,8 @@ def remove_schema_empty_definitions(self): del self.schema["allOf"] # If we don't have anything left in "definitions", remove it - if self.schema.get("definitions") == {}: - del self.schema["definitions"] + if self.schema.get(self.defs_notation) == {}: + del self.schema[self.defs_notation] def remove_schema_notfound_configs(self): """ @@ -791,9 +791,9 @@ def remove_schema_notfound_configs(self): # Top-level properties self.schema, params_removed = self.remove_schema_notfound_configs_single_schema(self.schema) # Sub-schemas in definitions - for d_key, definition in self.schema.get("definitions", {}).items(): + for d_key, definition in self.schema.get(self.defs_notation, {}).items(): cleaned_schema, p_removed = self.remove_schema_notfound_configs_single_schema(definition) - self.schema["definitions"][d_key] = cleaned_schema + self.schema[self.defs_notation][d_key] = cleaned_schema params_removed.extend(p_removed) return params_removed From 544381310c6e5ac57664524f63105441246c34fd Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 19 Aug 2024 18:33:52 +0200 Subject: [PATCH 15/63] migrate template part 1 --- nf_core/pipeline-template/conf/colors.config | 61 ++++++++ nf_core/pipeline-template/nextflow.config | 44 ++++-- .../pipeline-template/nextflow_schema.json | 55 +++---- .../utils_nfcore_pipeline_pipeline/main.nf | 17 +- .../nf-core/utils_nfcore_pipeline/main.nf | 46 ------ .../nf-core/utils_nfvalidation_plugin/main.nf | 25 +-- .../tests/main.nf.test | 145 +----------------- .../tests/nextflow.config | 1 + .../pipeline-template/workflows/pipeline.nf | 2 +- 9 files changed, 130 insertions(+), 266 deletions(-) create mode 100644 nf_core/pipeline-template/conf/colors.config create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config diff --git a/nf_core/pipeline-template/conf/colors.config b/nf_core/pipeline-template/conf/colors.config new file mode 100644 index 0000000000..00f2963267 --- /dev/null +++ b/nf_core/pipeline-template/conf/colors.config @@ -0,0 +1,61 @@ +// For now, there is no easy way to set monochromeLogs +colors { + // Reset / Meta + reset = "\033[0m" + bold = "\033[1m" + dim = "\033[2m" + underlined = "\033[4m" + blink = "\033[5m" + reverse = "\033[7m" + hidden = "\033[8m" + + // Regular Colors + black = "\033[0;30m" + red = "\033[0;31m" + green = "\033[0;32m" + yellow = "\033[0;33m" + blue = "\033[0;34m" + purple = "\033[0;35m" + cyan = "\033[0;36m" + white = "\033[0;37m" + + // Bold + bblack = "\033[1;30m" + bred = "\033[1;31m" + bgreen = "\033[1;32m" + byellow = "\033[1;33m" + bblue = "\033[1;34m" + bpurple = "\033[1;35m" + bcyan = "\033[1;36m" + bwhite = "\033[1;37m" + + // Underline + ublack = "\033[4;30m" + ured = "\033[4;31m" + ugreen = "\033[4;32m" + uyellow = "\033[4;33m" + ublue = "\033[4;34m" + upurple = "\033[4;35m" + ucyan = "\033[4;36m" + uwhite = "\033[4;37m" + + // High Intensity + iblack = "\033[0;90m" + ired = "\033[0;91m" + igreen = "\033[0;92m" + iyellow = "\033[0;93m" + iblue = "\033[0;94m" + ipurple = "\033[0;95m" + icyan = "\033[0;96m" + iwhite = "\033[0;97m" + + // Bold High Intensity + biblack = "\033[1;90m" + bired = "\033[1;91m" + bigreen = "\033[1;92m" + biyellow = "\033[1;93m" + biblue = "\033[1;94m" + bipurple = "\033[1;95m" + bicyan = "\033[1;96m" + biwhite = "\033[1;97m" +} \ No newline at end of file diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 0469d6cfec..3d3841ba49 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -38,6 +38,8 @@ params { monochrome_logs = false hook_url = null help = false + helpFull = false + showHidden = false version = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' @@ -59,10 +61,6 @@ params { max_time = '240.h' // Schema validation default options - validationFailUnrecognisedParams = false - validationLenientMode = false - validationSchemaIgnoreParams = 'genomes,igenomes_base' - validationShowHiddenParams = false validate_params = true } @@ -200,11 +198,6 @@ podman.registry = 'quay.io' singularity.registry = 'quay.io' charliecloud.registry = 'quay.io' -// Nextflow plugins -plugins { - id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet -} - {% if igenomes -%} // Load igenomes.config if required if (!params.igenomes_ignore) { @@ -267,6 +260,39 @@ manifest { doi = '' } +// Nextflow plugins +plugins { + id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} + +includeConfig "conf/colors.config" + +validation { + parametersSchema = "${projectDir}/nextflow_schema.json" + help { + enabled = true + command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " + beforeText = """ +-${colors.dim}----------------------------------------------------${colors.reset}- + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} +${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} +${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} +${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} +${colors.purple} ${manifest.name} ${manifest.version}${colors.reset} +-${colors.dim}----------------------------------------------------${colors.reset}- +""" + afterText = """${manifest.doi ? "* The pipeline\n" : ""}${manifest.doi.tokenize(",").collect { " https://doi.org/${it.trim().replace('https://doi.org/','')}"}.join("\n")}${manifest.doi ? "\n" : ""} +* The nf-core framework + https://doi.org/10.1038/s41587-020-0439-x + +* Software dependencies + https://github.com/${manifest.name}/blob/master/CITATIONS.md +""" + } +} + + // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 4a1a22c3ee..14120bbabc 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/{{ name }}/master/nextflow_schema.json", "title": "{{ name }} pipeline parameters", "description": "{{ description }}", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -73,6 +73,20 @@ "fa_icon": "fas fa-ban", "hidden": true, "help_text": "Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`." + }, + "igenomes_base": { + "type": "string", + "format": "directory-path", + "description": "The base path to the igenomes reference files", + "fa_icon": "fas fa-ban", + "hidden": true, + "default": "s3://ngi-igenomes/igenomes/" + }, + "genomes": { + "type": "object", + "description": "An object containing all reference data availabe in igenomes", + "fa_icon": "fas fa-ban", + "hidden": true } } }, @@ -169,12 +183,6 @@ "description": "Less common options for the pipeline, typically set in a config file.", "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", "properties": { - "help": { - "type": "boolean", - "description": "Display help text.", - "fa_icon": "fas fa-question-circle", - "hidden": true - }, "version": { "type": "boolean", "description": "Display version and exit.", @@ -253,27 +261,6 @@ "fa_icon": "fas fa-check-square", "hidden": true }, - "validationShowHiddenParams": { - "type": "boolean", - "fa_icon": "far fa-eye-slash", - "description": "Show all params when using `--help`", - "hidden": true, - "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "validationFailUnrecognisedParams": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters fails when an unrecognised parameter is found.", - "hidden": true, - "help_text": "By default, when an unrecognised parameter is found, it returns a warinig." - }, - "validationLenientMode": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters in lenient more.", - "hidden": true, - "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." - }, "pipelines_testdata_base_path": { "type": "string", "fa_icon": "far fa-check-circle", @@ -286,19 +273,19 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, {% if igenomes %}{ - "$ref": "#/definitions/reference_genome_options" + "$ref": "#/$defs/reference_genome_options" },{% endif %} {% if nf_core_configs %}{ - "$ref": "#/definitions/institutional_config_options" + "$ref": "#/$defs/institutional_config_options" },{% endif %} { - "$ref": "#/definitions/max_job_request_options" + "$ref": "#/$defs/max_job_request_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 72c9be85eb..c83d2ae0cb 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -9,8 +9,8 @@ */ include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' -include { paramsSummaryMap } from 'plugin/nf-validation' -include { fromSamplesheet } from 'plugin/nf-validation' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' @@ -30,7 +30,6 @@ workflow PIPELINE_INITIALISATION { take: version // boolean: Display version and exit - help // boolean: Display help text validate_params // boolean: Boolean whether to validate parameters against the schema at runtime monochrome_logs // boolean: Do not use coloured log outputs nextflow_cli_args // array: List of positional nextflow CLI args @@ -54,16 +53,8 @@ workflow PIPELINE_INITIALISATION { // // Validate parameters and generate parameter summary to stdout // - pre_help_text = nfCoreLogo(monochrome_logs) - post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) - def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " UTILS_NFVALIDATION_PLUGIN ( - help, - workflow_command, - pre_help_text, - post_help_text, - validate_params, - "nextflow_schema.json" + validate_params ) // @@ -84,7 +75,7 @@ workflow PIPELINE_INITIALISATION { // Create channel from input file provided through params.input // Channel - .fromSamplesheet("input") + .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")) .map { meta, fastq_1, fastq_2 -> if (!fastq_2) { diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 14558c3927..a5dcd5f83d 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -61,25 +61,6 @@ def checkProfileProvided(nextflow_cli_args) { } } -// -// Citation string for pipeline -// -def workflowCitation() { - def temp_doi_ref = "" - String[] manifest_doi = workflow.manifest.doi.tokenize(",") - // Using a loop to handle multiple DOIs - // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers - // Removing ` ` since the manifest.doi is a string and not a proper list - for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" - return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + - "* The pipeline\n" + - temp_doi_ref + "\n" + - "* The nf-core framework\n" + - " https://doi.org/10.1038/s41587-020-0439-x\n\n" + - "* Software dependencies\n" + - " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" -} - // // Generate workflow version string // @@ -157,33 +138,6 @@ def paramsSummaryMultiqc(summary_params) { return yaml_file_text } -// -// nf-core logo -// -def nfCoreLogo(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) - String.format( - """\n - ${dashedLine(monochrome_logs)} - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} - ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} - ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} - ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} - ${dashedLine(monochrome_logs)} - """.stripIndent() - ) -} - -// -// Return dashed line -// -def dashedLine(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) - return "-${colors.dim}----------------------------------------------------${colors.reset}-" -} - // // ANSII colours used for terminal logging // diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf index 2585b65d1b..453894a64a 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf @@ -8,9 +8,8 @@ ======================================================================================== */ -include { paramsHelp } from 'plugin/nf-validation' -include { paramsSummaryLog } from 'plugin/nf-validation' -include { validateParameters } from 'plugin/nf-validation' +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' /* ======================================================================================== @@ -21,12 +20,7 @@ include { validateParameters } from 'plugin/nf-validation' workflow UTILS_NFVALIDATION_PLUGIN { take: - print_help // boolean: print help - workflow_command // string: default commmand used to run pipeline - pre_help_text // string: string to be printed before help text and summary log - post_help_text // string: string to be printed after help text and summary log validate_params // boolean: validate parameters - schema_filename // path: JSON schema file, null to use default value main: @@ -37,24 +31,11 @@ workflow UTILS_NFVALIDATION_PLUGIN { post_help_text = post_help_text ?: '' workflow_command = workflow_command ?: '' - // - // Print help message if needed - // - if (print_help) { - log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text - System.exit(0) - } - - // - // Print parameter summary to stdout - // - log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text - // // Validate parameters relative to the parameter JSON schema // if (validate_params){ - validateParameters(parameters_schema: schema_filename) + validateParameters() } emit: diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test index 5784a33f2f..354b817545 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test @@ -10,30 +10,21 @@ nextflow_workflow { tag "utils_nfvalidation_plugin" tag "subworkflows/utils_nfvalidation_plugin" + config "./nextflow.config" + test("Should run nothing") { when { params { - monochrome_logs = true test_data = '' } workflow { """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename + input[0] = validate_params """ } } @@ -45,147 +36,19 @@ nextflow_workflow { } } - test("Should run help") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with command") { - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with extra text") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = "pre-help-text" - post_help_text = "post-help-text" - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('pre-help-text') } }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } }, - { assert workflow.stdout.any { it.contains('post-help-text') } } - ) - } - } - test("Should validate params") { when { params { - monochrome_logs = true test_data = '' outdir = 1 } workflow { """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null validate_params = true - schema_filename = "$moduleTestDir/nextflow_schema.json" - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename + input[0] = validate_params """ } } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config new file mode 100644 index 0000000000..8d047ed59e --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config @@ -0,0 +1 @@ +validation.monochromeLogs = true \ No newline at end of file diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index d98c392f0c..4bb0913211 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -6,7 +6,7 @@ include { FASTQC } from '../modules/nf-core/fastqc/main' {% if multiqc %}include { MULTIQC } from '../modules/nf-core/multiqc/main'{% endif %} -include { paramsSummaryMap } from 'plugin/nf-validation' +include { paramsSummaryMap } from 'plugin/nf-schema' {% if multiqc %}include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline'{% endif %} include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' {% if citations or multiqc %}include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'{% endif %} From 5519c83d5b582af797ff4ae242fa98113bbe847a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 15:24:51 +0200 Subject: [PATCH 16/63] some small updates for the new release --- nf_core/pipeline-template/conf/colors.config | 61 ------------------- nf_core/pipeline-template/modules.json | 4 +- nf_core/pipeline-template/nextflow.config | 22 ++++--- .../nf-core/utils_nfschema_plugin/main.nf | 32 ++++++++++ .../nf-core/utils_nfschema_plugin/meta.yml | 22 +++++++ .../tests/main.nf.test | 27 ++++---- .../tests/nextflow.config | 8 +++ .../tests/nextflow_schema.json | 8 +-- .../nf-core/utils_nfvalidation_plugin/main.nf | 43 ------------- .../utils_nfvalidation_plugin/meta.yml | 44 ------------- .../tests/nextflow.config | 1 - .../utils_nfvalidation_plugin/tests/tags.yml | 2 - 12 files changed, 93 insertions(+), 181 deletions(-) delete mode 100644 nf_core/pipeline-template/conf/colors.config create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml rename nf_core/pipeline-template/subworkflows/nf-core/{utils_nfvalidation_plugin => utils_nfschema_plugin}/tests/main.nf.test (62%) create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config rename nf_core/pipeline-template/subworkflows/nf-core/{utils_nfvalidation_plugin => utils_nfschema_plugin}/tests/nextflow_schema.json (95%) delete mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf delete mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml delete mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config delete mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml diff --git a/nf_core/pipeline-template/conf/colors.config b/nf_core/pipeline-template/conf/colors.config deleted file mode 100644 index 00f2963267..0000000000 --- a/nf_core/pipeline-template/conf/colors.config +++ /dev/null @@ -1,61 +0,0 @@ -// For now, there is no easy way to set monochromeLogs -colors { - // Reset / Meta - reset = "\033[0m" - bold = "\033[1m" - dim = "\033[2m" - underlined = "\033[4m" - blink = "\033[5m" - reverse = "\033[7m" - hidden = "\033[8m" - - // Regular Colors - black = "\033[0;30m" - red = "\033[0;31m" - green = "\033[0;32m" - yellow = "\033[0;33m" - blue = "\033[0;34m" - purple = "\033[0;35m" - cyan = "\033[0;36m" - white = "\033[0;37m" - - // Bold - bblack = "\033[1;30m" - bred = "\033[1;31m" - bgreen = "\033[1;32m" - byellow = "\033[1;33m" - bblue = "\033[1;34m" - bpurple = "\033[1;35m" - bcyan = "\033[1;36m" - bwhite = "\033[1;37m" - - // Underline - ublack = "\033[4;30m" - ured = "\033[4;31m" - ugreen = "\033[4;32m" - uyellow = "\033[4;33m" - ublue = "\033[4;34m" - upurple = "\033[4;35m" - ucyan = "\033[4;36m" - uwhite = "\033[4;37m" - - // High Intensity - iblack = "\033[0;90m" - ired = "\033[0;91m" - igreen = "\033[0;92m" - iyellow = "\033[0;93m" - iblue = "\033[0;94m" - ipurple = "\033[0;95m" - icyan = "\033[0;96m" - iwhite = "\033[0;97m" - - // Bold High Intensity - biblack = "\033[1;90m" - bired = "\033[1;91m" - bigreen = "\033[1;92m" - biyellow = "\033[1;93m" - biblue = "\033[1;94m" - bipurple = "\033[1;95m" - bicyan = "\033[1;96m" - biwhite = "\033[1;97m" -} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index eb9391b29a..1a3b96490d 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -30,9 +30,9 @@ "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", "installed_by": ["subworkflows"] }, - "utils_nfvalidation_plugin": { + "utils_nfschema_plugin": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "a3e87febb28bd0461c22a917c5b2c1492053ef85", "installed_by": ["subworkflows"] } } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3d3841ba49..346292c980 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -265,22 +265,20 @@ plugins { id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } -includeConfig "conf/colors.config" - validation { parametersSchema = "${projectDir}/nextflow_schema.json" help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " beforeText = """ --${colors.dim}----------------------------------------------------${colors.reset}- - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} -${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} -${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} -${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} -${colors.purple} ${manifest.name} ${manifest.version}${colors.reset} --${colors.dim}----------------------------------------------------${colors.reset}- +-\033[2m----------------------------------------------------\033[0m- + \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m +\033[0;34m ___ __ __ __ ___ \033[0;32m/,-._.--~\'\033[0m +\033[0;34m |\\ | |__ __ / ` / \\ |__) |__ \033[0;33m} {\033[0m +\033[0;34m | \\| | \\__, \\__/ | \\ |___ \033[0;32m\\`-._,-`-,\033[0m + \033[0;32m`._,._,\'\033[0m +\033[0;35m ${manifest.name} ${manifest.version}\033[0m +-\033[2m----------------------------------------------------\033[0m- """ afterText = """${manifest.doi ? "* The pipeline\n" : ""}${manifest.doi.tokenize(",").collect { " https://doi.org/${it.trim().replace('https://doi.org/','')}"}.join("\n")}${manifest.doi ? "\n" : ""} * The nf-core framework @@ -290,6 +288,10 @@ ${colors.purple} ${manifest.name} ${manifest.version}${colors.reset} https://github.com/${manifest.name}/blob/master/CITATIONS.md """ } + summary { + beforeText = validation.help.beforeText + afterText = validation.help.afterText + } } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf new file mode 100644 index 0000000000..5d1dc7c928 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -0,0 +1,32 @@ +// +// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary +// + +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' + +workflow UTILS_NFSCHEMA_PLUGIN { + + take: + validate_params // boolean: validate the parameters + + main: + + // + // Print parameter summary to stdout. This will display the parameters + // that differ from the default given in the JSON schema + // + log.info paramsSummaryLog(workflow) + + // + // Validate the parameters using nextflow_schema.json or the schema + // given via the validation.parametersSchema configuration option + // + if(validate_params) { + validateParameters() + } + + emit: + dummy_emit = true +} + diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml new file mode 100644 index 0000000000..90b8cb1898 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -0,0 +1,22 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "utils_nfschema_plugin" +description: Run nf-schema to validate parameters and create a summary of changed parameters +keywords: + - validation + - JSON schema + - plugin + - parameters + - summary +components: [] +input: + - validate_params: + type: boolean + description: Validate the parameters and error if invalid. +output: + - dummy_emit: + type: boolean + description: Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nvnieuwk" +maintainers: + - "@nvnieuwk" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test similarity index 62% rename from nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test rename to nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test index 354b817545..703d3a9b86 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -1,14 +1,14 @@ +// TODO nf-core: Once you have added the required tests, please run the following command to build this file: +// nf-core subworkflows test utils_nfschema_plugin nextflow_workflow { - name "Test Workflow UTILS_NFVALIDATION_PLUGIN" + name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" script "../main.nf" - workflow "UTILS_NFVALIDATION_PLUGIN" + workflow "UTILS_NFSCHEMA_PLUGIN" + tag "subworkflows" tag "subworkflows_nfcore" - tag "plugin/nf-validation" - tag "'plugin/nf-validation'" - tag "utils_nfvalidation_plugin" - tag "subworkflows/utils_nfvalidation_plugin" + tag "subworkflows/utils_nfschema_plugin" config "./nextflow.config" @@ -17,13 +17,12 @@ nextflow_workflow { when { params { - test_data = '' + test_data = '' } workflow { """ - validate_params = false - + validate_params = false input[0] = validate_params """ } @@ -41,13 +40,13 @@ nextflow_workflow { when { params { - test_data = '' - outdir = 1 + test_data = '' + outdir = 1 } + workflow { """ - validate_params = true - + validate_params = true input[0] = validate_params """ } @@ -56,7 +55,7 @@ nextflow_workflow { then { assertAll( { assert workflow.failed }, - { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } ) } } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config new file mode 100644 index 0000000000..0907ac58f0 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -0,0 +1,8 @@ +plugins { + id "nf-schema@2.1.0" +} + +validation { + parametersSchema = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + monochromeLogs = true +} \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json similarity index 95% rename from nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json rename to nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json index 7626c1c93e..331e0d2f44 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", "title": ". pipeline parameters", "description": "", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -87,10 +87,10 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf deleted file mode 100644 index 453894a64a..0000000000 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf +++ /dev/null @@ -1,43 +0,0 @@ -// -// Subworkflow that uses the nf-validation plugin to render help text and parameter summary -// - -/* -======================================================================================== - IMPORT NF-VALIDATION PLUGIN -======================================================================================== -*/ - -include { paramsSummaryLog } from 'plugin/nf-schema' -include { validateParameters } from 'plugin/nf-schema' - -/* -======================================================================================== - SUBWORKFLOW DEFINITION -======================================================================================== -*/ - -workflow UTILS_NFVALIDATION_PLUGIN { - - take: - validate_params // boolean: validate parameters - - main: - - log.debug "Using schema file: ${schema_filename}" - - // Default values for strings - pre_help_text = pre_help_text ?: '' - post_help_text = post_help_text ?: '' - workflow_command = workflow_command ?: '' - - // - // Validate parameters relative to the parameter JSON schema - // - if (validate_params){ - validateParameters() - } - - emit: - dummy_emit = true -} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml deleted file mode 100644 index 3d4a6b04f5..0000000000 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml +++ /dev/null @@ -1,44 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "UTILS_NFVALIDATION_PLUGIN" -description: Use nf-validation to initiate and validate a pipeline -keywords: - - utility - - pipeline - - initialise - - validation -components: [] -input: - - print_help: - type: boolean - description: | - Print help message and exit - - workflow_command: - type: string - description: | - The command to run the workflow e.g. "nextflow run main.nf" - - pre_help_text: - type: string - description: | - Text to print before the help message - - post_help_text: - type: string - description: | - Text to print after the help message - - validate_params: - type: boolean - description: | - Validate the parameters and error if invalid. - - schema_filename: - type: string - description: | - The filename of the schema to validate against. -output: - - dummy_emit: - type: boolean - description: | - Dummy emit to make nf-core subworkflows lint happy -authors: - - "@adamrtalbot" -maintainers: - - "@adamrtalbot" - - "@maxulysse" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config deleted file mode 100644 index 8d047ed59e..0000000000 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow.config +++ /dev/null @@ -1 +0,0 @@ -validation.monochromeLogs = true \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml deleted file mode 100644 index 60b1cfff49..0000000000 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -subworkflows/utils_nfvalidation_plugin: - - subworkflows/nf-core/utils_nfvalidation_plugin/** From 171640b62dddcb81feddced4f674df88c47654ab Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 16:13:59 +0200 Subject: [PATCH 17/63] some more small updates to the template --- nf_core/pipeline-template/nextflow.config | 2 +- .../subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 346292c980..ef067dcfab 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -255,7 +255,7 @@ manifest { homePage = 'https://github.com/{{ name }}' description = """{{ description }}""" mainScript = 'main.nf' - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.10.0' version = '{{ version }}' doi = '' } diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index c83d2ae0cb..a994cfd584 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -8,7 +8,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' +include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' include { paramsSummaryMap } from 'plugin/nf-schema' include { samplesheetToList } from 'plugin/nf-schema' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' @@ -53,7 +53,7 @@ workflow PIPELINE_INITIALISATION { // // Validate parameters and generate parameter summary to stdout // - UTILS_NFVALIDATION_PLUGIN ( + UTILS_NFSCHEMA_PLUGIN ( validate_params ) From 249b3da383f17ab0beecfeb9e8346619edaf6776 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 16:59:22 +0200 Subject: [PATCH 18/63] update handling of ignored parameters --- .../.github/workflows/ci.yml | 2 +- nf_core/pipeline-template/README.md | 2 +- nf_core/pipelines/lint/nextflow_config.py | 15 +++++-------- nf_core/pipelines/schema.py | 22 ++++++++++++++----- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 6b2547765d..63fa99cec8 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: Check out pipeline code diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 7718d2e5f8..d31dee93f4 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -16,7 +16,7 @@ [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index a5b7cf325f..150a89e102 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -65,14 +65,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: * Should always be set to default value: ``https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}`` - * ``params.validationShowHiddenParams`` - - * Determines whether boilerplate params are showed by schema. Set to ``false`` by default - - * ``params.validationSchemaIgnoreParams`` - - * A comma separated string of inputs the schema validation should ignore. - **The following variables throw warnings if missing:** * ``manifest.mainScript``: The filename of the main pipeline script (should be ``main.nf``) @@ -151,8 +143,11 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["process.time"], ["params.outdir"], ["params.input"], - ["params.validationShowHiddenParams"], - ["params.validationSchemaIgnoreParams"], + ["validation.help.enabled"], + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"] ] # Throw a warning if these are missing config_warn = [ diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 9db9457516..78f4317094 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -50,6 +50,7 @@ def __init__(self): self.validation_plugin = None self.schema_draft = None self.defs_notation = None + self.ignored_params = [] # Update the validation plugin code everytime the schema gets changed def set_schema_filename(self, schema: str) -> None: @@ -69,16 +70,26 @@ def del_schema_filename(self) -> None: def _update_validation_plugin_from_config(self, config: str) -> None: plugin = "nf-schema" with open(Path(config)) as conf: - nf_schema_pattern = re.compile("id\s*[\"']nf-schema", re.MULTILINE) - nf_validation_pattern = re.compile("id\s*[\"']nf-validation", re.MULTILINE) + nf_schema_pattern = re.compile(r"id\s*[\"']nf-schema", re.MULTILINE) + nf_validation_pattern = re.compile(r"id\s*[\"']nf-validation", re.MULTILINE) config_content = conf.read() if re.search(nf_validation_pattern, config_content): plugin = "nf-validation" + self.ignored_params = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") + self.ignored_params.append("validationSchemaIgnoreParams") elif re.search(nf_schema_pattern, config_content): plugin = "nf-schema" + ignored_params_pattern = re.compile(r"defaultIgnoreParams\s*=\s*\[([^\]]*)\]", re.MULTILINE) + ignored_params_match = re.search(ignored_params_pattern, config_content) + ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default + if ignored_params_match and len(ignored_params_match.groups()) == 1: + ignored_params.extend(ignored_params_match.group(1).replace("\"", "").replace("'", '').replace(" ", "").split(",")) + self.ignored_params = ignored_params else: log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") + + self.validation_plugin = plugin # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": @@ -845,13 +856,12 @@ def add_schema_found_configs(self): Update defaults if they have changed """ params_added = [] - params_ignore = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") - params_ignore.append("validationSchemaIgnoreParams") + for p_key, p_val in self.pipeline_params.items(): s_key = self.schema_params.get(p_key) # Check if key is in schema parameters # Key is in pipeline but not in schema or ignored from schema - if p_key not in self.schema_params and p_key not in params_ignore: + if p_key not in self.schema_params and p_key not in self.ignored_params: if ( self.no_prompts or self.schema_from_scratch @@ -884,7 +894,7 @@ def add_schema_found_configs(self): elif ( s_key and (p_key not in self.schema_defaults) - and (p_key not in params_ignore) + and (p_key not in self.ignored_params) and (p_def := self.build_schema_param(p_val).get("default")) ): if self.no_prompts or Confirm.ask( From af34d0718db98544a8da402952aea87ca778a60e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 20 Aug 2024 17:50:32 +0200 Subject: [PATCH 19/63] update validation plugin fetching --- nf_core/pipeline-template/nextflow.config | 2 +- nf_core/pipelines/schema.py | 58 ++++++++++++----------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ef067dcfab..3a98c40f0a 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -61,7 +61,7 @@ params { max_time = '240.h' // Schema validation default options - validate_params = true + validate_params = true } diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 78f4317094..e9c5556dbb 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -69,35 +69,45 @@ def del_schema_filename(self) -> None: def _update_validation_plugin_from_config(self, config: str) -> None: plugin = "nf-schema" - with open(Path(config)) as conf: - nf_schema_pattern = re.compile(r"id\s*[\"']nf-schema", re.MULTILINE) - nf_validation_pattern = re.compile(r"id\s*[\"']nf-validation", re.MULTILINE) - config_content = conf.read() - if re.search(nf_validation_pattern, config_content): - plugin = "nf-validation" - self.ignored_params = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") - self.ignored_params.append("validationSchemaIgnoreParams") - elif re.search(nf_schema_pattern, config_content): + conf = nf_core.utils.fetch_wf_config(Path(self.schema_filename).parent) + + plugins = str(conf.get("plugins", "")).strip("\"").strip("'").strip(" ").split(",") + plugin_found = False + for plugin_instance in plugins: + if "nf-schema" in plugin_instance: plugin = "nf-schema" - ignored_params_pattern = re.compile(r"defaultIgnoreParams\s*=\s*\[([^\]]*)\]", re.MULTILINE) - ignored_params_match = re.search(ignored_params_pattern, config_content) - ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default - if ignored_params_match and len(ignored_params_match.groups()) == 1: - ignored_params.extend(ignored_params_match.group(1).replace("\"", "").replace("'", '').replace(" ", "").split(",")) - self.ignored_params = ignored_params - else: - log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") - + plugin_found = True + break + elif "nf-validation" in plugin_instance: + plugin = "nf-validation" + plugin_found = True + break + + if not plugin_found: + log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") + + if "nf-validation" in plugins: + plugin = "nf-validation" + elif "nf-schema" in plugins: + plugin = "nf-schema" - self.validation_plugin = plugin # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": self.defs_notation = "$defs" + ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default + ignored_params_config = conf.get("validation", {}).get("defaultIgnoreParams", []) + if len(ignored_params_config) > 0: + ignored_params.extend(ignored_params_config) + self.ignored_params = ignored_params self.schema_draft = "https://json-schema.org/draft/2020-12/schema" + else: self.defs_notation = "definitions" self.schema_draft = "https://json-schema.org/draft-07/schema" + self.get_wf_params() + self.ignored_params = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") + self.ignored_params.append("validationSchemaIgnoreParams") def get_schema_path( self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None @@ -325,17 +335,11 @@ def validate_default_params(self): if self.pipeline_params == {}: self.get_wf_params() - # Collect parameters to ignore - if "validationSchemaIgnoreParams" in self.pipeline_params: - params_ignore = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",") - else: - params_ignore = [] - # Go over group keys for group_key, group in schema_no_required.get(self.defs_notation, {}).items(): group_properties = group.get("properties") for param in group_properties: - if param in params_ignore: + if param in self.ignored_params: continue if param in self.pipeline_params: self.validate_config_default_parameter(param, group_properties[param], self.pipeline_params[param]) @@ -348,7 +352,7 @@ def validate_default_params(self): ungrouped_properties = self.schema.get("properties") if ungrouped_properties: for param in ungrouped_properties: - if param in params_ignore: + if param in self.ignored_params: continue if param in self.pipeline_params: self.validate_config_default_parameter( From e6381b71c67fcc1ecb18c8b3c6e796dd6df0dec7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 21 Aug 2024 10:59:31 +0200 Subject: [PATCH 20/63] update plugin linting --- nf_core/pipelines/lint/nextflow_config.py | 56 ++++++++--------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 150a89e102..bd2f4f768b 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -1,3 +1,4 @@ +import ast import logging import re from pathlib import Path @@ -330,6 +331,25 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) ) + # Lint for plugins + config_plugins = ast.literal_eval(self.nf_config.get("plugins", "").strip("\"")) + found_plugins = [] + if len(config_plugins) == 0: + failed.append("nextflow.config contains an empty plugins scope") + for plugin in config_plugins: + if "@" not in plugin: + failed.append(f"Plugin '{plugin}' does not have a pinned version") + found_plugins.append(plugin.split("@")[0]) + + if "nf-validation" in found_plugins and "nf-schema" in found_plugins: + failed.append("nextflow.config contains both nf-validation and nf-schema") + if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: + failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + + if "nf-validation" in found_plugins: + warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") + + # Check for the availability of the "test" configuration profile by parsing nextflow.config # Also check for the presence of nf-validation/nf-schema and check if they have pinned versions with open(Path(self.wf_path, "nextflow.config")) as f: @@ -359,42 +379,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: else: failed.append("nextflow.config does not contain configuration profile `test`") - # Lint for nf-validation and nf-schema - match_plugins = re.search(r"\bplugins\s*\{([^}]+)}", cleaned_content, re.MULTILINE) - if not match_plugins: - failed.append( - "nextflow.config does not contain `plugins` scope, but `nf-validation` or `nf-schema` plugins are required" - ) - else: - found_plugins = {} - for line in match_plugins.group(1).split("\n"): - cleaned_line = line.split("//")[0].strip().replace("\"", "'") - if "id" not in line: continue - match_line = re.search(r"\bid\s'([^']+)'", cleaned_line) - if not match_line: - failed.append(f"nextflow.config contains an invalid plugins identifier: {cleaned_line}") - continue - plugin = match_line.group(1) - name = plugin.split("@")[0] - version = "" - if "@" in plugin: - version = plugin.split("@")[1] - found_plugins[name] = version - - if len(found_plugins) == 0: - failed.append("nextflow.config contains an empty plugins scope") - elif "nf-validation" in found_plugins and "nf-schema" in found_plugins: - failed.append("nextflow.config contains both nf-validation and nf-schema") - elif "nf-validation" in found_plugins and found_plugins["nf-validation"] == "": - failed.append("nextflow.config contains an unpinned version of nf-validation") - elif "nf-schema" in found_plugins and found_plugins["nf-schema"] == "": - failed.append("nextflow.config contains an unpinned version of nf-schema") - elif "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: - failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") - - if "nf-validation" in found_plugins: - warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") - # Check that the default values in nextflow.config match the default values defined in the nextflow_schema.json ignore_defaults = [] for item in ignore_configs: From e66a732e7db6452fb12b0f311762b19bbe7f3418 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 21 Aug 2024 11:01:16 +0200 Subject: [PATCH 21/63] ignore genomes for now --- nf_core/pipeline-template/nextflow.config | 2 +- nf_core/pipeline-template/nextflow_schema.json | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3a98c40f0a..bb55678001 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -266,7 +266,7 @@ plugins { } validation { - parametersSchema = "${projectDir}/nextflow_schema.json" + defaultIgnoreParams = ["genomes"] help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 14120bbabc..908d310171 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -81,12 +81,6 @@ "fa_icon": "fas fa-ban", "hidden": true, "default": "s3://ngi-igenomes/igenomes/" - }, - "genomes": { - "type": "object", - "description": "An object containing all reference data availabe in igenomes", - "fa_icon": "fas fa-ban", - "hidden": true } } }, From b6ae8d01dd440558cb3a2af821a04ea8fc89591a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 21 Aug 2024 12:23:52 +0200 Subject: [PATCH 22/63] improve config linting for validation even more --- nf_core/pipelines/lint/nextflow_config.py | 59 ++++++++++++++--------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index bd2f4f768b..6e292deb09 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -144,11 +144,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["process.time"], ["params.outdir"], ["params.input"], - ["validation.help.enabled"], - ["validation.help.beforeText"], - ["validation.help.afterText"], - ["validation.summary.beforeText"], - ["validation.summary.afterText"] + ["validation.help.enabled"] ] # Throw a warning if these are missing config_warn = [ @@ -157,6 +153,11 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["trace.file"], ["report.file"], ["dag.file"], + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.help.command"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"] ] # Old depreciated vars - fail if present config_fail_ifdefined = [ @@ -168,6 +169,35 @@ def nextflow_config(self) -> Dict[str, List[str]]: "params.enable_conda", ] + # Lint for plugins + config_plugins = ast.literal_eval(self.nf_config.get("plugins", "").strip("\"")) + found_plugins = [] + if len(config_plugins) == 0: + failed.append("nextflow.config contains an empty plugins scope") + for plugin in config_plugins: + if "@" not in plugin: + failed.append(f"Plugin '{plugin}' does not have a pinned version") + found_plugins.append(plugin.split("@")[0]) + + if "nf-validation" in found_plugins and "nf-schema" in found_plugins: + failed.append("nextflow.config contains both nf-validation and nf-schema") + if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: + failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + + if "nf-schema" in found_plugins: + if self.nf_config.get("validation.help.enabled", "false") == "false": + failed.append("The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages") + config_fail_ifdefined.extend([ + "params.validationFailUnrecognisedParams", + "params.validationLenientMode", + "params.validationSchemaIgnoreParams", + "params.validationShowHiddenParams" + ]) + + if "nf-validation" in found_plugins: + warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") + + # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get("nextflow_config", []) if self.lint_config is not None else [] @@ -331,25 +361,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) ) - # Lint for plugins - config_plugins = ast.literal_eval(self.nf_config.get("plugins", "").strip("\"")) - found_plugins = [] - if len(config_plugins) == 0: - failed.append("nextflow.config contains an empty plugins scope") - for plugin in config_plugins: - if "@" not in plugin: - failed.append(f"Plugin '{plugin}' does not have a pinned version") - found_plugins.append(plugin.split("@")[0]) - - if "nf-validation" in found_plugins and "nf-schema" in found_plugins: - failed.append("nextflow.config contains both nf-validation and nf-schema") - if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: - failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") - - if "nf-validation" in found_plugins: - warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") - - # Check for the availability of the "test" configuration profile by parsing nextflow.config # Also check for the presence of nf-validation/nf-schema and check if they have pinned versions with open(Path(self.wf_path, "nextflow.config")) as f: From 1be17370b30fe34adccb5b698d26145e05522e99 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 21 Aug 2024 14:56:37 +0200 Subject: [PATCH 23/63] check plugin includes in pipeline --- nf_core/pipelines/lint/__init__.py | 3 ++ nf_core/pipelines/lint/nextflow_config.py | 4 ++- nf_core/pipelines/lint/plugin_includes.py | 38 +++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 nf_core/pipelines/lint/plugin_includes.py diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index ed833d3219..219cdd8f5c 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -45,6 +45,7 @@ from .nfcore_yml import nfcore_yml from .pipeline_name_conventions import pipeline_name_conventions from .pipeline_todos import pipeline_todos +from .plugin_includes import plugin_includes from .readme import readme from .schema_description import schema_description from .schema_lint import schema_lint @@ -92,6 +93,7 @@ class PipelineLint(nf_core.utils.Pipeline): nfcore_yml = nfcore_yml pipeline_name_conventions = pipeline_name_conventions pipeline_todos = pipeline_todos + plugin_includes = plugin_includes readme = readme schema_description = schema_description schema_lint = schema_lint @@ -135,6 +137,7 @@ def _get_all_lint_tests(release_mode): "actions_awsfulltest", "readme", "pipeline_todos", + "plugin_includes", "pipeline_name_conventions", "template_strings", "schema_lint", diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 6e292deb09..a735c5042b 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -170,7 +170,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ] # Lint for plugins - config_plugins = ast.literal_eval(self.nf_config.get("plugins", "").strip("\"")) + config_plugins = ast.literal_eval(self.nf_config.get("plugins", "")) found_plugins = [] if len(config_plugins) == 0: failed.append("nextflow.config contains an empty plugins scope") @@ -185,6 +185,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") if "nf-schema" in found_plugins: + passed.append("Found nf-schema plugin") if self.nf_config.get("validation.help.enabled", "false") == "false": failed.append("The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages") config_fail_ifdefined.extend([ @@ -195,6 +196,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ]) if "nf-validation" in found_plugins: + passed.append("Found nf-validation plugin") warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") diff --git a/nf_core/pipelines/lint/plugin_includes.py b/nf_core/pipelines/lint/plugin_includes.py new file mode 100644 index 0000000000..247c4e493b --- /dev/null +++ b/nf_core/pipelines/lint/plugin_includes.py @@ -0,0 +1,38 @@ +import ast +import glob +import logging +import re +from typing import Dict, List + +log = logging.getLogger(__name__) + + +def plugin_includes(self) -> Dict[str, List[str]]: + """Checks the include statements in the all *.nf files for plugin includes + + When nf-schema is used in an nf-core pipeline, the include statements of the plugin + functions have to use nf-schema instead of nf-validation and vice versa + """ + config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", ""))] + validation_plugin = "nf-validation" if "nf-validation" in config_plugins else "nf-schema" + + passed = [] + warned = [] + failed = [] + ignored = [] + + plugin_include_pattern = re.compile(r"^include\s*{[^}]+}\s*from\s*[\"']plugin/([^\"']+)[\"']\s*$", re.MULTILINE) + workflow_files = [file for file in glob.glob(f"{self.wf_path}/**/*.nf", recursive=True) if not file.startswith("./modules/")] + test_passed = True + for file in workflow_files: + with open(file, "r") as of: + plugin_includes = re.findall(plugin_include_pattern, of.read()) + for include in plugin_includes: + if include not in ["nf-validation", "nf-schema"]: continue + if include != validation_plugin: + test_passed = False + failed.append(f"Found a `{include}` plugin import in `{file[2:]}`, but `{validation_plugin}` was used in `nextflow.config`") + + if test_passed: passed.append("No wrong validation plugin imports have been found") + + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} From 2f1a4428dc73e15aea2fc001bec4df323700035e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 14:51:07 +0200 Subject: [PATCH 24/63] make nf-schema optional in the template --- .../assets/schema_input.json | 2 +- nf_core/pipeline-template/main.nf | 1 - nf_core/pipeline-template/modules.json | 4 +- nf_core/pipeline-template/nextflow.config | 9 ++-- .../pipeline-template/nextflow_schema.json | 3 +- .../utils_nfcore_pipeline_pipeline/main.nf | 18 +++++--- .../nf-core/utils_nfcore_pipeline/main.nf | 46 +++++++++++++++++++ .../pipeline-template/workflows/pipeline.nf | 2 +- nf_core/pipelines/create/templatefeatures.yml | 10 ++++ nf_core/pipelines/lint/nextflow_config.py | 20 ++++---- 10 files changed, 90 insertions(+), 25 deletions(-) diff --git a/nf_core/pipeline-template/assets/schema_input.json b/nf_core/pipeline-template/assets/schema_input.json index e76b95fa99..28a468adaf 100644 --- a/nf_core/pipeline-template/assets/schema_input.json +++ b/nf_core/pipeline-template/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/{{ name }}/master/assets/schema_input.json", "title": "{{ name }} pipeline - params.input schema", "description": "Schema for the file provided with params.input", diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index fddfc5489e..f644f52afc 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -76,7 +76,6 @@ workflow { // PIPELINE_INITIALISATION ( params.version, - params.help, params.validate_params, params.monochrome_logs, args, diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 1a3b96490d..0a883b60b7 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -29,12 +29,12 @@ "branch": "master", "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", "installed_by": ["subworkflows"] - }, + }{% if nf_schema %}, "utils_nfschema_plugin": { "branch": "master", "git_sha": "a3e87febb28bd0461c22a917c5b2c1492053ef85", "installed_by": ["subworkflows"] - } + }{% endif %} } } } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index bb55678001..4acc34e409 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -37,9 +37,9 @@ params { plaintext_email = false monochrome_logs = false hook_url = null - help = false + {% if nf_schema %}help = false helpFull = false - showHidden = false + showHidden = false{% endif %} version = false pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' @@ -260,13 +260,14 @@ manifest { doi = '' } +{% if nf_schema -%} // Nextflow plugins plugins { id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { - defaultIgnoreParams = ["genomes"] + defaultIgnoreParams = ["genomes", "helpFull", "showHidden", "help-full", "show-hidden"] // The last 4 parameters are here because of a bug in nf-schema. This will be fixed in a later version help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " @@ -293,7 +294,7 @@ validation { afterText = validation.help.afterText } } - +{% endif -%} // Load modules.config for DSL2 module specific options includeConfig 'conf/modules.config' diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 908d310171..c729c370bf 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -40,8 +40,7 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" - } - {% endif %} + }{% endif %} } }, {%- if igenomes %} diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index a994cfd584..1fef6912b3 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -8,17 +8,14 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' +{% if nf_schema %}include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' include { paramsSummaryMap } from 'plugin/nf-schema' -include { samplesheetToList } from 'plugin/nf-schema' -include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +include { samplesheetToList } from 'plugin/nf-schema'{% endif %} include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' -include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' -include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' include { imNotification } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' -include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' /* ======================================================================================== @@ -50,12 +47,14 @@ workflow PIPELINE_INITIALISATION { workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 ) + {% if nf_schema %} // // Validate parameters and generate parameter summary to stdout // UTILS_NFSCHEMA_PLUGIN ( validate_params ) + {% endif %} // // Check config provided to the pipeline @@ -74,6 +73,7 @@ workflow PIPELINE_INITIALISATION { // // Create channel from input file provided through params.input // + {% if nf_schema %} Channel .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")) .map { @@ -93,6 +93,12 @@ workflow PIPELINE_INITIALISATION { return [ meta, fastqs.flatten() ] } .set { ch_samplesheet } + {% else %} + Channel + .fromPath(params.input) + .splitCsv(header: true, strip: true) + .set { ch_samplesheet } + {% endif %} emit: samplesheet = ch_samplesheet diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a5dcd5f83d..14558c3927 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -61,6 +61,25 @@ def checkProfileProvided(nextflow_cli_args) { } } +// +// Citation string for pipeline +// +def workflowCitation() { + def temp_doi_ref = "" + String[] manifest_doi = workflow.manifest.doi.tokenize(",") + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" + return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + + "* The pipeline\n" + + temp_doi_ref + "\n" + + "* The nf-core framework\n" + + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + + "* Software dependencies\n" + + " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" +} + // // Generate workflow version string // @@ -138,6 +157,33 @@ def paramsSummaryMultiqc(summary_params) { return yaml_file_text } +// +// nf-core logo +// +def nfCoreLogo(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + String.format( + """\n + ${dashedLine(monochrome_logs)} + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} + ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} + ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} + ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} + ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} + ${dashedLine(monochrome_logs)} + """.stripIndent() + ) +} + +// +// Return dashed line +// +def dashedLine(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" +} + // // ANSII colours used for terminal logging // diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 4bb0913211..8d2773820d 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -6,7 +6,7 @@ include { FASTQC } from '../modules/nf-core/fastqc/main' {% if multiqc %}include { MULTIQC } from '../modules/nf-core/multiqc/main'{% endif %} -include { paramsSummaryMap } from 'plugin/nf-schema' +{% if nf_schema %}include { paramsSummaryMap } from 'plugin/nf-schema'{% endif %} {% if multiqc %}include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline'{% endif %} include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' {% if citations or multiqc %}include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'{% endif %} diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index b97bf347ab..7a24b3d91a 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -242,3 +242,13 @@ changelog: - "CHANGELOG.md" nfcore_pipelines: False custom_pipelines: True +nf_schema: + skippable_paths: + - "subworkflows/nf-core/utils_nfschema_plugin" + short_description: "Use nf-schema for this pipeline." + help_text: | + nf-schema is used to validate input parameters based on a JSON schema. + It also provides helper functionality to create help messages, get a summary + of changed parameters and validate and convert a samplesheet to a channel. + nfcore_pipelines: False + custom_pipelines: False diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index a735c5042b..1dfdabaf10 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -143,8 +143,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["process.memory"], ["process.time"], ["params.outdir"], - ["params.input"], - ["validation.help.enabled"] + ["params.input"] ] # Throw a warning if these are missing config_warn = [ @@ -152,12 +151,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["timeline.file"], ["trace.file"], ["report.file"], - ["dag.file"], - ["validation.help.beforeText"], - ["validation.help.afterText"], - ["validation.help.command"], - ["validation.summary.beforeText"], - ["validation.summary.afterText"] + ["dag.file"] ] # Old depreciated vars - fail if present config_fail_ifdefined = [ @@ -188,6 +182,16 @@ def nextflow_config(self) -> Dict[str, List[str]]: passed.append("Found nf-schema plugin") if self.nf_config.get("validation.help.enabled", "false") == "false": failed.append("The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages") + config_fail.extend([ + ["validation.help.enabled"] + ]) + config_warn.extend([ + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.help.command"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"] + ]) config_fail_ifdefined.extend([ "params.validationFailUnrecognisedParams", "params.validationLenientMode", From 364b54bde2994927d3124bfdf53d97bd1ecc6350 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 15:57:29 +0200 Subject: [PATCH 25/63] small fixes for template without nf-schema --- .../utils_nfcore_pipeline_pipeline/main.nf | 33 ++++++++++--------- .../pipeline-template/workflows/pipeline.nf | 6 ++-- nf_core/pipelines/lint/nextflow_config.py | 6 ++-- nf_core/pipelines/lint/plugin_includes.py | 2 +- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index d96f203544..0867a0a4c9 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -77,9 +77,14 @@ workflow PIPELINE_INITIALISATION { // // Create channel from input file provided through params.input // - {% if nf_schema %} - Channel - .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")) + + Channel{% if nf_schema %} + .fromList(samplesheetToList(params.input, "${projectDir}/assets/schema_input.json")){% else %} + .fromPath(params.input) + .splitCsv(header: true, strip: true) + .map { row -> + [[id:row.sample], row.fastq_1, row.fastq_2] + }{% endif %} .map { meta, fastq_1, fastq_2 -> if (!fastq_2) { @@ -97,12 +102,6 @@ workflow PIPELINE_INITIALISATION { return [ meta, fastqs.flatten() ] } .set { ch_samplesheet } - {% else %} - Channel - .fromPath(params.input) - .splitCsv(header: true, strip: true) - .set { ch_samplesheet } - {% endif %} emit: samplesheet = ch_samplesheet @@ -130,19 +129,21 @@ workflow PIPELINE_COMPLETION { main: - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - // // Completion email and summary // workflow.onComplete { {%- if email %} if (email || email_on_fail) { - {%- if multiqc %} - completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, multiqc_report.toList()) - {%- else %} - completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, []) - {%- endif %} + completionEmail( + {% if nf_schema %}paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json"){% else %}{}{% endif %}, + email, + email_on_fail, + plaintext_email, + outdir, + monochrome_logs, + {% if multiqc %}multiqc_report.toList(){% else %}[]{% endif %} + ) } {%- endif %} diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 8d2773820d..ca5de36a61 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -60,9 +60,13 @@ workflow {{ short_name|upper }} { Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + {% if nf_schema %} summary_params = paramsSummaryMap( workflow, parameters_schema: "nextflow_schema.json") ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + ch_multiqc_files = ch_multiqc_files.mix( + ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + {% endif %} {%- if citations %} ch_multiqc_custom_methods_description = params.multiqc_methods_description ? @@ -72,8 +76,6 @@ workflow {{ short_name|upper }} { methodsDescriptionText(ch_multiqc_custom_methods_description)) {%- endif %} - ch_multiqc_files = ch_multiqc_files.mix( - ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) {%- if citations %} ch_multiqc_files = ch_multiqc_files.mix( diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 1dfdabaf10..f5689813a4 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -164,10 +164,10 @@ def nextflow_config(self) -> Dict[str, List[str]]: ] # Lint for plugins - config_plugins = ast.literal_eval(self.nf_config.get("plugins", "")) + config_plugins = ast.literal_eval(self.nf_config.get("plugins", "[]")) found_plugins = [] if len(config_plugins) == 0: - failed.append("nextflow.config contains an empty plugins scope") + warned.append("nextflow.config contains an empty plugins scope") for plugin in config_plugins: if "@" not in plugin: failed.append(f"Plugin '{plugin}' does not have a pinned version") @@ -176,7 +176,7 @@ def nextflow_config(self) -> Dict[str, List[str]]: if "nf-validation" in found_plugins and "nf-schema" in found_plugins: failed.append("nextflow.config contains both nf-validation and nf-schema") if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: - failed.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + warned.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") if "nf-schema" in found_plugins: passed.append("Found nf-schema plugin") diff --git a/nf_core/pipelines/lint/plugin_includes.py b/nf_core/pipelines/lint/plugin_includes.py index 247c4e493b..740c001a11 100644 --- a/nf_core/pipelines/lint/plugin_includes.py +++ b/nf_core/pipelines/lint/plugin_includes.py @@ -13,7 +13,7 @@ def plugin_includes(self) -> Dict[str, List[str]]: When nf-schema is used in an nf-core pipeline, the include statements of the plugin functions have to use nf-schema instead of nf-validation and vice versa """ - config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", ""))] + config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", "[]"))] validation_plugin = "nf-validation" if "nf-validation" in config_plugins else "nf-schema" passed = [] From 4e631384906d2c02ab537276f22ea47c3ceb438d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 16:02:58 +0200 Subject: [PATCH 26/63] bump nextflow version --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .../nf-core/utils_nextflow_pipeline/tests/nextflow.config | 2 +- .../nf-core/utils_nfcore_pipeline/tests/nextflow.config | 2 +- nf_core/pipelines/lint/readme.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 7e90febb74..f07b31e9de 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -38,7 +38,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: go to subdirectory and change nextflow workdir diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index a95a477459..56c6c822a9 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -39,7 +39,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" steps: - name: go to working directory diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config index d0a926bf6d..0fa4abaf43 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.10.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config index d0a926bf6d..0fa4abaf43 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.10.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/nf_core/pipelines/lint/readme.py b/nf_core/pipelines/lint/readme.py index 4c16243690..1c09104258 100644 --- a/nf_core/pipelines/lint/readme.py +++ b/nf_core/pipelines/lint/readme.py @@ -36,7 +36,7 @@ def readme(self): if "nextflow_badge" not in ignore_configs: # Check that there is a readme badge showing the minimum required version of Nextflow - # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) + # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) # and that it has the correct version nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" match = re.search(nf_badge_re, content) From 11efc5547be76f1a6412d049dbfcd10cd337c660 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 16:08:41 +0200 Subject: [PATCH 27/63] update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98fb925338..b4fcfe8646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ - add templatefeatures.yml to python package ([#3112](https://github.com/nf-core/tools/pull/3112)) - add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125)) - add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126)) +- Use nf-schema instead of nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116)) +- add option to exclude nf-schema from the template ([#3116](https://github.com/nf-core/tools/pull/3116)) ### Linting @@ -26,6 +28,10 @@ - Conda module linting: Include package name in log file ([#3014](https://github.com/nf-core/tools/pull/3014)) - Restructure pipeline tests and move pipeline linting into subfolder ([#3070](https://github.com/nf-core/tools/pull/3070)) - Fix module linting warning for process_high_memory ([#3086](https://github.com/nf-core/tools/issues/3086)) +- Linting will now fail when an unpinned plugin is used ([#3116](https://github.com/nf-core/tools/pull/3116)) +- Linting will now check if the schema is correct for the used validation plugin ([#3116])(https://github.com/nf-core/tools/pull/3116) +- Linting will now check the use of the right validation plugin include statements in the workflow scripts ([#3116])(https://github.com/nf-core/tools/pull/3116) +- Full linting for correct use of nf-schema and nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116)) ### Download From 47c3fe324280a54d4cae4227ab7679ac488b1edb Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 16:13:04 +0200 Subject: [PATCH 28/63] fix ruff linting --- nf_core/pipelines/lint/nextflow_config.py | 53 +++++++++++------------ nf_core/pipelines/lint/plugin_includes.py | 24 ++++++---- nf_core/pipelines/schema.py | 17 +++++--- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index f5689813a4..255042ce5a 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -143,16 +143,10 @@ def nextflow_config(self) -> Dict[str, List[str]]: ["process.memory"], ["process.time"], ["params.outdir"], - ["params.input"] + ["params.input"], ] # Throw a warning if these are missing - config_warn = [ - ["manifest.mainScript"], - ["timeline.file"], - ["trace.file"], - ["report.file"], - ["dag.file"] - ] + config_warn = [["manifest.mainScript"], ["timeline.file"], ["trace.file"], ["report.file"], ["dag.file"]] # Old depreciated vars - fail if present config_fail_ifdefined = [ "params.nf_required_version", @@ -181,28 +175,33 @@ def nextflow_config(self) -> Dict[str, List[str]]: if "nf-schema" in found_plugins: passed.append("Found nf-schema plugin") if self.nf_config.get("validation.help.enabled", "false") == "false": - failed.append("The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages") - config_fail.extend([ - ["validation.help.enabled"] - ]) - config_warn.extend([ - ["validation.help.beforeText"], - ["validation.help.afterText"], - ["validation.help.command"], - ["validation.summary.beforeText"], - ["validation.summary.afterText"] - ]) - config_fail_ifdefined.extend([ - "params.validationFailUnrecognisedParams", - "params.validationLenientMode", - "params.validationSchemaIgnoreParams", - "params.validationShowHiddenParams" - ]) + failed.append( + "The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages" + ) + config_fail.extend([["validation.help.enabled"]]) + config_warn.extend( + [ + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.help.command"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"], + ] + ) + config_fail_ifdefined.extend( + [ + "params.validationFailUnrecognisedParams", + "params.validationLenientMode", + "params.validationSchemaIgnoreParams", + "params.validationShowHiddenParams", + ] + ) if "nf-validation" in found_plugins: passed.append("Found nf-validation plugin") - warned.append("nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/") - + warned.append( + "nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/" + ) # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get("nextflow_config", []) if self.lint_config is not None else [] diff --git a/nf_core/pipelines/lint/plugin_includes.py b/nf_core/pipelines/lint/plugin_includes.py index 740c001a11..6a57b1d280 100644 --- a/nf_core/pipelines/lint/plugin_includes.py +++ b/nf_core/pipelines/lint/plugin_includes.py @@ -16,23 +16,29 @@ def plugin_includes(self) -> Dict[str, List[str]]: config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", "[]"))] validation_plugin = "nf-validation" if "nf-validation" in config_plugins else "nf-schema" - passed = [] - warned = [] - failed = [] - ignored = [] + passed: list[str] = [] + warned: list[str] = [] + failed: list[str] = [] + ignored: list[str] = [] plugin_include_pattern = re.compile(r"^include\s*{[^}]+}\s*from\s*[\"']plugin/([^\"']+)[\"']\s*$", re.MULTILINE) - workflow_files = [file for file in glob.glob(f"{self.wf_path}/**/*.nf", recursive=True) if not file.startswith("./modules/")] + workflow_files = [ + file for file in glob.glob(f"{self.wf_path}/**/*.nf", recursive=True) if not file.startswith("./modules/") + ] test_passed = True for file in workflow_files: - with open(file, "r") as of: + with open(file) as of: plugin_includes = re.findall(plugin_include_pattern, of.read()) for include in plugin_includes: - if include not in ["nf-validation", "nf-schema"]: continue + if include not in ["nf-validation", "nf-schema"]: + continue if include != validation_plugin: test_passed = False - failed.append(f"Found a `{include}` plugin import in `{file[2:]}`, but `{validation_plugin}` was used in `nextflow.config`") + failed.append( + f"Found a `{include}` plugin import in `{file[2:]}`, but `{validation_plugin}` was used in `nextflow.config`" + ) - if test_passed: passed.append("No wrong validation plugin imports have been found") + if test_passed: + passed.append("No wrong validation plugin imports have been found") return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index e9c5556dbb..ccbc86645e 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -5,7 +5,6 @@ import logging import tempfile import webbrowser -import re from pathlib import Path from typing import Union @@ -58,10 +57,10 @@ def set_schema_filename(self, schema: str) -> None: basepath = "/".join(str(schema).split("/")[:-1]) config = f"{basepath}/nextflow.config" if basepath != "" else "nextflow.config" self._update_validation_plugin_from_config(config) - + def get_schema_filename(self) -> str: return self._schema_filename - + def del_schema_filename(self) -> None: del self._schema_filename @@ -71,7 +70,7 @@ def _update_validation_plugin_from_config(self, config: str) -> None: plugin = "nf-schema" conf = nf_core.utils.fetch_wf_config(Path(self.schema_filename).parent) - plugins = str(conf.get("plugins", "")).strip("\"").strip("'").strip(" ").split(",") + plugins = str(conf.get("plugins", "")).strip('"').strip("'").strip(" ").split(",") plugin_found = False for plugin_instance in plugins: if "nf-schema" in plugin_instance: @@ -95,7 +94,7 @@ def _update_validation_plugin_from_config(self, config: str) -> None: # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": self.defs_notation = "$defs" - ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default + ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default ignored_params_config = conf.get("validation", {}).get("defaultIgnoreParams", []) if len(ignored_params_config) > 0: ignored_params.extend(ignored_params_config) @@ -447,7 +446,9 @@ def validate_schema(self, schema=None): # Add a small check for older nf-schema JSON schemas if "defs" in schema: - raise AssertionError(f'Using "defs" for schema definitions is not supported. Please use {self.defs_notation} instead') + raise AssertionError( + f'Using "defs" for schema definitions is not supported. Please use {self.defs_notation} instead' + ) for d_key, d_schema in schema.get(self.defs_notation, {}).items(): # Check that this definition is mentioned in allOf @@ -458,7 +459,9 @@ def validate_schema(self, schema=None): if allOf["$ref"] == f"#/{self.defs_notation}/{d_key}": in_allOf = True if not in_allOf: - raise AssertionError(f"Definition subschema `#/{self.defs_notation}/{d_key}` not included in schema `allOf`") + raise AssertionError( + f"Definition subschema `#/{self.defs_notation}/{d_key}` not included in schema `allOf`" + ) # TODO add support for nested parameters for d_param_id in d_schema.get("properties", {}): From 09bfca2663275c37173ce8018c858164974479f8 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 22 Aug 2024 16:29:39 +0200 Subject: [PATCH 29/63] add a plugin includes test --- tests/pipelines/lint/test_plugin_includes.py | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/pipelines/lint/test_plugin_includes.py diff --git a/tests/pipelines/lint/test_plugin_includes.py b/tests/pipelines/lint/test_plugin_includes.py new file mode 100644 index 0000000000..21e6107c14 --- /dev/null +++ b/tests/pipelines/lint/test_plugin_includes.py @@ -0,0 +1,23 @@ +import nf_core.pipelines.lint + +from ..test_lint import TestLint + +class TestLintNextflowConfig(TestLint): + def setUp(self) -> None: + super().setUp() + self.new_pipeline = self._make_pipeline_copy() + + def test_default_values_match(self): + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + result = lint_obj.plugin_includes() + assert len(result["failed"]) == 0 + assert len(result["warned"]) == 0 + + def test_wrong_include(self): + test_path = self.new_pipeline / "test.nf" + with open(test_path, "w") as of: + of.write("include { paramsSummary } from 'plugin/nf-validation'\n") + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + result = lint_obj.plugin_includes() + assert len(result["failed"]) == 1 + assert len(result["warned"]) == 0 \ No newline at end of file From 44dbbf7a2b095144cb954bb1ed09578776979c92 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 23 Aug 2024 09:50:51 +0200 Subject: [PATCH 30/63] update review comments --- nf_core/pipeline-template/nextflow.config | 7 ++++--- nf_core/pipelines/create/templatefeatures.yml | 7 ++++--- nf_core/pipelines/lint/nextflow_config.py | 1 - nf_core/pipelines/schema.py | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ad800cc55b..3b3183ece1 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -273,6 +273,7 @@ validation { help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " + {% if is_nfcore %} beforeText = """ -\033[2m----------------------------------------------------\033[0m- \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m @@ -289,12 +290,12 @@ validation { * Software dependencies https://github.com/${manifest.name}/blob/master/CITATIONS.md -""" - } +"""{% endif %} + }{% if is_nfcore %} summary { beforeText = validation.help.beforeText afterText = validation.help.afterText - } + }[% endif %] } {% endif -%} diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 29b53eb0a7..224bc38266 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -245,13 +245,14 @@ changelog: nf_schema: skippable_paths: - "subworkflows/nf-core/utils_nfschema_plugin" - short_description: "Use nf-schema for this pipeline." + short_description: "Use nf-schema" + description: "Use nf-schema for this pipeline." help_text: | nf-schema is used to validate input parameters based on a JSON schema. It also provides helper functionality to create help messages, get a summary of changed parameters and validate and convert a samplesheet to a channel. - nfcore_pipelines: False - custom_pipelines: False + nfcore_pipelines: True + custom_pipelines: True license: skippable_paths: - "LICENSE" diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 255042ce5a..71eb6f118f 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -367,7 +367,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: ) # Check for the availability of the "test" configuration profile by parsing nextflow.config - # Also check for the presence of nf-validation/nf-schema and check if they have pinned versions with open(Path(self.wf_path, "nextflow.config")) as f: content = f.read() diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index ccbc86645e..04f5ca2ace 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -447,7 +447,7 @@ def validate_schema(self, schema=None): # Add a small check for older nf-schema JSON schemas if "defs" in schema: raise AssertionError( - f'Using "defs" for schema definitions is not supported. Please use {self.defs_notation} instead' + f'Using "defs" for schema definitions is not supported. Please use "{self.defs_notation}" instead' ) for d_key, d_schema in schema.get(self.defs_notation, {}).items(): From ea51814bc2b1549c14605ba66591e7043bb5e4b3 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Fri, 23 Aug 2024 10:17:15 +0200 Subject: [PATCH 31/63] remove duplicate code --- nf_core/pipelines/schema.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 04f5ca2ace..dbe851415f 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -85,11 +85,6 @@ def _update_validation_plugin_from_config(self, config: str) -> None: if not plugin_found: log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") - if "nf-validation" in plugins: - plugin = "nf-validation" - elif "nf-schema" in plugins: - plugin = "nf-schema" - self.validation_plugin = plugin # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": From b85ea4da6e8287aef01993009cfeac2e2c71b0cd Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 26 Aug 2024 17:21:14 +0200 Subject: [PATCH 32/63] update nf-schema subwf --- nf_core/pipeline-template/modules.json | 2 +- .../utils_nfcore_pipeline_pipeline/main.nf | 11 ++++++++-- .../nf-core/utils_nfschema_plugin/main.nf | 20 ++++++++++++++++--- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 0a883b60b7..666546b6fb 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -32,7 +32,7 @@ }{% if nf_schema %}, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "a3e87febb28bd0461c22a917c5b2c1492053ef85", + "git_sha": "bbd5a41f4535a8defafe6080e00ea74c45f4f96c", "installed_by": ["subworkflows"] }{% endif %} } diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 0867a0a4c9..16f5784683 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -56,7 +56,9 @@ workflow PIPELINE_INITIALISATION { // Validate parameters and generate parameter summary to stdout // UTILS_NFSCHEMA_PLUGIN ( - validate_params + workflow, + validate_params, + null ) {% endif %} @@ -128,6 +130,11 @@ workflow PIPELINE_COMPLETION { {% if multiqc %}multiqc_report // string: Path to MultiQC report{% endif %} main: + {% if nf_schema %} + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + {% else %} + summary_params = [:] + {% endif %} // // Completion email and summary @@ -136,7 +143,7 @@ workflow PIPELINE_COMPLETION { {%- if email %} if (email || email_on_fail) { completionEmail( - {% if nf_schema %}paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json"){% else %}{}{% endif %}, + summary_params, email, email_on_fail, plaintext_email, diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf index 5d1dc7c928..4994303ea0 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -8,7 +8,13 @@ include { validateParameters } from 'plugin/nf-schema' workflow UTILS_NFSCHEMA_PLUGIN { take: - validate_params // boolean: validate the parameters + input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow + validate_params // boolean: validate the parameters + parameters_schema // string: path to the parameters JSON schema. + // this has to be the same as the schema given to `validation.parametersSchema` + // when this input is empty it will automatically use the configured schema or + // "${projectDir}/nextflow_schema.json" as default. This input should not be empty + // for meta pipelines main: @@ -16,14 +22,22 @@ workflow UTILS_NFSCHEMA_PLUGIN { // Print parameter summary to stdout. This will display the parameters // that differ from the default given in the JSON schema // - log.info paramsSummaryLog(workflow) + if(parameters_schema) { + log.info paramsSummaryLog(input_workflow, parameters_schema:parameters_schema) + } else { + log.info paramsSummaryLog(input_workflow) + } // // Validate the parameters using nextflow_schema.json or the schema // given via the validation.parametersSchema configuration option // if(validate_params) { - validateParameters() + if(parameters_schema) { + validateParameters(parameters_schema:parameters_schema) + } else { + validateParameters() + } } emit: From 0fb97db609f912b91b89129d423634b162dc4538 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 26 Aug 2024 17:27:44 +0200 Subject: [PATCH 33/63] fix some issues --- nf_core/pipeline-template/nextflow.config | 2 +- tests/pipelines/lint/test_plugin_includes.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3c238ee0ce..b4c7f41134 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -297,7 +297,7 @@ validation { summary { beforeText = validation.help.beforeText afterText = validation.help.afterText - }[% endif %] + }{% endif %} } {% endif -%} diff --git a/tests/pipelines/lint/test_plugin_includes.py b/tests/pipelines/lint/test_plugin_includes.py index 21e6107c14..ca14d5456d 100644 --- a/tests/pipelines/lint/test_plugin_includes.py +++ b/tests/pipelines/lint/test_plugin_includes.py @@ -2,11 +2,12 @@ from ..test_lint import TestLint + class TestLintNextflowConfig(TestLint): def setUp(self) -> None: super().setUp() self.new_pipeline = self._make_pipeline_copy() - + def test_default_values_match(self): lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) result = lint_obj.plugin_includes() @@ -20,4 +21,4 @@ def test_wrong_include(self): lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) result = lint_obj.plugin_includes() assert len(result["failed"]) == 1 - assert len(result["warned"]) == 0 \ No newline at end of file + assert len(result["warned"]) == 0 From 7dee4476669957b3b53cd3e77dd7cdbc7b544a1a Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 26 Aug 2024 17:33:20 +0200 Subject: [PATCH 34/63] fix linting utils_nfschema_plugin --- .../nf-core/utils_nfschema_plugin/meta.yml | 13 ++++ .../utils_nfschema_plugin/tests/main.nf.test | 63 +++++++++++++++++-- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml index 90b8cb1898..f7d9f02885 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -9,9 +9,22 @@ keywords: - summary components: [] input: + - input_workflow: + type: object + description: | + The workflow object of the used pipeline. + This object contains meta data used to create the params summary log - validate_params: type: boolean description: Validate the parameters and error if invalid. + - parameters_schema: + type: string + description: | + Path to the parameters JSON schema. + This has to be the same as the schema given to the `validation.parametersSchema` config + option. When this input is empty it will automatically use the configured schema or + "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way + for meta pipelines. output: - dummy_emit: type: boolean diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test index 703d3a9b86..842dc432af 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -1,5 +1,3 @@ -// TODO nf-core: Once you have added the required tests, please run the following command to build this file: -// nf-core subworkflows test utils_nfschema_plugin nextflow_workflow { name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" @@ -9,6 +7,7 @@ nextflow_workflow { tag "subworkflows" tag "subworkflows_nfcore" tag "subworkflows/utils_nfschema_plugin" + tag "plugin/nf-schema" config "./nextflow.config" @@ -23,7 +22,9 @@ nextflow_workflow { workflow { """ validate_params = false - input[0] = validate_params + input[0] = workflow + input[1] = validate_params + input[2] = "" """ } } @@ -47,7 +48,61 @@ nextflow_workflow { workflow { """ validate_params = true - input[0] = validate_params + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } + + test("Should run nothing - custom schema") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params - custom schema") { + + when { + + params { + test_data = '' + outdir = 1 + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" """ } } From 6d01b0faf3c1f62dcc22724197bcf599089a8b3b Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 26 Aug 2024 17:36:32 +0200 Subject: [PATCH 35/63] update utils subwfs --- nf_core/pipeline-template/modules.json | 4 +- .../nf-core/utils_nextflow_pipeline/main.nf | 24 +++++----- .../tests/nextflow.config | 2 +- .../nf-core/utils_nfcore_pipeline/main.nf | 45 +++++++++---------- .../tests/nextflow.config | 2 +- 5 files changed, 37 insertions(+), 40 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 19df47df2e..ec1cf1d963 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -23,12 +23,12 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "20c03aede5a80ff520a905cea1f8ca121b5bb661", + "git_sha": "2aacc9ef22bfc627b18eca17e9a3bb13c58aab64", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", + "git_sha": "2fdce49d30c0254f76bc0f13c55c17455c1251ab", "installed_by": ["subworkflows"] }{% if nf_schema %}, "utils_nfschema_plugin": { diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index e770d91b97..b54fc16c71 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -2,10 +2,6 @@ // Subworkflow with functionality that may be useful for any Nextflow pipeline // -import org.yaml.snakeyaml.Yaml -import groovy.json.JsonOutput -import nextflow.extension.FilesEx - /* ======================================================================================== SUBWORKFLOW DEFINITION @@ -58,7 +54,7 @@ workflow UTILS_NEXTFLOW_PIPELINE { // Generate version string // def getWorkflowVersion() { - String version_string = "" + def version_string = "" as String if (workflow.manifest.version) { def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' version_string += "${prefix_v}${workflow.manifest.version}" @@ -79,10 +75,10 @@ def dumpParametersToJSON(outdir) { def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') def filename = "params_${timestamp}.json" def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = JsonOutput.toJson(params) - temp_pf.text = JsonOutput.prettyPrint(jsonStr) + def jsonStr = groovy.json.JsonOutput.toJson(params) + temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) - FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") temp_pf.delete() } @@ -90,7 +86,7 @@ def dumpParametersToJSON(outdir) { // When running with -profile conda, warn if channels have not been set-up appropriately // def checkCondaChannels() { - Yaml parser = new Yaml() + def parser = new org.yaml.snakeyaml.Yaml() def channels = [] try { def config = parser.load("conda config --show channels".execute().text) @@ -102,14 +98,16 @@ def checkCondaChannels() { // Check that all channels are present // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda'] + def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order def channel_priority_violation = false - def n = required_channels_in_order.size() - for (int i = 0; i < n - 1; i++) { - channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + + required_channels_in_order.eachWithIndex { channel, index -> + if (index < required_channels_in_order.size() - 1) { + channel_priority_violation |= !(channels.indexOf(channel) < channels.indexOf(required_channels_in_order[index+1])) + } } if (channels_missing | channel_priority_violation) { diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config index 0fa4abaf43..a09572e5bb 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.10.0' + nextflowVersion = '!>=23.04.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 14558c3927..cbd8495bb6 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -2,9 +2,6 @@ // Subworkflow with utility functions specific to the nf-core pipeline template // -import org.yaml.snakeyaml.Yaml -import nextflow.extension.FilesEx - /* ======================================================================================== SUBWORKFLOW DEFINITION @@ -34,7 +31,7 @@ workflow UTILS_NFCORE_PIPELINE { // Warn if a -profile or Nextflow config has not been provided to run the pipeline // def checkConfigProvided() { - valid_config = true + def valid_config = true as Boolean if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + @@ -66,11 +63,13 @@ def checkProfileProvided(nextflow_cli_args) { // def workflowCitation() { def temp_doi_ref = "" - String[] manifest_doi = workflow.manifest.doi.tokenize(",") + def manifest_doi = workflow.manifest.doi.tokenize(",") // Using a loop to handle multiple DOIs // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers // Removing ` ` since the manifest.doi is a string and not a proper list - for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" + manifest_doi.each { doi_ref -> + temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" + } return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + temp_doi_ref + "\n" + @@ -84,7 +83,7 @@ def workflowCitation() { // Generate workflow version string // def getWorkflowVersion() { - String version_string = "" + def version_string = "" as String if (workflow.manifest.version) { def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' version_string += "${prefix_v}${workflow.manifest.version}" @@ -102,8 +101,8 @@ def getWorkflowVersion() { // Get software versions for pipeline // def processVersionsFromYAML(yaml_file) { - Yaml yaml = new Yaml() - versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } + def yaml = new org.yaml.snakeyaml.Yaml() + def versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } return yaml.dumpAsMap(versions).trim() } @@ -124,7 +123,7 @@ def workflowVersionToYAML() { def softwareVersionsToYAML(ch_versions) { return ch_versions .unique() - .map { processVersionsFromYAML(it) } + .map { version -> processVersionsFromYAML(version) } .unique() .mix(Channel.of(workflowVersionToYAML())) } @@ -134,19 +133,19 @@ def softwareVersionsToYAML(ch_versions) { // def paramsSummaryMultiqc(summary_params) { def summary_section = '' - for (group in summary_params.keySet()) { + summary_params.keySet().each { group -> def group_params = summary_params.get(group) // This gets the parameters of that particular group if (group_params) { summary_section += "

$group

\n" summary_section += "
\n" - for (param in group_params.keySet()) { + group_params.keySet().sort().each { param -> summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" } summary_section += "
\n" } } - String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + def yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" as String yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" @@ -161,7 +160,7 @@ def paramsSummaryMultiqc(summary_params) { // nf-core logo // def nfCoreLogo(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map String.format( """\n ${dashedLine(monochrome_logs)} @@ -180,7 +179,7 @@ def nfCoreLogo(monochrome_logs=true) { // Return dashed line // def dashedLine(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map return "-${colors.dim}----------------------------------------------------${colors.reset}-" } @@ -188,7 +187,7 @@ def dashedLine(monochrome_logs=true) { // ANSII colours used for terminal logging // def logColours(monochrome_logs=true) { - Map colorcodes = [:] + def colorcodes = [:] as Map // Reset / Meta colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" @@ -287,7 +286,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi } def summary = [:] - for (group in summary_params.keySet()) { + summary_params.keySet().sort().each { group -> summary << summary_params[group] } @@ -344,10 +343,10 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi def sendmail_html = sendmail_template.toString() // Send the HTML e-mail - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map if (email_address) { try { - if (plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + if (plaintext_email) { throw new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") sendmail_tf.withWriter { w -> w << sendmail_html } @@ -364,13 +363,13 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi // Write summary e-mail HTML to a file def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); + nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); output_hf.delete() // Write summary e-mail TXT to a file def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } - FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); + nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); output_tf.delete() } @@ -378,7 +377,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi // Print pipeline summary on completion // def completionSummary(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map if (workflow.success) { if (workflow.stats.ignoredCount == 0) { log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" @@ -395,7 +394,7 @@ def completionSummary(monochrome_logs=true) { // def imNotification(summary_params, hook_url) { def summary = [:] - for (group in summary_params.keySet()) { + summary_params.keySet().sort().each { group -> summary << summary_params[group] } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config index 0fa4abaf43..d0a926bf6d 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.10.0' + nextflowVersion = '!>=23.04.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } From a7dcdd7b6a02a8cd4a4876a0b888d43de3252f66 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 09:19:09 +0200 Subject: [PATCH 36/63] fix some linting issues --- nf_core/pipelines/lint/files_exist.py | 4 +- nf_core/pipelines/lint/nextflow_config.py | 67 +++++++++++------------ nf_core/pipelines/schema.py | 2 +- 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/nf_core/pipelines/lint/files_exist.py b/nf_core/pipelines/lint/files_exist.py index ad0605dcf7..e944882187 100644 --- a/nf_core/pipelines/lint/files_exist.py +++ b/nf_core/pipelines/lint/files_exist.py @@ -200,9 +200,7 @@ def files_exist(self) -> Dict[str, List[str]]: Path("Singularity"), ] files_warn_ifexists = [Path(".travis.yml")] - files_fail_ifinconfig: List[Tuple[Path, List[Dict[str, str]]]] = [ - (Path("lib", "nfcore_external_java_deps.jar"), [{"plugins": "nf-validation"}, {"plugins": "nf-schema"}]), - ] + files_fail_ifinconfig: List[Tuple[Path, List[Dict[str, str]]]] = [] # Remove files that should be ignored according to the linting config ignore_files = self.lint_config.get("files_exist", []) if self.lint_config is not None else [] diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 71eb6f118f..2159d7939a 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -160,48 +160,47 @@ def nextflow_config(self) -> Dict[str, List[str]]: # Lint for plugins config_plugins = ast.literal_eval(self.nf_config.get("plugins", "[]")) found_plugins = [] - if len(config_plugins) == 0: - warned.append("nextflow.config contains an empty plugins scope") for plugin in config_plugins: if "@" not in plugin: failed.append(f"Plugin '{plugin}' does not have a pinned version") found_plugins.append(plugin.split("@")[0]) - if "nf-validation" in found_plugins and "nf-schema" in found_plugins: - failed.append("nextflow.config contains both nf-validation and nf-schema") - if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: - warned.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") + if "nf-validation" in found_plugins or "nf-schema" in found_plugins: + if "nf-validation" in found_plugins and "nf-schema" in found_plugins: + failed.append("nextflow.config contains both nf-validation and nf-schema") + if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: + warned.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") - if "nf-schema" in found_plugins: - passed.append("Found nf-schema plugin") - if self.nf_config.get("validation.help.enabled", "false") == "false": - failed.append( - "The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages" + if "nf-schema" in found_plugins: + passed.append("Found nf-schema plugin") + if self.nf_config.get("validation.help.enabled", "false") == "false": + failed.append( + "The help message has not been enabled. Set the `validation.help.enabled` configuration option to `true` to enable help messages" + ) + config_fail.extend([["validation.help.enabled"]]) + config_warn.extend( + [ + ["validation.help.beforeText"], + ["validation.help.afterText"], + ["validation.help.command"], + ["validation.summary.beforeText"], + ["validation.summary.afterText"], + ] + ) + config_fail_ifdefined.extend( + [ + "params.validationFailUnrecognisedParams", + "params.validationLenientMode", + "params.validationSchemaIgnoreParams", + "params.validationShowHiddenParams", + ] ) - config_fail.extend([["validation.help.enabled"]]) - config_warn.extend( - [ - ["validation.help.beforeText"], - ["validation.help.afterText"], - ["validation.help.command"], - ["validation.summary.beforeText"], - ["validation.summary.afterText"], - ] - ) - config_fail_ifdefined.extend( - [ - "params.validationFailUnrecognisedParams", - "params.validationLenientMode", - "params.validationSchemaIgnoreParams", - "params.validationShowHiddenParams", - ] - ) - if "nf-validation" in found_plugins: - passed.append("Found nf-validation plugin") - warned.append( - "nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/" - ) + if "nf-validation" in found_plugins: + passed.append("Found nf-validation plugin") + warned.append( + "nf-validation has been detected in the pipeline. Please migrate to nf-schema: https://nextflow-io.github.io/nf-schema/latest/migration_guide/" + ) # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get("nextflow_config", []) if self.lint_config is not None else [] diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index dbe851415f..13b8f62641 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -83,7 +83,7 @@ def _update_validation_plugin_from_config(self, config: str) -> None: break if not plugin_found: - log.warning("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema") + log.info("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema notation for the JSON schema.") self.validation_plugin = plugin # Previous versions of nf-schema used "defs", but it's advised to use "$defs" From fcdf65be8eef12b7f2c94f7e575c1fbaef8f51da Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 09:21:38 +0200 Subject: [PATCH 37/63] ruff --- nf_core/pipelines/schema.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 13b8f62641..76f085a49d 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -83,7 +83,9 @@ def _update_validation_plugin_from_config(self, config: str) -> None: break if not plugin_found: - log.info("Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema notation for the JSON schema.") + log.info( + "Could not find nf-schema or nf-validation in the pipeline config. Defaulting to nf-schema notation for the JSON schema." + ) self.validation_plugin = plugin # Previous versions of nf-schema used "defs", but it's advised to use "$defs" From 56150dc2e95f0cd99752f263d56a649e5a9822b0 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 09:23:59 +0200 Subject: [PATCH 38/63] add linting doc for plugin_includes --- docs/api/_src/pipeline_lint_tests/plugin_includes.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/api/_src/pipeline_lint_tests/plugin_includes.md diff --git a/docs/api/_src/pipeline_lint_tests/plugin_includes.md b/docs/api/_src/pipeline_lint_tests/plugin_includes.md new file mode 100644 index 0000000000..48bddadc80 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/plugin_includes.md @@ -0,0 +1,5 @@ +# plugin_includes + +```{eval-rst} +.. automethod:: nf_core.pipelines.lint.PipelineLint.plugin_includes +``` From 5a29c4ef39faefc7f50e30f55c093411adb3f5e0 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 09:38:47 +0200 Subject: [PATCH 39/63] readd file exists check --- nf_core/pipelines/lint/files_exist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipelines/lint/files_exist.py b/nf_core/pipelines/lint/files_exist.py index e944882187..7bfbae8ec9 100644 --- a/nf_core/pipelines/lint/files_exist.py +++ b/nf_core/pipelines/lint/files_exist.py @@ -198,6 +198,7 @@ def files_exist(self) -> Dict[str, List[str]]: Path("parameters.settings.json"), Path("pipeline_template.yml"), # saving information in .nf-core.yml Path("Singularity"), + Path("lib", "nfcore_external_java_deps.jar"), ] files_warn_ifexists = [Path(".travis.yml")] files_fail_ifinconfig: List[Tuple[Path, List[Dict[str, str]]]] = [] From 18738ef7292891c4f7cd29676af2d4457769822c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 09:50:58 +0200 Subject: [PATCH 40/63] fix launch command --- nf_core/pipelines/launch.py | 23 ++++++++++++++++------- tests/pipelines/test_launch.py | 10 +++++----- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/nf_core/pipelines/launch.py b/nf_core/pipelines/launch.py index e03982a25a..a14a8d6d72 100644 --- a/nf_core/pipelines/launch.py +++ b/nf_core/pipelines/launch.py @@ -263,15 +263,22 @@ def set_schema_inputs(self): def merge_nxf_flag_schema(self): """Take the Nextflow flag schema and merge it with the pipeline schema""" + if "allOf" not in self.schema_obj.schema: + self.schema_obj.schema["allOf"] = [] # Add the coreNextflow subschema to the schema definitions - if "definitions" not in self.schema_obj.schema: + if "$defs" in self.schema_obj.schema or "definitions" not in self.schema_obj.schema: + if "$defs" not in self.schema_obj.schema: + self.schema_obj["$defs"] = {} + self.schema_obj.schema["$defs"].update(self.nxf_flag_schema) + self.schema_obj.schema["allOf"].insert(0, {"$ref": "#/$defs/coreNextflow"}) + + if "definitions" in self.schema_obj.schema: self.schema_obj.schema["definitions"] = {} - self.schema_obj.schema["definitions"].update(self.nxf_flag_schema) + self.schema_obj.schema["definitions"].update(self.nxf_flag_schema) + self.schema_obj.schema["allOf"].insert(0, {"$ref": "#/definitions/coreNextflow"}) # Add the new defintion to the allOf key so that it's included in validation # Put it at the start of the list so that it comes first - if "allOf" not in self.schema_obj.schema: - self.schema_obj.schema["allOf"] = [] - self.schema_obj.schema["allOf"].insert(0, {"$ref": "#/definitions/coreNextflow"}) + def prompt_web_gui(self): """Ask whether to use the web-based or cli wizard to collect params""" @@ -379,7 +386,8 @@ def sanitise_web_response(self): for param_id, param_obj in self.schema_obj.schema.get("properties", {}).items(): questionary_objects[param_id] = self.single_param_to_questionary(param_id, param_obj, print_help=False) - for _, definition in self.schema_obj.schema.get("definitions", {}).items(): + definitions_schemas = self.schema_obj.schema.get("$defs", self.schema_obj.schema.get("definitions", {})).items() + for _, definition in definitions_schemas: for param_id, param_obj in definition.get("properties", {}).items(): questionary_objects[param_id] = self.single_param_to_questionary(param_id, param_obj, print_help=False) @@ -399,9 +407,10 @@ def prompt_schema(self): """Go through the pipeline schema and prompt user to change defaults""" answers = {} # Start with the subschema in the definitions - use order of allOf + definitions_schemas = self.schema_obj.schema.get("$defs", self.schema_obj.schema.get("definitions", {})).items() for allOf in self.schema_obj.schema.get("allOf", []): d_key = allOf["$ref"][14:] - answers.update(self.prompt_group(d_key, self.schema_obj.schema["definitions"][d_key])) + answers.update(self.prompt_group(d_key, definitions_schemas[d_key])) # Top level schema params for param_id, param_obj in self.schema_obj.schema.get("properties", {}).items(): diff --git a/tests/pipelines/test_launch.py b/tests/pipelines/test_launch.py index 7c6e3d6196..d63e7b6dc5 100644 --- a/tests/pipelines/test_launch.py +++ b/tests/pipelines/test_launch.py @@ -47,7 +47,7 @@ def test_launch_file_exists_overwrite(self, mock_webbrowser, mock_lauch_web_gui, def test_get_pipeline_schema(self): """Test loading the params schema from a pipeline""" self.launcher.get_pipeline_schema() - assert len(self.launcher.schema_obj.schema["definitions"]["input_output_options"]["properties"]) > 2 + assert len(self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]) > 2 @with_temporary_folder def test_make_pipeline_schema(self, tmp_path): @@ -60,8 +60,8 @@ def test_make_pipeline_schema(self, tmp_path): Path(test_pipeline_dir, "nextflow_schema.json").unlink() self.launcher = nf_core.pipelines.launch.Launch(test_pipeline_dir, params_out=self.nf_params_fn) self.launcher.get_pipeline_schema() - assert len(self.launcher.schema_obj.schema["definitions"]["input_output_options"]["properties"]) >= 2 - assert self.launcher.schema_obj.schema["definitions"]["input_output_options"]["properties"]["outdir"] == { + assert len(self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]) >= 2 + assert self.launcher.schema_obj.schema["$defs"]["input_output_options"]["properties"]["outdir"] == { "type": "string", "format": "directory-path", "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", @@ -91,8 +91,8 @@ def test_nf_merge_schema(self): self.launcher.get_pipeline_schema() self.launcher.set_schema_inputs() self.launcher.merge_nxf_flag_schema() - assert self.launcher.schema_obj.schema["allOf"][0] == {"$ref": "#/definitions/coreNextflow"} - assert "-resume" in self.launcher.schema_obj.schema["definitions"]["coreNextflow"]["properties"] + assert self.launcher.schema_obj.schema["allOf"][0] == {"$ref": "#/$defs/coreNextflow"} + assert "-resume" in self.launcher.schema_obj.schema["$defs"]["coreNextflow"]["properties"] def test_ob_to_questionary_string(self): """Check converting a python dict to a pyenquirer format - simple strings""" From ebf6a27c7d9869e19d8bf5c67467bf59cb7d543c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 10:26:00 +0200 Subject: [PATCH 41/63] fix schema tests --- nf_core/pipelines/schema.py | 13 ++++++----- tests/pipelines/test_schema.py | 41 ++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 76f085a49d..6c1c198bb8 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -54,9 +54,7 @@ def __init__(self): # Update the validation plugin code everytime the schema gets changed def set_schema_filename(self, schema: str) -> None: self._schema_filename = schema - basepath = "/".join(str(schema).split("/")[:-1]) - config = f"{basepath}/nextflow.config" if basepath != "" else "nextflow.config" - self._update_validation_plugin_from_config(config) + self._update_validation_plugin_from_config() def get_schema_filename(self) -> str: return self._schema_filename @@ -66,9 +64,12 @@ def del_schema_filename(self) -> None: schema_filename = property(get_schema_filename, set_schema_filename, del_schema_filename) - def _update_validation_plugin_from_config(self, config: str) -> None: + def _update_validation_plugin_from_config(self) -> None: plugin = "nf-schema" - conf = nf_core.utils.fetch_wf_config(Path(self.schema_filename).parent) + if self.schema_filename: + conf = nf_core.utils.fetch_wf_config(Path(self.schema_filename).parent) + else: + conf = nf_core.utils.fetch_wf_config(Path(self.pipeline_dir)) plugins = str(conf.get("plugins", "")).strip('"').strip("'").strip(" ").split(",") plugin_found = False @@ -464,7 +465,7 @@ def validate_schema(self, schema=None): for d_param_id in d_schema.get("properties", {}): # Check that we don't have any duplicate parameter IDs in different definitions if d_param_id in param_keys: - raise AssertionError(f"Duplicate parameter found in schema `definitions`: `{d_param_id}`") + raise AssertionError(f"Duplicate parameter found in schema `{self.defs_notation}`: `{d_param_id}`") param_keys.append(d_param_id) num_params += 1 diff --git a/tests/pipelines/test_schema.py b/tests/pipelines/test_schema.py index 777d959e66..2abaf07bd2 100644 --- a/tests/pipelines/test_schema.py +++ b/tests/pipelines/test_schema.py @@ -24,6 +24,9 @@ class TestSchema(unittest.TestCase): def setUp(self): """Create a new PipelineSchema object""" self.schema_obj = nf_core.pipelines.schema.PipelineSchema() + self.schema_obj.schema_draft = "https://json-schema.org/draft/2020-12/schema" + self.schema_obj.defs_notation = "$defs" + self.schema_obj.validation_plugin = "nf-schema" self.root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # Create a test pipeline in temp directory @@ -105,7 +108,7 @@ def test_schema_docs(self): docs = self.schema_obj.print_documentation() assert self.schema_obj.schema["title"] in docs assert self.schema_obj.schema["description"] in docs - for definition in self.schema_obj.schema.get("definitions", {}).values(): + for definition in self.schema_obj.schema.get("$defs", {}).values(): assert definition["title"] in docs assert definition["description"] in docs @@ -175,43 +178,43 @@ def test_validate_schema_fail_duplicate_ids(self): Check that the schema validation fails when we have duplicate IDs in definition subschema """ self.schema_obj.schema = { - "$schema": "http://json-schema.org/draft-07/schema", - "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"foo": {}}}}, - "allOf": [{"$ref": "#/definitions/groupOne"}, {"$ref": "#/definitions/groupTwo"}], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"foo": {}}}}, + "allOf": [{"$ref": "#/$defs/groupOne"}, {"$ref": "#/$defs/groupTwo"}], } with pytest.raises(AssertionError) as exc_info: self.schema_obj.validate_schema(self.schema_obj.schema) - assert exc_info.value.args[0] == "Duplicate parameter found in schema `definitions`: `foo`" + assert exc_info.value.args[0] == "Duplicate parameter found in schema `$defs`: `foo`" def test_validate_schema_fail_missing_def(self): """ - Check that the schema validation fails when we a definition in allOf is not in definitions + Check that the schema validation fails when we a definition in allOf is not in $defs """ self.schema_obj.schema = { - "$schema": "http://json-schema.org/draft-07/schema", - "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, - "allOf": [{"$ref": "#/definitions/groupOne"}], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, + "allOf": [{"$ref": "#/$defs/groupOne"}], } with pytest.raises(AssertionError) as exc_info: self.schema_obj.validate_schema(self.schema_obj.schema) - assert exc_info.value.args[0] == "Definition subschema `#/definitions/groupTwo` not included in schema `allOf`" + assert exc_info.value.args[0] == "Definition subschema `#/$defs/groupTwo` not included in schema `allOf`" def test_validate_schema_fail_unexpected_allof(self): """ Check that the schema validation fails when we an unrecognised definition is in allOf """ self.schema_obj.schema = { - "$schema": "http://json-schema.org/draft-07/schema", - "definitions": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": {"groupOne": {"properties": {"foo": {}}}, "groupTwo": {"properties": {"bar": {}}}}, "allOf": [ - {"$ref": "#/definitions/groupOne"}, - {"$ref": "#/definitions/groupTwo"}, - {"$ref": "#/definitions/groupThree"}, + {"$ref": "#/$defs/groupOne"}, + {"$ref": "#/$defs/groupTwo"}, + {"$ref": "#/$defs/groupThree"}, ], } with pytest.raises(AssertionError) as exc_info: self.schema_obj.validate_schema(self.schema_obj.schema) - assert exc_info.value.args[0] == "Subschema `groupThree` found in `allOf` but not `definitions`" + assert exc_info.value.args[0] == "Subschema `groupThree` found in `allOf` but not `$defs`" def test_make_skeleton_schema(self): """Test making a new schema skeleton""" @@ -267,7 +270,7 @@ def test_remove_schema_notfound_configs_childschema(self): even when they're in a group """ self.schema_obj.schema = { - "definitions": { + "$defs": { "subSchemaId": { "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}}, "required": ["foo"], @@ -277,8 +280,8 @@ def test_remove_schema_notfound_configs_childschema(self): self.schema_obj.pipeline_params = {"bar": True} self.schema_obj.no_prompts = True params_removed = self.schema_obj.remove_schema_notfound_configs() - assert len(self.schema_obj.schema["definitions"]["subSchemaId"]["properties"]) == 1 - assert "required" not in self.schema_obj.schema["definitions"]["subSchemaId"] + assert len(self.schema_obj.schema["$defs"]["subSchemaId"]["properties"]) == 1 + assert "required" not in self.schema_obj.schema["$defs"]["subSchemaId"] assert len(params_removed) == 1 assert "foo" in params_removed From 04fabe5d38eb2b948903645137d98a37cd532567 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 10:26:47 +0200 Subject: [PATCH 42/63] ruff --- nf_core/pipelines/launch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipelines/launch.py b/nf_core/pipelines/launch.py index a14a8d6d72..a80639ea94 100644 --- a/nf_core/pipelines/launch.py +++ b/nf_core/pipelines/launch.py @@ -279,7 +279,6 @@ def merge_nxf_flag_schema(self): # Add the new defintion to the allOf key so that it's included in validation # Put it at the start of the list so that it comes first - def prompt_web_gui(self): """Ask whether to use the web-based or cli wizard to collect params""" log.info( From 0649a756bd4727926951be3a17b6bf643ea93dae Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 10:31:07 +0200 Subject: [PATCH 43/63] readd file exists check --- tests/pipelines/lint/test_files_exist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/lint/test_files_exist.py b/tests/pipelines/lint/test_files_exist.py index 85ba817536..65217903f8 100644 --- a/tests/pipelines/lint/test_files_exist.py +++ b/tests/pipelines/lint/test_files_exist.py @@ -62,7 +62,7 @@ def test_files_exist_pass_conditional(self): lib_dir.mkdir() (lib_dir / "nfcore_external_java_deps.jar").touch() results = lint_obj.files_exist() - assert results["failed"] == [] + assert results["failed"] == ["File must be removed: `lib/nfcore_external_java_deps.jar`"] assert results["ignored"] == [] def test_files_exist_fail_conditional(self): From 9fd98692e9eb22514f9a107509b12c4cfc2df14e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 10:41:23 +0200 Subject: [PATCH 44/63] fix lint test --- tests/subworkflows/test_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index c8b97fd0b0..56574b865c 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -22,7 +22,7 @@ def test_subworkflows_lint_empty(self): """Test linting a pipeline with no subworkflows installed""" self.subworkflow_remove.remove("utils_nextflow_pipeline", force=True) self.subworkflow_remove.remove("utils_nfcore_pipeline", force=True) - self.subworkflow_remove.remove("utils_nfvalidation_plugin", force=True) + self.subworkflow_remove.remove("utils_nfschema_plugin", force=True) nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) assert "No subworkflows from https://github.com/nf-core/modules.git installed in pipeline" in self.caplog.text From 6cccc1e8aa650ccc13f4e9159265d42f84f9f9e5 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Aug 2024 11:16:07 +0200 Subject: [PATCH 45/63] update textual snapshots --- .../__snapshots__/test_create_app.ambr | 762 +++++++++--------- 1 file changed, 381 insertions(+), 381 deletions(-) diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index 08676899a1..3ed8349ee1 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -851,257 +851,257 @@ font-weight: 700; } - .terminal-988282695-matrix { + .terminal-2541420365-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-988282695-title { + .terminal-2541420365-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-988282695-r1 { fill: #c5c8c6 } - .terminal-988282695-r2 { fill: #e3e3e3 } - .terminal-988282695-r3 { fill: #989898 } - .terminal-988282695-r4 { fill: #e1e1e1 } - .terminal-988282695-r5 { fill: #4ebf71;font-weight: bold } - .terminal-988282695-r6 { fill: #1e1e1e } - .terminal-988282695-r7 { fill: #507bb3 } - .terminal-988282695-r8 { fill: #e2e2e2 } - .terminal-988282695-r9 { fill: #808080 } - .terminal-988282695-r10 { fill: #dde6ed;font-weight: bold } - .terminal-988282695-r11 { fill: #001541 } - .terminal-988282695-r12 { fill: #0178d4 } - .terminal-988282695-r13 { fill: #454a50 } - .terminal-988282695-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-988282695-r15 { fill: #000000 } - .terminal-988282695-r16 { fill: #14191f } - .terminal-988282695-r17 { fill: #e4e4e4 } - .terminal-988282695-r18 { fill: #7ae998 } - .terminal-988282695-r19 { fill: #0a180e;font-weight: bold } - .terminal-988282695-r20 { fill: #008139 } - .terminal-988282695-r21 { fill: #fea62b;font-weight: bold } - .terminal-988282695-r22 { fill: #a7a9ab } - .terminal-988282695-r23 { fill: #e2e3e3 } + .terminal-2541420365-r1 { fill: #c5c8c6 } + .terminal-2541420365-r2 { fill: #e3e3e3 } + .terminal-2541420365-r3 { fill: #989898 } + .terminal-2541420365-r4 { fill: #e1e1e1 } + .terminal-2541420365-r5 { fill: #4ebf71;font-weight: bold } + .terminal-2541420365-r6 { fill: #1e1e1e } + .terminal-2541420365-r7 { fill: #507bb3 } + .terminal-2541420365-r8 { fill: #e2e2e2 } + .terminal-2541420365-r9 { fill: #808080 } + .terminal-2541420365-r10 { fill: #dde6ed;font-weight: bold } + .terminal-2541420365-r11 { fill: #001541 } + .terminal-2541420365-r12 { fill: #0178d4 } + .terminal-2541420365-r13 { fill: #454a50 } + .terminal-2541420365-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-2541420365-r15 { fill: #000000 } + .terminal-2541420365-r16 { fill: #14191f } + .terminal-2541420365-r17 { fill: #e4e4e4 } + .terminal-2541420365-r18 { fill: #7ae998 } + .terminal-2541420365-r19 { fill: #0a180e;font-weight: bold } + .terminal-2541420365-r20 { fill: #008139 } + .terminal-2541420365-r21 { fill: #fea62b;font-weight: bold } + .terminal-2541420365-r22 { fill: #a7a9ab } + .terminal-2541420365-r23 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - + - - nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add Github CI testsThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use reference genomesThe pipeline will be  Hide help  - ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference ▂▂ - genome files from  - iGenomes - - - Nf-core pipelines are configured to use a copy of the most common reference  - genome files. - - By selecting this option, your pipeline will include a configuration file  - specifying the paths to these files. - - The required code to use these files will also be included in the template.  - When the pipeline user provides an appropriate genome key, the pipeline will - automatically download the required reference files. - ▅▅ - For more information about reference genomes in nf-core pipelines, see the  - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add Github badgesThe README.md file of  Show help  - ▁▁▁▁▁▁▁▁the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add configuration The pipeline will  Show help  - ▁▁▁▁▁▁▁▁        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add Github CI testsThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use reference genomesThe pipeline will be  Hide help  + ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference ▅▅ + genome files from  + iGenomes + + + Nf-core pipelines are configured to use a copy of the most common reference  + genome files. + + By selecting this option, your pipeline will include a configuration file  + specifying the paths to these files. + + The required code to use these files will also be included in the template.  + When the pipeline user provides an appropriate genome key, the pipeline will + automatically download the required reference files. + ▅▅ + For more information about reference genomes in nf-core pipelines, see the  + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add Github badgesThe README.md file of  Show help  + ▁▁▁▁▁▁▁▁the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add configuration The pipeline will  Show help  + ▁▁▁▁▁▁▁▁        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-2501463490-matrix { + .terminal-3458551717-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2501463490-title { + .terminal-3458551717-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2501463490-r1 { fill: #c5c8c6 } - .terminal-2501463490-r2 { fill: #e3e3e3 } - .terminal-2501463490-r3 { fill: #989898 } - .terminal-2501463490-r4 { fill: #e1e1e1 } - .terminal-2501463490-r5 { fill: #4ebf71;font-weight: bold } - .terminal-2501463490-r6 { fill: #1e1e1e } - .terminal-2501463490-r7 { fill: #507bb3 } - .terminal-2501463490-r8 { fill: #e2e2e2 } - .terminal-2501463490-r9 { fill: #808080 } - .terminal-2501463490-r10 { fill: #dde6ed;font-weight: bold } - .terminal-2501463490-r11 { fill: #001541 } - .terminal-2501463490-r12 { fill: #14191f } - .terminal-2501463490-r13 { fill: #454a50 } - .terminal-2501463490-r14 { fill: #7ae998 } - .terminal-2501463490-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-2501463490-r16 { fill: #0a180e;font-weight: bold } - .terminal-2501463490-r17 { fill: #000000 } - .terminal-2501463490-r18 { fill: #008139 } - .terminal-2501463490-r19 { fill: #fea62b;font-weight: bold } - .terminal-2501463490-r20 { fill: #a7a9ab } - .terminal-2501463490-r21 { fill: #e2e3e3 } + .terminal-3458551717-r1 { fill: #c5c8c6 } + .terminal-3458551717-r2 { fill: #e3e3e3 } + .terminal-3458551717-r3 { fill: #989898 } + .terminal-3458551717-r4 { fill: #e1e1e1 } + .terminal-3458551717-r5 { fill: #4ebf71;font-weight: bold } + .terminal-3458551717-r6 { fill: #1e1e1e } + .terminal-3458551717-r7 { fill: #507bb3 } + .terminal-3458551717-r8 { fill: #e2e2e2 } + .terminal-3458551717-r9 { fill: #808080 } + .terminal-3458551717-r10 { fill: #dde6ed;font-weight: bold } + .terminal-3458551717-r11 { fill: #001541 } + .terminal-3458551717-r12 { fill: #14191f } + .terminal-3458551717-r13 { fill: #454a50 } + .terminal-3458551717-r14 { fill: #7ae998 } + .terminal-3458551717-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-3458551717-r16 { fill: #0a180e;font-weight: bold } + .terminal-3458551717-r17 { fill: #000000 } + .terminal-3458551717-r18 { fill: #008139 } + .terminal-3458551717-r19 { fill: #fea62b;font-weight: bold } + .terminal-3458551717-r20 { fill: #a7a9ab } + .terminal-3458551717-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add Github CI testsThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use reference genomesThe pipeline will be  Show help  - ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from  - iGenomes▇▇ - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add Github badgesThe README.md file of  Show help  - ▁▁▁▁▁▁▁▁the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add configuration The pipeline will  Show help  - ▁▁▁▁▁▁▁▁        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use code lintersThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - prettier. - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Include citationsInclude pipeline tools Show help  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add Github CI testsThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use reference genomesThe pipeline will be  Show help  + ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from ▂▂ + iGenomes + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add Github badgesThe README.md file of  Show help  + ▁▁▁▁▁▁▁▁the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add configuration The pipeline will  Show help  + ▁▁▁▁▁▁▁▁        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use code lintersThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + editor-config and  + prettier. + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Include citationsInclude pipeline tools Show help  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2511,254 +2511,254 @@ font-weight: 700; } - .terminal-1052877418-matrix { + .terminal-3752496172-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1052877418-title { + .terminal-3752496172-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1052877418-r1 { fill: #c5c8c6 } - .terminal-1052877418-r2 { fill: #e3e3e3 } - .terminal-1052877418-r3 { fill: #989898 } - .terminal-1052877418-r4 { fill: #e1e1e1 } - .terminal-1052877418-r5 { fill: #4ebf71;font-weight: bold } - .terminal-1052877418-r6 { fill: #1e1e1e } - .terminal-1052877418-r7 { fill: #507bb3 } - .terminal-1052877418-r8 { fill: #e2e2e2 } - .terminal-1052877418-r9 { fill: #808080 } - .terminal-1052877418-r10 { fill: #dde6ed;font-weight: bold } - .terminal-1052877418-r11 { fill: #001541 } - .terminal-1052877418-r12 { fill: #454a50 } - .terminal-1052877418-r13 { fill: #7ae998 } - .terminal-1052877418-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-1052877418-r15 { fill: #0a180e;font-weight: bold } - .terminal-1052877418-r16 { fill: #000000 } - .terminal-1052877418-r17 { fill: #008139 } - .terminal-1052877418-r18 { fill: #fea62b;font-weight: bold } - .terminal-1052877418-r19 { fill: #a7a9ab } - .terminal-1052877418-r20 { fill: #e2e3e3 } + .terminal-3752496172-r1 { fill: #c5c8c6 } + .terminal-3752496172-r2 { fill: #e3e3e3 } + .terminal-3752496172-r3 { fill: #989898 } + .terminal-3752496172-r4 { fill: #e1e1e1 } + .terminal-3752496172-r5 { fill: #4ebf71;font-weight: bold } + .terminal-3752496172-r6 { fill: #1e1e1e } + .terminal-3752496172-r7 { fill: #507bb3 } + .terminal-3752496172-r8 { fill: #e2e2e2 } + .terminal-3752496172-r9 { fill: #808080 } + .terminal-3752496172-r10 { fill: #dde6ed;font-weight: bold } + .terminal-3752496172-r11 { fill: #001541 } + .terminal-3752496172-r12 { fill: #454a50 } + .terminal-3752496172-r13 { fill: #7ae998 } + .terminal-3752496172-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-3752496172-r15 { fill: #0a180e;font-weight: bold } + .terminal-3752496172-r16 { fill: #000000 } + .terminal-3752496172-r17 { fill: #008139 } + .terminal-3752496172-r18 { fill: #fea62b;font-weight: bold } + .terminal-3752496172-r19 { fill: #a7a9ab } + .terminal-3752496172-r20 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use reference genomesThe pipeline will be  Show help  - ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most common - reference genome files  - from iGenomes - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use multiqcThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include the MultiQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - module which generates  - an HTML report for  - quality control. - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use fastqcThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include the FastQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - module which performs  - quality control  - analysis of input FASTQ - files. - - - - - - - - - - - - - - - - - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use reference genomesThe pipeline will be  Show help  + ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most common + reference genome files  + from iGenomes + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use multiqcThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include the MultiQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + module which generates  + an HTML report for  + quality control. + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use fastqcThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include the FastQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + module which performs  + quality control  + analysis of input FASTQ + files. + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use nf-schemaUse nf-schema for this  Show help  + ▁▁▁▁▁▁▁▁pipeline.▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + + + + + + + + + + + + + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  From 2ccbdd9b64bd9e3cb2792408a3920c6bc314dba4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Aug 2024 11:37:05 +0200 Subject: [PATCH 46/63] skip linting before and after Text params for custom pipelines --- nf_core/pipelines/create/templatefeatures.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 7fd7880cba..057b5a6f33 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -141,6 +141,10 @@ is_nfcore: nextflow_config: - "manifest.name" - "manifest.homePage" + - "validation.help.beforeText" + - "validation.help.afterText" + - "validation.sumary.beforeText" + - "validation.sumary.afterText" multiqc_config: - "report_comment" nfcore_pipelines: False From ed01eba16b92d98db06eaa10bfe6282cd41821df Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 27 Aug 2024 11:42:50 +0200 Subject: [PATCH 47/63] typo --- nf_core/pipelines/create/templatefeatures.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 057b5a6f33..940d124085 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -143,8 +143,8 @@ is_nfcore: - "manifest.homePage" - "validation.help.beforeText" - "validation.help.afterText" - - "validation.sumary.beforeText" - - "validation.sumary.afterText" + - "validation.summary.beforeText" + - "validation.summary.afterText" multiqc_config: - "report_comment" nfcore_pipelines: False From 1b597f17c3ea20e5932f1b92b72887190a5a9a9c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:38:04 +0200 Subject: [PATCH 48/63] Update nf_core/pipeline-template/nextflow.config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index b4c7f41134..c9437d45f2 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -275,7 +275,7 @@ validation { help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " - {% if is_nfcore %} + {%- if is_nfcore %} beforeText = """ -\033[2m----------------------------------------------------\033[0m- \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m From 93cc41be471cd4e50eb83ed5547e4d65496b65fc Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:40:36 +0200 Subject: [PATCH 49/63] Update nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- .../subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 4fcda5a7da..92aaa52a11 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -130,7 +130,7 @@ workflow PIPELINE_COMPLETION { {% if multiqc %}multiqc_report // string: Path to MultiQC report{% endif %} main: - {% if nf_schema %} + {%- if nf_schema %} summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") {% else %} summary_params = [:] From 6e358f3fdb2e2a893a87e7a642e61d788d4c8d8c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:40:44 +0200 Subject: [PATCH 50/63] Update nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- .../subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 92aaa52a11..71c96dea1c 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -132,7 +132,7 @@ workflow PIPELINE_COMPLETION { main: {%- if nf_schema %} summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - {% else %} + {%- else %} summary_params = [:] {% endif %} From dbe9318ac9eb698f6c4d53c2768217289574ba47 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:40:53 +0200 Subject: [PATCH 51/63] Update nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- .../subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 71c96dea1c..ae66f3674c 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -134,7 +134,7 @@ workflow PIPELINE_COMPLETION { summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") {%- else %} summary_params = [:] - {% endif %} + {%- endif %} // // Completion email and summary From 4025b546d6a9090099636fe821e489d1fd07ba45 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:41:26 +0200 Subject: [PATCH 52/63] Update nf_core/pipelines/create/templatefeatures.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/pipelines/create/templatefeatures.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 940d124085..8d31852ab9 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -260,7 +260,7 @@ nf_schema: skippable_paths: - "subworkflows/nf-core/utils_nfschema_plugin" short_description: "Use nf-schema" - description: "Use nf-schema for this pipeline." + description: "Use the nf-schema Nextflow plugin for this pipeline." help_text: | nf-schema is used to validate input parameters based on a JSON schema. It also provides helper functionality to create help messages, get a summary From 679de826cfd3e6526d718e2d95a8ba2da057d512 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:42:58 +0200 Subject: [PATCH 53/63] Update tests/pipelines/lint/test_plugin_includes.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- tests/pipelines/lint/test_plugin_includes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/lint/test_plugin_includes.py b/tests/pipelines/lint/test_plugin_includes.py index ca14d5456d..8eb31e2671 100644 --- a/tests/pipelines/lint/test_plugin_includes.py +++ b/tests/pipelines/lint/test_plugin_includes.py @@ -3,7 +3,7 @@ from ..test_lint import TestLint -class TestLintNextflowConfig(TestLint): +class TestLintPluginIncludes(TestLint): def setUp(self) -> None: super().setUp() self.new_pipeline = self._make_pipeline_copy() From 3ac1bd8e42a986381c12941b94ce786f8fb393f4 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 12:46:56 +0200 Subject: [PATCH 54/63] review suggestions --- nf_core/pipeline-template/modules.json | 2 +- .../nf-core/utils_nextflow_pipeline/main.nf | 2 +- nf_core/pipelines/lint/files_exist.py | 22 ++----------------- tests/pipelines/lint/test_files_exist.py | 21 ------------------ 4 files changed, 4 insertions(+), 43 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index ec1cf1d963..9f44a4a672 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -23,7 +23,7 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "2aacc9ef22bfc627b18eca17e9a3bb13c58aab64", + "git_sha": "0710581f16d6f4e6b27ab6f410757c8259fdd6f4", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index b54fc16c71..28e32b200e 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -98,7 +98,7 @@ def checkCondaChannels() { // Check that all channels are present // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def required_channels_in_order = ['conda-forge', 'bioconda'] def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order diff --git a/nf_core/pipelines/lint/files_exist.py b/nf_core/pipelines/lint/files_exist.py index 7bfbae8ec9..99f2d1b937 100644 --- a/nf_core/pipelines/lint/files_exist.py +++ b/nf_core/pipelines/lint/files_exist.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Union log = logging.getLogger(__name__) @@ -201,7 +201,6 @@ def files_exist(self) -> Dict[str, List[str]]: Path("lib", "nfcore_external_java_deps.jar"), ] files_warn_ifexists = [Path(".travis.yml")] - files_fail_ifinconfig: List[Tuple[Path, List[Dict[str, str]]]] = [] # Remove files that should be ignored according to the linting config ignore_files = self.lint_config.get("files_exist", []) if self.lint_config is not None else [] @@ -240,24 +239,7 @@ def pf(file_path: Union[str, Path]) -> Path: failed.append(f"File must be removed: {self._wrap_quotes(file)}") else: passed.append(f"File not found check: {self._wrap_quotes(file)}") - # Files that cause an error if they exists together with a certain entry in nextflow.config - for file_cond in files_fail_ifinconfig: - if str(file_cond[0]) in ignore_files: - continue - in_config = False - for condition in file_cond[1]: - config_key, config_value = list(condition.items())[0] - if config_key in self.nf_config and config_value in self.nf_config[config_key]: - log.debug(f"Found {config_key} in nextflow.config with value {config_value}") - in_config = True - if pf(file_cond[0]).is_file() and in_config: - failed.append(f"File must be removed: {self._wrap_quotes(file_cond[0])}") - elif pf(file_cond[0]).is_file() and not in_config: - passed.append(f"File found check: {self._wrap_quotes(file_cond[0])}") - elif not pf(file_cond[0]).is_file() and not in_config: - failed.append(f"File not found check: {self._wrap_quotes(file_cond[0])}") - elif not pf(file_cond[0]).is_file() and in_config: - passed.append(f"File not found check: {self._wrap_quotes(file_cond[0])}") + # Files that cause a warning if they exist for file in files_warn_ifexists: if str(file) in ignore_files: diff --git a/tests/pipelines/lint/test_files_exist.py b/tests/pipelines/lint/test_files_exist.py index 65217903f8..97dd346cdf 100644 --- a/tests/pipelines/lint/test_files_exist.py +++ b/tests/pipelines/lint/test_files_exist.py @@ -54,27 +54,6 @@ def test_files_exist_pass(self): results = lint_obj.files_exist() assert results["failed"] == [] - def test_files_exist_pass_conditional(self): - lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) - lint_obj._load() - lint_obj.nf_config["plugins"] = [] - lib_dir = Path(self.new_pipeline, "lib") - lib_dir.mkdir() - (lib_dir / "nfcore_external_java_deps.jar").touch() - results = lint_obj.files_exist() - assert results["failed"] == ["File must be removed: `lib/nfcore_external_java_deps.jar`"] - assert results["ignored"] == [] - - def test_files_exist_fail_conditional(self): - lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) - lint_obj._load() - lib_dir = Path(self.new_pipeline, "lib") - lib_dir.mkdir() - (lib_dir / "nfcore_external_java_deps.jar").touch() - results = lint_obj.files_exist() - assert results["failed"] == ["File must be removed: `lib/nfcore_external_java_deps.jar`"] - assert results["ignored"] == [] - def test_files_exist_pass_conditional_nfschema(self): # replace nf-validation with nf-schema in nextflow.config with open(Path(self.new_pipeline, "nextflow.config")) as f: From d4266c0bb19027db883107d47a50ace106affff7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 13:32:53 +0200 Subject: [PATCH 55/63] use latest version of utils_nextflow_pipeline --- nf_core/pipeline-template/modules.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 9f44a4a672..caeaac2fdf 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -23,7 +23,7 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "0710581f16d6f4e6b27ab6f410757c8259fdd6f4", + "git_sha": "16f913a1ea8bb2cc48ccfc0b3b9406c5b3333799", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { From bc20099658a5a66e2ca3cc982b2e7cc4735bbab8 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 13:52:18 +0200 Subject: [PATCH 56/63] make help params snake_case --- nf_core/pipeline-template/nextflow.config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index c9437d45f2..8818ec54db 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -40,8 +40,8 @@ params { monochrome_logs = false hook_url = null {% if nf_schema %}help = false - helpFull = false - showHidden = false{% endif %} + help_full = false + show_hidden = false{% endif %} version = false {% if test_config %}pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/'{% endif %} @@ -275,6 +275,8 @@ validation { help { enabled = true command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " + fullParameter = "help_full" + showHiddenParameter = "show_hidden" {%- if is_nfcore %} beforeText = """ -\033[2m----------------------------------------------------\033[0m- From 829d594be396edd5a2d54d8536972689b64b5fe6 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:33:58 +0200 Subject: [PATCH 57/63] Update nf_core/pipelines/create/templatefeatures.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipelines/create/templatefeatures.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/create/templatefeatures.yml b/nf_core/pipelines/create/templatefeatures.yml index 8d31852ab9..7388ecc7ba 100644 --- a/nf_core/pipelines/create/templatefeatures.yml +++ b/nf_core/pipelines/create/templatefeatures.yml @@ -262,7 +262,7 @@ nf_schema: short_description: "Use nf-schema" description: "Use the nf-schema Nextflow plugin for this pipeline." help_text: | - nf-schema is used to validate input parameters based on a JSON schema. + [nf-schema](https://nextflow-io.github.io/nf-schema/latest/) is used to validate input parameters based on a JSON schema. It also provides helper functionality to create help messages, get a summary of changed parameters and validate and convert a samplesheet to a channel. nfcore_pipelines: True From 1ff7cbcfe58015b636c8edd852ac422e74598770 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:34:25 +0200 Subject: [PATCH 58/63] Update nf_core/pipelines/lint/plugin_includes.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipelines/lint/plugin_includes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipelines/lint/plugin_includes.py b/nf_core/pipelines/lint/plugin_includes.py index 6a57b1d280..4fc40ae26c 100644 --- a/nf_core/pipelines/lint/plugin_includes.py +++ b/nf_core/pipelines/lint/plugin_includes.py @@ -16,10 +16,10 @@ def plugin_includes(self) -> Dict[str, List[str]]: config_plugins = [plugin.split("@")[0] for plugin in ast.literal_eval(self.nf_config.get("plugins", "[]"))] validation_plugin = "nf-validation" if "nf-validation" in config_plugins else "nf-schema" - passed: list[str] = [] - warned: list[str] = [] - failed: list[str] = [] - ignored: list[str] = [] + passed: List[str] = [] + warned: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] plugin_include_pattern = re.compile(r"^include\s*{[^}]+}\s*from\s*[\"']plugin/([^\"']+)[\"']\s*$", re.MULTILINE) workflow_files = [ From aed94601c7241546def06c226758c96688587cb2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 14:37:03 +0200 Subject: [PATCH 59/63] apply review suggestions --- nf_core/pipelines/lint/nextflow_config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/pipelines/lint/nextflow_config.py b/nf_core/pipelines/lint/nextflow_config.py index 2159d7939a..790fc21797 100644 --- a/nf_core/pipelines/lint/nextflow_config.py +++ b/nf_core/pipelines/lint/nextflow_config.py @@ -168,8 +168,6 @@ def nextflow_config(self) -> Dict[str, List[str]]: if "nf-validation" in found_plugins or "nf-schema" in found_plugins: if "nf-validation" in found_plugins and "nf-schema" in found_plugins: failed.append("nextflow.config contains both nf-validation and nf-schema") - if "nf-validation" not in found_plugins and "nf-schema" not in found_plugins: - warned.append("nextflow.config does not contain `nf-validation` or `nf-schema` in the plugins scope") if "nf-schema" in found_plugins: passed.append("Found nf-schema plugin") From 4407901d8c7a039d97ee45390ead386611de42e2 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 14:41:53 +0200 Subject: [PATCH 60/63] update utils_nextflow_pipeline commit messages --- nf_core/pipeline-template/modules.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index caeaac2fdf..367202155a 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -23,7 +23,7 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "16f913a1ea8bb2cc48ccfc0b3b9406c5b3333799", + "git_sha": "d20fb2a9cc3e2835e9d067d1046a63252eb17352", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { From 330e43c76109f500ee88bdeddd0ef578751d8376 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 14:48:37 +0200 Subject: [PATCH 61/63] update ignoring of help params --- nf_core/pipelines/schema.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 6c1c198bb8..95ed5e5b6e 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -92,7 +92,11 @@ def _update_validation_plugin_from_config(self) -> None: # Previous versions of nf-schema used "defs", but it's advised to use "$defs" if plugin == "nf-schema": self.defs_notation = "$defs" - ignored_params = ["help", "helpFull", "showHidden"] # Help parameter should be ignored by default + ignored_params = [ + conf.get("validation.help.shortParameter", "help"), + conf.get("validation.help.fullParameter", "helpFull"), + conf.get("validation.help.showHiddenParameter", "showHidden"), + ] # Help parameter should be ignored by default ignored_params_config = conf.get("validation", {}).get("defaultIgnoreParams", []) if len(ignored_params_config) > 0: ignored_params.extend(ignored_params_config) From 3600db75edf62709fcd48c6174c6b698c95c93f3 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 14:57:12 +0200 Subject: [PATCH 62/63] update file exists docstring --- nf_core/pipelines/lint/files_exist.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nf_core/pipelines/lint/files_exist.py b/nf_core/pipelines/lint/files_exist.py index 99f2d1b937..bd25ff33d0 100644 --- a/nf_core/pipelines/lint/files_exist.py +++ b/nf_core/pipelines/lint/files_exist.py @@ -87,6 +87,7 @@ def files_exist(self) -> Dict[str, List[str]]: lib/Workflow.groovy lib/WorkflowMain.groovy lib/WorkflowPIPELINE.groovy + lib/nfcore_external_java_deps.jar parameters.settings.json pipeline_template.yml # saving information in .nf-core.yml Singularity @@ -98,12 +99,6 @@ def files_exist(self) -> Dict[str, List[str]]: .travis.yml - Files that *must not* be present if a certain entry is present in ``nextflow.config``: - - .. code-block:: bash - - lib/nfcore_external_java_deps.jar # if "nf-validation" is in nextflow.config - .. tip:: You can configure the ``nf-core pipelines lint`` tests to ignore any of these checks by setting the ``files_exist`` key as follows in your ``.nf-core.yml`` config file. For example: From e816fafe8a1c984bf54b7f78979c35f494b1a2d9 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 27 Aug 2024 15:32:20 +0200 Subject: [PATCH 63/63] update snapshot --- .../__snapshots__/test_create_app.ambr | 504 +++++++++--------- 1 file changed, 252 insertions(+), 252 deletions(-) diff --git a/tests/pipelines/__snapshots__/test_create_app.ambr b/tests/pipelines/__snapshots__/test_create_app.ambr index 3ed8349ee1..af88080668 100644 --- a/tests/pipelines/__snapshots__/test_create_app.ambr +++ b/tests/pipelines/__snapshots__/test_create_app.ambr @@ -2233,255 +2233,255 @@ font-weight: 700; } - .terminal-3458551717-matrix { + .terminal-1373392807-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3458551717-title { + .terminal-1373392807-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3458551717-r1 { fill: #c5c8c6 } - .terminal-3458551717-r2 { fill: #e3e3e3 } - .terminal-3458551717-r3 { fill: #989898 } - .terminal-3458551717-r4 { fill: #e1e1e1 } - .terminal-3458551717-r5 { fill: #4ebf71;font-weight: bold } - .terminal-3458551717-r6 { fill: #1e1e1e } - .terminal-3458551717-r7 { fill: #507bb3 } - .terminal-3458551717-r8 { fill: #e2e2e2 } - .terminal-3458551717-r9 { fill: #808080 } - .terminal-3458551717-r10 { fill: #dde6ed;font-weight: bold } - .terminal-3458551717-r11 { fill: #001541 } - .terminal-3458551717-r12 { fill: #14191f } - .terminal-3458551717-r13 { fill: #454a50 } - .terminal-3458551717-r14 { fill: #7ae998 } - .terminal-3458551717-r15 { fill: #e2e3e3;font-weight: bold } - .terminal-3458551717-r16 { fill: #0a180e;font-weight: bold } - .terminal-3458551717-r17 { fill: #000000 } - .terminal-3458551717-r18 { fill: #008139 } - .terminal-3458551717-r19 { fill: #fea62b;font-weight: bold } - .terminal-3458551717-r20 { fill: #a7a9ab } - .terminal-3458551717-r21 { fill: #e2e3e3 } + .terminal-1373392807-r1 { fill: #c5c8c6 } + .terminal-1373392807-r2 { fill: #e3e3e3 } + .terminal-1373392807-r3 { fill: #989898 } + .terminal-1373392807-r4 { fill: #e1e1e1 } + .terminal-1373392807-r5 { fill: #4ebf71;font-weight: bold } + .terminal-1373392807-r6 { fill: #1e1e1e } + .terminal-1373392807-r7 { fill: #507bb3 } + .terminal-1373392807-r8 { fill: #e2e2e2 } + .terminal-1373392807-r9 { fill: #808080 } + .terminal-1373392807-r10 { fill: #dde6ed;font-weight: bold } + .terminal-1373392807-r11 { fill: #001541 } + .terminal-1373392807-r12 { fill: #14191f } + .terminal-1373392807-r13 { fill: #454a50 } + .terminal-1373392807-r14 { fill: #7ae998 } + .terminal-1373392807-r15 { fill: #e2e3e3;font-weight: bold } + .terminal-1373392807-r16 { fill: #0a180e;font-weight: bold } + .terminal-1373392807-r17 { fill: #000000 } + .terminal-1373392807-r18 { fill: #008139 } + .terminal-1373392807-r19 { fill: #fea62b;font-weight: bold } + .terminal-1373392807-r20 { fill: #a7a9ab } + .terminal-1373392807-r21 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - + - - nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add Github CI testsThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - actions for Continuous - Integration (CI)  - testing - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use reference genomesThe pipeline will be  Show help  - ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most  - common reference  - genome files from ▂▂ - iGenomes - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add Github badgesThe README.md file of  Show help  - ▁▁▁▁▁▁▁▁the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - include GitHub badges - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Add configuration The pipeline will  Show help  - ▁▁▁▁▁▁▁▁        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - profiles containing  - custom parameters  - requried to run  - nf-core pipelines at  - different institutions - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use code lintersThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - and CI tests to lint  - your code: pre-commit, - editor-config and  - prettier. - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Include citationsInclude pipeline tools Show help  - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add Github CI testsThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include several GitHub▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + actions for Continuous + Integration (CI)  + testing + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use reference genomesThe pipeline will be  Show help  + ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most  + common reference  + genome files from ▃▃ + iGenomes + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add Github badgesThe README.md file of  Show help  + ▁▁▁▁▁▁▁▁the pipeline will ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + include GitHub badges + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Add configuration The pipeline will  Show help  + ▁▁▁▁▁▁▁▁        filesinclude configuration ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + profiles containing  + custom parameters  + requried to run  + nf-core pipelines at  + different institutions + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use code lintersThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include code linters ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + and CI tests to lint  + your code: pre-commit, + editor-config and  + prettier. + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Include citationsInclude pipeline tools Show help  + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit  @@ -2511,254 +2511,254 @@ font-weight: 700; } - .terminal-3752496172-matrix { + .terminal-3384959394-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3752496172-title { + .terminal-3384959394-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3752496172-r1 { fill: #c5c8c6 } - .terminal-3752496172-r2 { fill: #e3e3e3 } - .terminal-3752496172-r3 { fill: #989898 } - .terminal-3752496172-r4 { fill: #e1e1e1 } - .terminal-3752496172-r5 { fill: #4ebf71;font-weight: bold } - .terminal-3752496172-r6 { fill: #1e1e1e } - .terminal-3752496172-r7 { fill: #507bb3 } - .terminal-3752496172-r8 { fill: #e2e2e2 } - .terminal-3752496172-r9 { fill: #808080 } - .terminal-3752496172-r10 { fill: #dde6ed;font-weight: bold } - .terminal-3752496172-r11 { fill: #001541 } - .terminal-3752496172-r12 { fill: #454a50 } - .terminal-3752496172-r13 { fill: #7ae998 } - .terminal-3752496172-r14 { fill: #e2e3e3;font-weight: bold } - .terminal-3752496172-r15 { fill: #0a180e;font-weight: bold } - .terminal-3752496172-r16 { fill: #000000 } - .terminal-3752496172-r17 { fill: #008139 } - .terminal-3752496172-r18 { fill: #fea62b;font-weight: bold } - .terminal-3752496172-r19 { fill: #a7a9ab } - .terminal-3752496172-r20 { fill: #e2e3e3 } + .terminal-3384959394-r1 { fill: #c5c8c6 } + .terminal-3384959394-r2 { fill: #e3e3e3 } + .terminal-3384959394-r3 { fill: #989898 } + .terminal-3384959394-r4 { fill: #e1e1e1 } + .terminal-3384959394-r5 { fill: #4ebf71;font-weight: bold } + .terminal-3384959394-r6 { fill: #1e1e1e } + .terminal-3384959394-r7 { fill: #507bb3 } + .terminal-3384959394-r8 { fill: #e2e2e2 } + .terminal-3384959394-r9 { fill: #808080 } + .terminal-3384959394-r10 { fill: #dde6ed;font-weight: bold } + .terminal-3384959394-r11 { fill: #001541 } + .terminal-3384959394-r12 { fill: #454a50 } + .terminal-3384959394-r13 { fill: #7ae998 } + .terminal-3384959394-r14 { fill: #e2e3e3;font-weight: bold } + .terminal-3384959394-r15 { fill: #0a180e;font-weight: bold } + .terminal-3384959394-r16 { fill: #000000 } + .terminal-3384959394-r17 { fill: #008139 } + .terminal-3384959394-r18 { fill: #fea62b;font-weight: bold } + .terminal-3384959394-r19 { fill: #a7a9ab } + .terminal-3384959394-r20 { fill: #e2e3e3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - nf-core create + nf-core create - - - - nf-core create — Create a new pipeline with the nf-core pipeline template - - - Template features - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use reference genomesThe pipeline will be  Show help  - ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - copy of the most common - reference genome files  - from iGenomes - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use multiqcThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include the MultiQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - module which generates  - an HTML report for  - quality control. - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use fastqcThe pipeline will  Show help  - ▁▁▁▁▁▁▁▁include the FastQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - module which performs  - quality control  - analysis of input FASTQ - files. - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -         Use nf-schemaUse nf-schema for this  Show help  - ▁▁▁▁▁▁▁▁pipeline.▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - - - - - - - - - - - - - - - ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ -  Back  Continue  - ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ - -  d Toggle dark mode  q Quit  + + + + nf-core create — Create a new pipeline with the nf-core pipeline template + + + Template features + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use reference genomesThe pipeline will be  Show help  + ▁▁▁▁▁▁▁▁configured to use a ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + copy of the most common + reference genome files  + from iGenomes + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use multiqcThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include the MultiQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + module which generates  + an HTML report for  + quality control. + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use fastqcThe pipeline will  Show help  + ▁▁▁▁▁▁▁▁include the FastQC ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + module which performs  + quality control  + analysis of input FASTQ + files. + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +         Use nf-schemaUse the nf-schema  Show help  + ▁▁▁▁▁▁▁▁Nextflow plugin for ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + this pipeline. + + + + + + + + + + + + + + ▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔▔ +  Back  Continue  + ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ + +  d Toggle dark mode  q Quit