diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index f76187d6af..46152d92a7 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -84,6 +84,10 @@ jobs: - name: remove TODO run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; + # Replace zenodo.XXXXXX to pass readme linting + - name: replace zenodo.XXXXXX + run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; + # Run nf-core linting - name: nf-core lint run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned diff --git a/CHANGELOG.md b/CHANGELOG.md index b50612ff5b..5df2a15469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Added support for the apptainer container engine via `-profile apptainer`. ([#2244](https://github.com/nf-core/tools/issues/2244)) [Contributed by @jfy133] - Added config `docker.registry` to pipeline template for a configurable default container registry when using Docker containers. Defaults to `quay.io` ([#2133](https://github.com/nf-core/tools/pull/2133)) - Add tower.yml file to the pipeline template ([#2251](https://github.com/nf-core/tools/pull/2251)) +- Add mastodon badge to README ([#2253](https://github.com/nf-core/tools/pull/2253)) - Removed `quay.io` from all module Docker container references as this is now supplied at pipeline level. ([#2249](https://github.com/nf-core/tools/pull/2249)) ### Linting @@ -24,6 +25,7 @@ - Update modules lint test to fail if enable_conda is found ([#2213](https://github.com/nf-core/tools/pull/2213)) - Read module lint configuration from `.nf-core.yml`, not `.nf-core-lint.yml` ([#2221](https://github.com/nf-core/tools/pull/2221)) - `nf-core schema lint` now defaults to linting `nextflow_schema.json` if no filename is provided ([#2225](https://github.com/nf-core/tools/pull/2225)) +- Warn if `/zenodo.XXXXXX` is present in the Readme ([#2254](https://github.com/nf-core/tools/pull/2254)) ### Modules diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index 5f6dd87030..ae5c542837 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -17,16 +17,12 @@ def readme(self): [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) - * Bioconda badge + .. note:: This badge are a markdown image ``![alt-text]()`` *inside* a markdown link ``[markdown image]()``, so a bit fiddly to write. - * If your pipeline contains a file called ``environment.yml`` in the root directory, a bioconda badge is required - * Required badge code: + * Zenodo release - .. code-block:: md - - [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + * If pipeline is released but still contains a 'zenodo.XXXXXXX' tag, the test fails - .. note:: These badges are a markdown image ``![alt-text]()`` *inside* a markdown link ``[markdown image]()``, so a bit fiddly to write. """ passed = [] warned = [] @@ -62,4 +58,16 @@ def readme(self): else: warned.append("README did not have a Nextflow minimum version badge.") + if "zenodo_doi" not in ignore_configs: + # Check that zenodo.XXXXXXX has been replaced with the zendo.DOI + zenodo_re = r"/zenodo\.X+" + match = re.search(zenodo_re, content) + if match: + warned.append( + "README contains the placeholder `zenodo.XXXXXXX`. " + "This should be replaced with the zenodo doi (after the first release)." + ) + else: + passed.append("README Zenodo placeholder was replaced with DOI.") + return {"passed": passed, "warned": warned, "failed": failed} diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 9b42bb3bdf..2c8197dcba 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,3 +1,5 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords @@ -5,6 +7,8 @@ name: "{{ component_name_underscore }}" description: write your description here keywords: - sort + - example + - genomics tools: - "{{ component }}": {% if not_empty_template -%} @@ -32,9 +36,9 @@ input: ## TODO nf-core: Delete / customise this example input {%- endif %} - {{ 'bam:' if not_empty_template else "input:" }} - type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + type: file + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} {% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as output @@ -55,9 +59,9 @@ output: ## TODO nf-core: Delete / customise this example output {%- endif %} - {{ 'bam:' if not_empty_template else "output:" }} - type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + type: file + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} authors: - "{{ author }}" diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 6aea64705f..e5aaa9e0c4 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -235,26 +235,9 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("process_capitals", "Process name is not in capital letters", self.main_nf)) # Check that process labels are correct - correct_process_labels = ["process_single", "process_low", "process_medium", "process_high", "process_long"] - process_label = [l for l in lines if l.lstrip().startswith("label")] - if len(process_label) > 0: - try: - process_label = re.search("process_[A-Za-z]+", process_label[0]).group(0) - except AttributeError: - process_label = re.search("'([A-Za-z_-]+)'", process_label[0]).group(0) - finally: - if not process_label in correct_process_labels: - self.warned.append( - ( - "process_standard_label", - f"Process label ({process_label}) is not among standard labels: `{'`,`'.join(correct_process_labels)}`", - self.main_nf, - ) - ) - else: - self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) - else: - self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) + check_process_labels(self, lines) + + # Deprecated enable_conda for i, l in enumerate(lines): url = None l = l.strip(" '\"") @@ -412,6 +395,56 @@ def check_process_section(self, lines, fix_version, progress_bar): return docker_tag == singularity_tag +def check_process_labels(self, lines): + correct_process_labels = ["process_single", "process_low", "process_medium", "process_high", "process_long"] + all_labels = [l.strip() for l in lines if l.lstrip().startswith("label ")] + bad_labels = [] + good_labels = [] + if len(all_labels) > 0: + for label in all_labels: + try: + label = re.match("^label\s+([a-zA-Z0-9_-]+)$", label).group(1) + except AttributeError: + self.warned.append( + ( + "process_standard_label", + f"Specified label appears to contain non-alphanumerics: {label}", + self.main_nf, + ) + ) + continue + if label not in correct_process_labels: + bad_labels.append(label) + else: + good_labels.append(label) + if len(good_labels) > 1: + self.warned.append( + ( + "process_standard_label", + f"Conflicting process labels found: `{'`,`'.join(good_labels)}`", + self.main_nf, + ) + ) + elif len(good_labels) == 1: + self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) + else: + self.warned.append(("process_standard_label", "Standard process label not found", self.main_nf)) + if len(bad_labels) > 0: + self.warned.append( + ("process_standard_label", f"Non-standard labels found: `{'`,`'.join(bad_labels)}`", self.main_nf) + ) + if len(all_labels) > len(set(all_labels)): + self.warned.append( + ( + "process_standard_label", + f"Duplicate labels found: `{'`,`'.join(sorted(all_labels))}`", + self.main_nf, + ) + ) + else: + self.warned.append(("process_standard_label", "Process label not specified", self.main_nf)) + + def _parse_input(self, line_raw): """ Return list of input channel names from an input line. diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d6ec296999..dd5e954f25 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -1,5 +1,7 @@ +import json from pathlib import Path +import jsonschema.validators import yaml from nf_core.modules.modules_differ import ModulesDiffer @@ -10,17 +12,15 @@ def meta_yml(module_lint_object, module): Lint a ``meta.yml`` file The lint test checks that the module has - a ``meta.yml`` file and that it contains - the required keys: ``name``, input`` and - ``output``. + a ``meta.yml`` file and that it follows the + JSON schema defined in the ``modules/yaml-schema.json`` + file in the nf-core/modules repository. In addition it checks that the module name and module input is consistent between the ``meta.yml`` and the ``main.nf``. """ - required_keys = ["name", "output"] - required_keys_lists = ["input", "output"] # Check if we have a patch file, get original file in that case meta_yaml = None if module.is_patched: @@ -42,21 +42,31 @@ def meta_yml(module_lint_object, module): module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return - # Confirm that all required keys are given - contains_required_keys = True - all_list_children = True - for rk in required_keys: - if rk not in meta_yaml.keys(): - module.failed.append(("meta_required_keys", f"`{rk}` not specified in YAML", module.meta_yml)) - contains_required_keys = False - elif rk in meta_yaml.keys() and not isinstance(meta_yaml[rk], list) and rk in required_keys_lists: - module.failed.append(("meta_required_keys", f"`{rk}` is not a list", module.meta_yml)) - all_list_children = False - if contains_required_keys: - module.passed.append(("meta_required_keys", "`meta.yml` contains all required keys", module.meta_yml)) + # Confirm that the meta.yml file is valid according to the JSON schema + valid_meta_yml = True + try: + with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/yaml-schema.json"), "r") as fh: + schema = json.load(fh) + jsonschema.validators.validate(instance=meta_yaml, schema=schema) + module.passed.append(("meta_yml_valid", "Module `meta.yml` is valid", module.meta_yml)) + except jsonschema.exceptions.ValidationError as e: + valid_meta_yml = False + hint = "" + if len(e.path) > 0: + hint = f"\nCheck the entry for `{e.path[0]}`." + if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: + hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + module.failed.append( + ( + "meta_yml_valid", + f"The `meta.yml` of the module {module.module_name} is not valid: {e.message}.{hint}", + module.meta_yml, + ) + ) + return # Confirm that all input and output channels are specified - if contains_required_keys and all_list_children: + if valid_meta_yml: if "input" in meta_yaml: meta_input = [list(x.keys())[0] for x in meta_yaml["input"]] for input in module.inputs: diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 1bb2cba4f1..e2ca15a8e6 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -19,6 +19,7 @@ {% endif -%} {%- if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} {%- if branded -%}[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core){% endif -%} +{%- if branded -%}[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core){% endif -%} {%- if branded -%}[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) {% endif -%} diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml index 60b546a012..c1bccd5f76 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,8 +1,11 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: - custom - - version + - software + - versions + tools: - custom: description: Custom module used to dump software versions within the nf-core pipeline template diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml index 3db57b6fb1..4c5b454ddf 100644 --- a/nf_core/subworkflow-template/subworkflows/meta.yml +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json name: "{{ subworkflow_name }}" ## TODO nf-core: Add a description of the subworkflow and list keywords description: Sort SAM/BAM/CRAM file diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 9bab9eddeb..90697fbf90 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -4,6 +4,7 @@ import pytest import nf_core.modules +from nf_core.modules.lint import main_nf from ..utils import GITLAB_URL, set_wd from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf @@ -43,7 +44,7 @@ def test_modules_lint_empty(self): def test_modules_lint_new_modules(self): - """lint all modules in nf-core/modules repo clone""" + """lint a new module""" module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=True, all_modules=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" @@ -105,3 +106,113 @@ def test_modules_lint_patched_modules(self): assert len(module_lint.failed) == 1 assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 + + +# A skeleton object with the passed/warned/failed list attrs +# Use this in place of a ModuleLint object to test behaviour of +# linting methods which don't need the full setup +class MockModuleLint: + def __init__(self): + self.passed = [] + self.warned = [] + self.failed = [] + + self.main_nf = "main_nf" + + +PROCESS_LABEL_GOOD = ( + """ + label process_high + cpus 12 + """, + 1, + 0, + 0, +) +PROCESS_LABEL_NON_ALPHANUMERIC = ( + """ + label a:label:with:colons + cpus 12 + """, + 0, + 2, + 0, +) +PROCESS_LABEL_GOOD_CONFLICTING = ( + """ + label process_high + label process_low + cpus 12 + """, + 0, + 1, + 0, +) +PROCESS_LABEL_GOOD_DUPLICATES = ( + """ + label process_high + label process_high + cpus 12 + """, + 0, + 2, + 0, +) +PROCESS_LABEL_GOOD_AND_NONSTANDARD = ( + """ + label process_high + label process_extra_label + cpus 12 + """, + 1, + 1, + 0, +) +PROCESS_LABEL_NONSTANDARD = ( + """ + label process_extra_label + cpus 12 + """, + 0, + 2, + 0, +) +PROCESS_LABEL_NONSTANDARD_DUPLICATES = ( + """ + label process_extra_label + label process_extra_label + cpus 12 + """, + 0, + 3, + 0, +) +PROCESS_LABEL_NONE_FOUND = ( + """ + cpus 12 + """, + 0, + 1, + 0, +) + +PROCESS_LABEL_TEST_CASES = [ + PROCESS_LABEL_GOOD, + PROCESS_LABEL_NON_ALPHANUMERIC, + PROCESS_LABEL_GOOD_CONFLICTING, + PROCESS_LABEL_GOOD_DUPLICATES, + PROCESS_LABEL_GOOD_AND_NONSTANDARD, + PROCESS_LABEL_NONSTANDARD, + PROCESS_LABEL_NONSTANDARD_DUPLICATES, + PROCESS_LABEL_NONE_FOUND, +] + + +def test_modules_lint_check_process_labels(self): + for test_case in PROCESS_LABEL_TEST_CASES: + process, passed, warned, failed = test_case + mocked_ModuleLint = MockModuleLint() + main_nf.check_process_labels(mocked_ModuleLint, process.splitlines()) + assert len(mocked_ModuleLint.passed) == passed + assert len(mocked_ModuleLint.warned) == warned + assert len(mocked_ModuleLint.failed) == failed diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 494378e490..95cc2cad95 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -18,7 +18,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed" +CORRECT_SHA = "0245a9277d51a47c8aa68d264d294cf45312fab8" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" diff --git a/tests/test_modules.py b/tests/test_modules.py index 74596822c1..047369b7c3 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -45,6 +45,17 @@ def create_modules_repo_dummy(tmp_dir): with requests_cache.disabled(): module_create.create() + # Remove doi from meta.yml which makes lint fail + meta_yml = os.path.join(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") + with open(meta_yml, "r") as fh: + lines = fh.readlines() + for line_index in range(len(lines)): + if "doi" in lines[line_index]: + to_pop = line_index + lines.pop(to_pop) + with open(meta_yml, "w") as fh: + fh.writelines(lines) + return root_dir @@ -162,6 +173,7 @@ def test_modulesrepo_class(self): test_modules_install_trimgalore_twice, ) from .modules.lint import ( + test_modules_lint_check_process_labels, test_modules_lint_empty, test_modules_lint_gitlab_modules, test_modules_lint_multiple_remotes, diff --git a/tests/utils.py b/tests/utils.py index 85fef70eea..d39d172a66 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -80,8 +80,8 @@ def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): anaconda_mock = { "latest_version": version.split("--")[0], "summary": "", - "doc_url": "", - "dev_url": "", + "doc_url": "http://test", + "dev_url": "http://test", "files": [{"version": version.split("--")[0]}], "license": "", }