From faf85296f65dae7b000eeaea12fe0c96494224fe Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 22 Jan 2024 11:43:55 +0100 Subject: [PATCH 1/6] linting a pipeline also lints the installing subworkflows --- nf_core/lint/__init__.py | 23 +++++++++++++++++++++-- nf_core/lint_utils.py | 10 +++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 797ebbcc91..6713b97a04 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -20,6 +20,7 @@ import nf_core.lint_utils import nf_core.modules.lint +import nf_core.subworkflows.lint import nf_core.utils from nf_core import __version__ from nf_core.lint_utils import console @@ -53,6 +54,8 @@ def run_linting( Returns: An object of type :class:`PipelineLint` that contains all the linting results. + An object of type :class:`ComponentLint` that contains all the linting results for the modules. + An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. """ # Verify that the requested tests exist @@ -87,6 +90,8 @@ def run_linting( # Create the modules lint object module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) + # Create the subworkflows lint object + subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) # Verify that the pipeline is correctly configured and has a modules.json file module_lint_obj.has_valid_directory() @@ -98,10 +103,18 @@ def run_linting( module_lint_tests = list( set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) ) + # Select only the subworkflow lint tests + subworkflow_lint_tests = list( + set(key).intersection( + set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) + ) + ) else: # If no key is supplied, run the default modules tests module_lint_tests = ("module_changes", "module_version") + subworkflow_lint_tests = ("subworkflow_changes", "subworkflow_version") module_lint_obj.filter_tests_by_key(module_lint_tests) + subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) # Set up files for modules linting test module_lint_obj.set_up_pipeline_files() @@ -119,11 +132,17 @@ def run_linting( module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) if len(module_lint_obj.all_remote_components) > 0: module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) + # Run the subworkflows lint tests + if len(subworkflow_lint_obj.all_local_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) + if len(subworkflow_lint_obj.all_remote_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) # Print the results lint_obj._print_results(show_passed) module_lint_obj._print_results(show_passed, sort_by=sort_by) - nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj) + subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) + nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) nf_core.lint_utils.print_fixes(lint_obj) # Save results to Markdown file @@ -142,7 +161,7 @@ def run_linting( if release_mode: log.info("Reminder: Lint tests were run in --release mode.") - return lint_obj, module_lint_obj + return lint_obj, module_lint_obj, subworkflow_lint_obj class PipelineLint(nf_core.utils.Pipeline): diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index c2fd75d375..f9c1e54e38 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -16,13 +16,13 @@ console = Console(force_terminal=nf_core.utils.rich_force_colors()) -def print_joint_summary(lint_obj, module_lint_obj): - """Print a joint summary of the general pipe lint tests and the module lint tests""" - nbr_passed = len(lint_obj.passed) + len(module_lint_obj.passed) +def print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj): + """Print a joint summary of the general pipe lint tests and the module and subworkflow lint tests""" + nbr_passed = len(lint_obj.passed) + len(module_lint_obj.passed) + len(subworkflow_lint_obj.passed) nbr_ignored = len(lint_obj.ignored) nbr_fixed = len(lint_obj.fixed) - nbr_warned = len(lint_obj.warned) + len(module_lint_obj.warned) - nbr_failed = len(lint_obj.failed) + len(module_lint_obj.failed) + nbr_warned = len(lint_obj.warned) + len(module_lint_obj.warned) + len(subworkflow_lint_obj.warned) + nbr_failed = len(lint_obj.failed) + len(module_lint_obj.failed) + len(subworkflow_lint_obj.failed) summary_colour = "red" if nbr_failed > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) From b2a4ca3305ca6fa3628f5f3b769f08d7f8371bc6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 22 Jan 2024 12:48:40 +0100 Subject: [PATCH 2/6] warn instead of fail when no components are installed from the repo --- nf_core/__main__.py | 4 ++-- nf_core/components/lint/__init__.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index cffabe6c3f..ec22aac93b 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -600,7 +600,7 @@ def lint( # Run the lint tests! try: - lint_obj, module_lint_obj = run_linting( + lint_obj, module_lint_obj, subworkflow_lint_obj = run_linting( dir, release, fix, @@ -613,7 +613,7 @@ def lint( json, ctx.obj["hide_progress"], ) - if len(lint_obj.failed) + len(module_lint_obj.failed) > 0: + if len(lint_obj.failed) + len(module_lint_obj.failed) + len(subworkflow_lint_obj.failed) > 0: sys.exit(1) except AssertionError as e: log.critical(e) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 3c2fb9dde3..b1f40dc666 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -97,9 +97,7 @@ def __init__( ) ) if not self.all_remote_components: - raise LookupError( - f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline." - ) + log.warning(f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline.") local_component_dir = Path(self.dir, self.component_type, "local") self.all_local_components = [] if local_component_dir.exists(): From ccdb2efb59ef75a5e5f08406d50c2fe61c9bb12f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 22 Jan 2024 12:53:08 +0100 Subject: [PATCH 3/6] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index acafa5e7bc..893e0e1c93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ ### Linting +- linting a pipeline also lints the installed subworkflows ([#2677](https://github.com/nf-core/tools/pull/2677)) + ### Modules - Fix linting of a pipeline with patched custom module ([#2669](https://github.com/nf-core/tools/pull/2669)) From 93f6af833037edc1fbb81cdbc1179c1e8e2e739d Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 22 Jan 2024 14:36:14 +0100 Subject: [PATCH 4/6] update modules in template --- nf_core/pipeline-template/modules.json | 6 +- .../dumpsoftwareversions/environment.yml | 2 +- .../custom/dumpsoftwareversions/main.nf | 4 +- .../dumpsoftwareversions/tests/main.nf.test | 7 +- .../tests/main.nf.test.snap | 50 ++-- .../modules/nf-core/fastqc/tests/main.nf.test | 265 ++++++++++++------ .../nf-core/fastqc/tests/main.nf.test.snap | 12 +- .../modules/nf-core/multiqc/environment.yml | 2 +- .../modules/nf-core/multiqc/main.nf | 6 +- .../modules/nf-core/multiqc/meta.yml | 1 - .../nf-core/multiqc/tests/main.nf.test | 48 +++- .../nf-core/multiqc/tests/main.nf.test.snap | 21 ++ 12 files changed, 294 insertions(+), 130 deletions(-) create mode 100644 nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 223bcacc9c..8660da2d42 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,17 +7,17 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "bba7e362e4afead70653f84d8700588ea28d0f9e", + "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "65ad3e0b9a4099592e1102e92e10455dc661cf53", + "git_sha": "c9488585ce7bd35ccd2a30faa2371454c8112fb9", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "4ab13872435962dadc239979554d13709e20bf29", + "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", "installed_by": ["modules"] } } diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml index f0c63f6984..9b3272bc11 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.17 + - bioconda::multiqc=1.19 diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf index 7685b33cde..f2187611cc 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -4,8 +4,8 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.17--pyhdfd78af_0' : - 'biocontainers/multiqc:1.17--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : + 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: path versions diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test index eec1db10a2..b1e1630bb3 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test @@ -31,7 +31,12 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out.versions, + file(process.out.mqc_yml[0]).readLines()[0..10], + file(process.out.yml[0]).readLines()[0..7] + ).match() + } ) } } diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap index 4274ed57aa..5f59a936d7 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap @@ -1,27 +1,33 @@ { "Should run without failures": { "content": [ - { - "0": [ - "software_versions.yml:md5,1c851188476409cda5752ce971b20b58" - ], - "1": [ - "software_versions_mqc.yml:md5,2570f4ba271ad08357b0d3d32a9cf84d" - ], - "2": [ - "versions.yml:md5,3843ac526e762117eedf8825b40683df" - ], - "mqc_yml": [ - "software_versions_mqc.yml:md5,2570f4ba271ad08357b0d3d32a9cf84d" - ], - "versions": [ - "versions.yml:md5,3843ac526e762117eedf8825b40683df" - ], - "yml": [ - "software_versions.yml:md5,1c851188476409cda5752ce971b20b58" - ] - } + [ + "versions.yml:md5,76d454d92244589d32455833f7c1ba6d" + ], + [ + "data: \"\\n\\n \\n \\n \\n \\n \\n \\n \\n\\", + " \\n\\n\\n \\n \\n\\", + " \\ \\n\\n\\n\\n \\n \\", + " \\ \\n \\n\\n\\n\\n\\", + " \\n\\n \\n \\n\\", + " \\ \\n\\n\\n\\n\\n\\n \\n\\", + " \\ \\n \\n\\n\\n\\n\\", + " \\n\\n \\n \\n\\" + ], + [ + "CUSTOM_DUMPSOFTWAREVERSIONS:", + " python: 3.11.7", + " yaml: 5.4.1", + "TOOL1:", + " tool1: 0.11.9", + "TOOL2:", + " tool2: '1.9'", + "Workflow:" + ] ], - "timestamp": "2023-11-03T14:43:22.157011" + "timestamp": "2024-01-09T23:01:18.710682" } -} +} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test index b9e8f926eb..1f21c66469 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test @@ -3,24 +3,20 @@ nextflow_process { name "Test Process FASTQC" script "../main.nf" process "FASTQC" + tag "modules" tag "modules_nfcore" tag "fastqc" - test("Single-Read") { + test("sarscov2 single-end [fastq]") { when { - params { - outdir = "$outputDir" - } process { """ - input[0] = [ + input[0] = Channel.of([ [ id: 'test', single_end:true ], - [ - file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) - ] - ] + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] + ]) """ } } @@ -28,82 +24,189 @@ nextflow_process { then { assertAll ( { assert process.success }, + // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. // looks like this:
Mon 2 Oct 2023
test.gz
// https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 - { assert process.out.html.get(0).get(1) ==~ ".*/test_fastqc.html" }, - { assert path(process.out.html.get(0).get(1)).getText().contains("") }, - { assert snapshot(process.out.versions).match("versions") }, - { assert process.out.zip.get(0).get(1) ==~ ".*/test_fastqc.zip" } + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 paired-end [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("") }, + { assert path(process.out.html[0][1][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 interleaved [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } ) } } -// TODO -// // -// // Test with paired-end data -// // -// workflow test_fastqc_paired_end { -// input = [ -// [id: 'test', single_end: false], // meta map -// [ -// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), -// file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) -// ] -// ] - -// FASTQC ( input ) -// } - -// // -// // Test with interleaved data -// // -// workflow test_fastqc_interleaved { -// input = [ -// [id: 'test', single_end: false], // meta map -// file(params.test_data['sarscov2']['illumina']['test_interleaved_fastq_gz'], checkIfExists: true) -// ] - -// FASTQC ( input ) -// } - -// // -// // Test with bam data -// // -// workflow test_fastqc_bam { -// input = [ -// [id: 'test', single_end: false], // meta map -// file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) -// ] - -// FASTQC ( input ) -// } - -// // -// // Test with multiple samples -// // -// workflow test_fastqc_multiple { -// input = [ -// [id: 'test', single_end: false], // meta map -// [ -// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), -// file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true), -// file(params.test_data['sarscov2']['illumina']['test2_1_fastq_gz'], checkIfExists: true), -// file(params.test_data['sarscov2']['illumina']['test2_2_fastq_gz'], checkIfExists: true) -// ] -// ] - -// FASTQC ( input ) -// } - -// // -// // Test with custom prefix -// // -// workflow test_fastqc_custom_prefix { -// input = [ -// [ id:'mysample', single_end:true ], // meta map -// file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) -// ] - -// FASTQC ( input ) -// } + + test("sarscov2 paired-end [bam]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 multiple [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, + { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, + { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("") }, + { assert path(process.out.html[0][1][1]).text.contains("") }, + { assert path(process.out.html[0][1][2]).text.contains("") }, + { assert path(process.out.html[0][1][3]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 custom_prefix") { + + when { + process { + """ + input[0] = Channel.of([ + [ id:'mysample', single_end:true ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 single-end [fastq] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.of([ + [ id: 'test', single_end:true ], + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out.html.collect { file(it[1]).getName() } + + process.out.zip.collect { file(it[1]).getName() } + + process.out.versions ).match() } + ) + } + } + } diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap index 636a32cead..5d624bb82e 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap @@ -1,10 +1,20 @@ { + "sarscov2 single-end [fastq] - stub": { + "content": [ + [ + "test.html", + "test.zip", + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "timestamp": "2024-01-17T18:40:57.254299" + }, "versions": { "content": [ [ "versions.yml:md5,e1cc25ca8af856014824abd842e93978" ] ], - "timestamp": "2023-10-09T23:40:54+0000" + "timestamp": "2024-01-17T18:36:50.033627" } } \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml index bc0bdb5b68..7625b75206 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.18 + - bioconda::multiqc=1.19 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index 00cc48d275..1b9f7c431d 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.18--pyhdfd78af_0' : - 'biocontainers/multiqc:1.18--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : + 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" @@ -43,7 +43,7 @@ process MULTIQC { stub: """ - touch multiqc_data + mkdir multiqc_data touch multiqc_plots touch multiqc_report.html diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml index f1aa660eb7..45a9bc35e1 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml @@ -1,4 +1,3 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json name: multiqc description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test index c2dad217c4..d0438eda6b 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test @@ -7,12 +7,9 @@ nextflow_process { tag "modules_nfcore" tag "multiqc" - test("MULTIQC: FASTQC") { + test("sarscov2 single-end [fastqc]") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) @@ -26,20 +23,17 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert path(process.out.report.get(0)).exists() }, - { assert path(process.out.data.get(0)).exists() }, - { assert path(process.out.versions.get(0)).getText().contains("multiqc") } + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("versions") } ) } } - test("MULTIQC: FASTQC and a config file") { + test("sarscov2 single-end [fastqc] [config]") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) @@ -53,9 +47,35 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert path(process.out.report.get(0)).exists() }, - { assert path(process.out.data.get(0)).exists() }, - { assert path(process.out.versions.get(0)).getText().contains("multiqc") } + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 single-end [fastqc] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[1] = [] + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.report.collect { file(it).getName() } + + process.out.data.collect { file(it).getName() } + + process.out.plots.collect { file(it).getName() } + + process.out.versions ).match() } ) } diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap new file mode 100644 index 0000000000..d37e73040d --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -0,0 +1,21 @@ +{ + "versions": { + "content": [ + [ + "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + ] + ], + "timestamp": "2024-01-09T23:02:49.911994" + }, + "sarscov2 single-end [fastqc] - stub": { + "content": [ + [ + "multiqc_report.html", + "multiqc_data", + "multiqc_plots", + "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + ] + ], + "timestamp": "2024-01-09T23:03:14.524346" + } +} \ No newline at end of file From af40445ceb8ac56817ddc811af8dec70757e9fac Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 24 Jan 2024 14:55:00 +0100 Subject: [PATCH 5/6] keep error when no subworkflows installed but allow it when linting a pipeline --- nf_core/__main__.py | 5 ++++- nf_core/components/lint/__init__.py | 4 +++- nf_core/lint/__init__.py | 20 +++++++++++++------- nf_core/lint_utils.py | 13 ++++++++++--- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ec22aac93b..7970c34081 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -613,7 +613,10 @@ def lint( json, ctx.obj["hide_progress"], ) - if len(lint_obj.failed) + len(module_lint_obj.failed) + len(subworkflow_lint_obj.failed) > 0: + swf_failed = 0 + if subworkflow_lint_obj is not None: + swf_failed = len(subworkflow_lint_obj.failed) + if len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed > 0: sys.exit(1) except AssertionError as e: log.critical(e) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index b1f40dc666..3c2fb9dde3 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -97,7 +97,9 @@ def __init__( ) ) if not self.all_remote_components: - log.warning(f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline.") + raise LookupError( + f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline." + ) local_component_dir = Path(self.dir, self.component_type, "local") self.all_local_components = [] if local_component_dir.exists(): diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 6713b97a04..b4d0c10ff3 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -91,7 +91,10 @@ def run_linting( # Create the modules lint object module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) # Create the subworkflows lint object - subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) + try: + subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) + except LookupError: + subworkflow_lint_obj = None # Verify that the pipeline is correctly configured and has a modules.json file module_lint_obj.has_valid_directory() @@ -114,7 +117,8 @@ def run_linting( module_lint_tests = ("module_changes", "module_version") subworkflow_lint_tests = ("subworkflow_changes", "subworkflow_version") module_lint_obj.filter_tests_by_key(module_lint_tests) - subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) # Set up files for modules linting test module_lint_obj.set_up_pipeline_files() @@ -133,15 +137,17 @@ def run_linting( if len(module_lint_obj.all_remote_components) > 0: module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) # Run the subworkflows lint tests - if len(subworkflow_lint_obj.all_local_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) - if len(subworkflow_lint_obj.all_remote_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) + if subworkflow_lint_obj is not None: + if len(subworkflow_lint_obj.all_local_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) + if len(subworkflow_lint_obj.all_remote_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) # Print the results lint_obj._print_results(show_passed) module_lint_obj._print_results(show_passed, sort_by=sort_by) - subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) nf_core.lint_utils.print_fixes(lint_obj) diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index f9c1e54e38..6eca6522d4 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -18,11 +18,18 @@ def print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj): """Print a joint summary of the general pipe lint tests and the module and subworkflow lint tests""" - nbr_passed = len(lint_obj.passed) + len(module_lint_obj.passed) + len(subworkflow_lint_obj.passed) + swf_passed = 0 + swf_warned = 0 + swf_failed = 0 + if subworkflow_lint_obj is not None: + swf_passed = len(subworkflow_lint_obj.passed) + swf_warned = len(subworkflow_lint_obj.warned) + swf_failed = len(subworkflow_lint_obj.failed) + nbr_passed = len(lint_obj.passed) + len(module_lint_obj.passed) + swf_passed nbr_ignored = len(lint_obj.ignored) nbr_fixed = len(lint_obj.fixed) - nbr_warned = len(lint_obj.warned) + len(module_lint_obj.warned) + len(subworkflow_lint_obj.warned) - nbr_failed = len(lint_obj.failed) + len(module_lint_obj.failed) + len(subworkflow_lint_obj.failed) + nbr_warned = len(lint_obj.warned) + len(module_lint_obj.warned) + swf_warned + nbr_failed = len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed summary_colour = "red" if nbr_failed > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) From c9a48398b3352b5f974aa6cfbcc1f2abb6e1fe51 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 24 Jan 2024 15:20:14 +0100 Subject: [PATCH 6/6] fix pipeline linting test --- tests/test_cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index abb4fba87d..1261e3a9e9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -263,9 +263,10 @@ def test_create(self, mock_create): @mock.patch("nf_core.lint.run_linting") def test_lint(self, mock_lint, mock_is_pipeline): """Test nf-core lint""" - mock_lint_results = (mock.MagicMock, mock.MagicMock) + mock_lint_results = (mock.MagicMock, mock.MagicMock, mock.MagicMock) mock_lint_results[0].failed = [] mock_lint_results[1].failed = [] + mock_lint_results[2].failed = [] mock_lint.return_value = mock_lint_results temp_pipeline_dir = tempfile.NamedTemporaryFile()
Process Name \\", + " \\ Software Version
CUSTOM_DUMPSOFTWAREVERSIONSpython3.11.7
yaml5.4.1
TOOL1tool10.11.9
TOOL2tool21.9
WorkflowNextflow
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls