From ddcab5d6e766fc9402709a11bbe4f628870e0151 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Wed, 14 May 2025 16:06:41 -0400 Subject: [PATCH] Update to Nextflow 25.04 --- .github/workflows/ci.yml | 2 +- README.md | 2 +- bin/multiqc_mappings_config.py | 13 -- conf/base.config | 6 - main.nf | 36 ++++++ modules/local/aspera_cli/main.nf | 12 +- modules/local/aspera_cli/nextflow.config | 12 -- modules/local/multiqc_mappings_config/main.nf | 27 ---- .../multiqc_mappings_config/nextflow.config | 9 -- .../tests/main.nf.test | 24 ---- .../tests/main.nf.test.snap | 25 ---- modules/local/sra_fastq_ftp/main.nf | 12 +- modules/local/sra_fastq_ftp/nextflow.config | 12 -- modules/local/sra_ids_to_runinfo/main.nf | 7 +- .../local/sra_ids_to_runinfo/nextflow.config | 8 -- modules/local/sra_runinfo_to_ftp/main.nf | 7 +- .../local/sra_runinfo_to_ftp/nextflow.config | 9 -- modules/local/sra_to_samplesheet/main.nf | 75 ----------- .../local/sra_to_samplesheet/nextflow.config | 8 -- .../sra_to_samplesheet/tests/main.nf.test | 27 ---- .../tests/main.nf.test.snap | 117 ------------------ .../custom/sratoolsncbisettings/main.nf | 2 +- .../templates/detect_ncbi_settings.sh | 5 - modules/nf-core/sratools/fasterqdump/main.nf | 9 +- .../sratools/fasterqdump/nextflow.config | 5 - modules/nf-core/sratools/prefetch/main.nf | 2 +- .../nf-core/sratools/prefetch/nextflow.config | 8 -- .../prefetch/templates/retry_with_backoff.sh | 6 - modules/nf-core/untar/environment.yml | 11 -- modules/nf-core/untar/main.nf | 63 ---------- modules/nf-core/untar/meta.yml | 46 ------- modules/nf-core/untar/tests/main.nf.test | 45 ------- modules/nf-core/untar/tests/main.nf.test.snap | 42 ------- nextflow.config | 6 +- .../main.nf | 6 - .../nextflow.config | 1 - .../nf-core/utils_nfcore_pipeline/main.nf | 31 +++-- workflows/sra/main.nf | 93 ++++---------- workflows/sra/nextflow.config | 5 - 39 files changed, 88 insertions(+), 748 deletions(-) delete mode 100755 bin/multiqc_mappings_config.py delete mode 100644 modules/local/multiqc_mappings_config/main.nf delete mode 100644 modules/local/multiqc_mappings_config/nextflow.config delete mode 100644 modules/local/multiqc_mappings_config/tests/main.nf.test delete mode 100644 modules/local/multiqc_mappings_config/tests/main.nf.test.snap delete mode 100644 modules/local/sra_ids_to_runinfo/nextflow.config delete mode 100644 modules/local/sra_runinfo_to_ftp/nextflow.config delete mode 100644 modules/local/sra_to_samplesheet/main.nf delete mode 100644 modules/local/sra_to_samplesheet/nextflow.config delete mode 100644 modules/local/sra_to_samplesheet/tests/main.nf.test delete mode 100644 modules/local/sra_to_samplesheet/tests/main.nf.test.snap delete mode 100644 modules/nf-core/sratools/prefetch/nextflow.config delete mode 100644 modules/nf-core/untar/environment.yml delete mode 100644 modules/nf-core/untar/main.nf delete mode 100644 modules/nf-core/untar/meta.yml delete mode 100644 modules/nf-core/untar/tests/main.nf.test delete mode 100644 modules/nf-core/untar/tests/main.nf.test.snap diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efd086d6..bf4d2808 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: fail-fast: false matrix: NXF_VER: - - "24.04.2" + - "25.04.0" - "latest-everything" profile: - "conda" diff --git a/README.md b/README.md index 4410c6e2..5aff003b 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![GitHub Actions CI Status](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml) [![GitHub Actions Linting Status](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/fetchngs/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5070524-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.5070524)[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A525.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/bin/multiqc_mappings_config.py b/bin/multiqc_mappings_config.py deleted file mode 100755 index 3ffe35ec..00000000 --- a/bin/multiqc_mappings_config.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -import sys - -with open(sys.argv[1], "r") as fin, open(sys.argv[2], "w") as fout: - header = fin.readline().split(",") - config = "sample_names_rename_buttons:\n" - config += "\n".join([" - " + x.strip('"') for x in header]) - config += "sample_names_rename:\n" - rename = [] - for line in fin: - rename.append(f" - [{', '.join(line.strip().split(','))}]") - fout.write(config + "\n".join(sorted(rename)) + "\n") diff --git a/conf/base.config b/conf/base.config index b7341737..14d13c68 100644 --- a/conf/base.config +++ b/conf/base.config @@ -13,12 +13,6 @@ process { memory = { 6.GB * task.attempt } time = { 4.h * task.attempt } - publishDir = [ - path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/main.nf b/main.nf index 6280a384..468be723 100644 --- a/main.nf +++ b/main.nf @@ -9,6 +9,8 @@ ---------------------------------------------------------------------------------------- */ +nextflow.preview.output = true + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS @@ -18,6 +20,7 @@ include { SRA } from './workflows/sra' include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_fetchngs_pipeline' include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_fetchngs_pipeline' +include { softwareVersionsToYAML } from './subworkflows/nf-core/utils_nfcore_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -40,6 +43,9 @@ workflow NFCORE_FETCHNGS { // SRA ( ids ) + emit: + samples = SRA.out.samples + metadata = SRA.out.metadata } /* @@ -81,6 +87,36 @@ workflow { params.monochrome_logs, params.hook_url ) + + publish: + samples = NFCORE_FETCHNGS.out.samples + metadata = NFCORE_FETCHNGS.out.metadata + versions = softwareVersionsToYAML() +} + +output { + samples { + path { sample -> + sample.fastq_1 >> 'fastq/' + sample.fastq_2 >> 'fastq/' + sample.md5_1 >> 'fastq/md5/' + sample.md5_2 >> 'fastq/md5/' + } + index { + path 'samplesheet/samplesheet.json' + } + } + + metadata { + path 'metadata' + } + + versions { + path '.' + index { + path 'nf_core_fetchngs_software_mqc_versions.yml' + } + } } /* diff --git a/modules/local/aspera_cli/main.nf b/modules/local/aspera_cli/main.nf index b38d17c0..104b55af 100644 --- a/modules/local/aspera_cli/main.nf +++ b/modules/local/aspera_cli/main.nf @@ -14,7 +14,7 @@ process ASPERA_CLI { output: tuple val(meta), path("*fastq.gz"), emit: fastq tuple val(meta), path("*md5") , emit: md5 - path "versions.yml" , emit: versions + tuple val("${task.process}"), val('aspera_cli'), eval('ascli --version'), topic: versions script: def args = task.ext.args ?: '' @@ -31,11 +31,6 @@ process ASPERA_CLI { echo "${meta.md5_1} ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5 md5sum -c ${meta.id}.fastq.gz.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - aspera_cli: \$(ascli --version) - END_VERSIONS """ } else { """ @@ -58,11 +53,6 @@ process ASPERA_CLI { echo "${meta.md5_2} ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5 md5sum -c ${meta.id}_2.fastq.gz.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - aspera_cli: \$(ascli --version) - END_VERSIONS """ } } diff --git a/modules/local/aspera_cli/nextflow.config b/modules/local/aspera_cli/nextflow.config index fa2dbd90..9a808242 100644 --- a/modules/local/aspera_cli/nextflow.config +++ b/modules/local/aspera_cli/nextflow.config @@ -1,17 +1,5 @@ process { withName: 'ASPERA_CLI' { ext.args = '-QT -l 300m -P33001' - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] } } diff --git a/modules/local/multiqc_mappings_config/main.nf b/modules/local/multiqc_mappings_config/main.nf deleted file mode 100644 index ac379065..00000000 --- a/modules/local/multiqc_mappings_config/main.nf +++ /dev/null @@ -1,27 +0,0 @@ - -process MULTIQC_MAPPINGS_CONFIG { - - conda "conda-forge::python=3.9.5" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/python:3.9--1' : - 'biocontainers/python:3.9--1' }" - - input: - path csv - - output: - path "*config.yml" , emit: yml - path "versions.yml", emit: versions - - script: - """ - multiqc_mappings_config.py \\ - $csv \\ - multiqc_config.yml - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS - """ -} diff --git a/modules/local/multiqc_mappings_config/nextflow.config b/modules/local/multiqc_mappings_config/nextflow.config deleted file mode 100644 index 11c58341..00000000 --- a/modules/local/multiqc_mappings_config/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: 'MULTIQC_MAPPINGS_CONFIG' { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} diff --git a/modules/local/multiqc_mappings_config/tests/main.nf.test b/modules/local/multiqc_mappings_config/tests/main.nf.test deleted file mode 100644 index dbb4d74f..00000000 --- a/modules/local/multiqc_mappings_config/tests/main.nf.test +++ /dev/null @@ -1,24 +0,0 @@ -nextflow_process { - - name "Test process: MULTIQC_MAPPINGS_CONFIG" - script "../main.nf" - process "MULTIQC_MAPPINGS_CONFIG" - - test("Should run without failures") { - - when { - process { - """ - input[0] = file(params.pipelines_testdata_base_path + 'csv/SRX9626017_SRR13191702.mappings.csv', checkIfExists: true) - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } - ) - } - } -} diff --git a/modules/local/multiqc_mappings_config/tests/main.nf.test.snap b/modules/local/multiqc_mappings_config/tests/main.nf.test.snap deleted file mode 100644 index 9862693f..00000000 --- a/modules/local/multiqc_mappings_config/tests/main.nf.test.snap +++ /dev/null @@ -1,25 +0,0 @@ -{ - "Should run without failures": { - "content": [ - { - "0": [ - "multiqc_config.yml:md5,7f3cb10fff83ba9eb3e8fa6862d1290a" - ], - "1": [ - "versions.yml:md5,dd4c66f0551d15510b36bb2e2b2fdd73" - ], - "versions": [ - "versions.yml:md5,dd4c66f0551d15510b36bb2e2b2fdd73" - ], - "yml": [ - "multiqc_config.yml:md5,7f3cb10fff83ba9eb3e8fa6862d1290a" - ] - } - ], - "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" - }, - "timestamp": "2024-09-17T09:16:46.557614" - } -} \ No newline at end of file diff --git a/modules/local/sra_fastq_ftp/main.nf b/modules/local/sra_fastq_ftp/main.nf index 017a08c9..ae47c3a6 100644 --- a/modules/local/sra_fastq_ftp/main.nf +++ b/modules/local/sra_fastq_ftp/main.nf @@ -15,7 +15,7 @@ process SRA_FASTQ_FTP { output: tuple val(meta), path("*fastq.gz"), emit: fastq tuple val(meta), path("*md5") , emit: md5 - path "versions.yml" , emit: versions + tuple val("${task.process}"), val('wget'), eval("echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')"), topic: versions script: def args = task.ext.args ?: '' @@ -28,11 +28,6 @@ process SRA_FASTQ_FTP { echo "${meta.md5_1} ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5 md5sum -c ${meta.id}.fastq.gz.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')) - END_VERSIONS """ } else { """ @@ -51,11 +46,6 @@ process SRA_FASTQ_FTP { echo "${meta.md5_2} ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5 md5sum -c ${meta.id}_2.fastq.gz.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')) - END_VERSIONS """ } } diff --git a/modules/local/sra_fastq_ftp/nextflow.config b/modules/local/sra_fastq_ftp/nextflow.config index 56e43959..26261f26 100644 --- a/modules/local/sra_fastq_ftp/nextflow.config +++ b/modules/local/sra_fastq_ftp/nextflow.config @@ -1,17 +1,5 @@ process { withName: 'SRA_FASTQ_FTP' { ext.args = '-t 5 -nv -c -T 60' - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] } } diff --git a/modules/local/sra_ids_to_runinfo/main.nf b/modules/local/sra_ids_to_runinfo/main.nf index 7d47f5e3..4e7f9e87 100644 --- a/modules/local/sra_ids_to_runinfo/main.nf +++ b/modules/local/sra_ids_to_runinfo/main.nf @@ -14,7 +14,7 @@ process SRA_IDS_TO_RUNINFO { output: path "*.tsv" , emit: tsv - path "versions.yml", emit: versions + tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions script: def metadata_fields = fields ? "--ena_metadata_fields ${fields}" : '' @@ -24,10 +24,5 @@ process SRA_IDS_TO_RUNINFO { id.txt \\ ${id}.runinfo.tsv \\ $metadata_fields - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS """ } diff --git a/modules/local/sra_ids_to_runinfo/nextflow.config b/modules/local/sra_ids_to_runinfo/nextflow.config deleted file mode 100644 index 9b9d0b16..00000000 --- a/modules/local/sra_ids_to_runinfo/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: 'SRA_IDS_TO_RUNINFO' { - publishDir = [ - path: { "${params.outdir}/metadata" }, - enabled: false - ] - } -} diff --git a/modules/local/sra_runinfo_to_ftp/main.nf b/modules/local/sra_runinfo_to_ftp/main.nf index 9c83cf53..40f6a03c 100644 --- a/modules/local/sra_runinfo_to_ftp/main.nf +++ b/modules/local/sra_runinfo_to_ftp/main.nf @@ -11,17 +11,12 @@ process SRA_RUNINFO_TO_FTP { output: path "*.tsv" , emit: tsv - path "versions.yml", emit: versions + tuple val("${task.process}"), val('python'), eval("python --version | sed 's/Python //g'"), topic: versions script: """ sra_runinfo_to_ftp.py \\ ${runinfo.join(',')} \\ ${runinfo.toString().tokenize(".")[0]}.runinfo_ftp.tsv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS """ } diff --git a/modules/local/sra_runinfo_to_ftp/nextflow.config b/modules/local/sra_runinfo_to_ftp/nextflow.config deleted file mode 100644 index 43263648..00000000 --- a/modules/local/sra_runinfo_to_ftp/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: 'SRA_RUNINFO_TO_FTP' { - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} diff --git a/modules/local/sra_to_samplesheet/main.nf b/modules/local/sra_to_samplesheet/main.nf deleted file mode 100644 index 92edf5df..00000000 --- a/modules/local/sra_to_samplesheet/main.nf +++ /dev/null @@ -1,75 +0,0 @@ - -process SRA_TO_SAMPLESHEET { - tag "$meta.id" - - executor 'local' - memory 100.MB - - input: - val meta - val pipeline - val strandedness - val mapping_fields - - output: - tuple val(meta), path("*samplesheet.csv"), emit: samplesheet - tuple val(meta), path("*mappings.csv") , emit: mappings - - exec: - // - // Create samplesheet containing metadata - // - - // Remove custom keys needed to download the data - def meta_clone = meta.clone() - meta_clone.remove("id") - meta_clone.remove("fastq_1") - meta_clone.remove("fastq_2") - meta_clone.remove("md5_1") - meta_clone.remove("md5_2") - meta_clone.remove("single_end") - - // Add relevant fields to the beginning of the map - pipeline_map = [ - sample : "${meta.id.split('_')[0..-2].join('_')}", - fastq_1 : meta.fastq_1, - fastq_2 : meta.fastq_2 - ] - - // Add nf-core pipeline specific entries - if (pipeline) { - if (pipeline == 'rnaseq') { - pipeline_map << [ strandedness: strandedness ] - } else if (pipeline == 'atacseq') { - pipeline_map << [ replicate: 1 ] - } else if (pipeline == 'taxprofiler') { - pipeline_map << [ fasta: '' ] - } - } - pipeline_map << meta_clone - - // Create a samplesheet - samplesheet = pipeline_map.keySet().collect{ '"' + it + '"'}.join(",") + '\n' - samplesheet += pipeline_map.values().collect{ '"' + it + '"'}.join(",") - - // Write samplesheet to file - def samplesheet_file = task.workDir.resolve("${meta.id}.samplesheet.csv") - samplesheet_file.text = samplesheet - - // - // Create sample id mappings file - // - mappings_map = pipeline_map.clone() - def fields = mapping_fields ? ['sample'] + mapping_fields.split(',').collect{ it.trim().toLowerCase() } : [] - if ((mappings_map.keySet() + fields).unique().size() != mappings_map.keySet().size()) { - error("Invalid option for '--sample_mapping_fields': ${mapping_fields}.\nValid options: ${mappings_map.keySet().join(', ')}") - } - - // Create mappings - mappings = fields.collect{ '"' + it + '"'}.join(",") + '\n' - mappings += mappings_map.subMap(fields).values().collect{ '"' + it + '"'}.join(",") - - // Write mappings to file - def mappings_file = task.workDir.resolve("${meta.id}.mappings.csv") - mappings_file.text = mappings -} diff --git a/modules/local/sra_to_samplesheet/nextflow.config b/modules/local/sra_to_samplesheet/nextflow.config deleted file mode 100644 index da241c1a..00000000 --- a/modules/local/sra_to_samplesheet/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: SRA_TO_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - enabled: false - ] - } -} diff --git a/modules/local/sra_to_samplesheet/tests/main.nf.test b/modules/local/sra_to_samplesheet/tests/main.nf.test deleted file mode 100644 index ed765158..00000000 --- a/modules/local/sra_to_samplesheet/tests/main.nf.test +++ /dev/null @@ -1,27 +0,0 @@ -nextflow_process { - - name "Test process: SRA_TO_SAMPLESHEET" - script "../main.nf" - process "SRA_TO_SAMPLESHEET" - - test("Should run without failures") { - - when { - process { - """ - input[0] = [id:'ERX1188904_ERR1109373', run_accession:'ERR1109373', experiment_accession:'ERX1188904', sample_accession:'SAMEA3643867', experiment_alias:'ena-EXPERIMENT-CAM-03-11-2015-17:01:52:847-7', run_alias:'ena-RUN-CAM-03-11-2015-17:01:52:847-7', sample_alias:'sample_56', study_alias:'ena-STUDY-CAM-02-11-2015-17:42:24:189-13', library_layout:'PAIRED', experiment_title:'Illumina HiSeq 2500 paired end sequencing', sample_title:'RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome', sample_description:'RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome', fastq_md5:'8d7d7b854d0207d1226477a30103fade;9fd57225d6c07a31843276d6df9b15c0;5a62e8f785687dce890cfb4fe3e607f9', fastq_ftp:'ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_2.fastq.gz', fastq_1:'./results/fastq/ERX1188904_ERR1109373_1.fastq.gz', fastq_2:'./results/fastq/ERX1188904_ERR1109373_2.fastq.gz', md5_1:'9fd57225d6c07a31843276d6df9b15c0', md5_2:'5a62e8f785687dce890cfb4fe3e607f9', single_end:false] - input[1] = 'rnaseq' - input[2] = 'auto' - input[3] = 'experiment_accession,run_accession,sample_accession,experiment_alias,run_alias,sample_alias,experiment_title,sample_title,sample_description' - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } - ) - } - } -} diff --git a/modules/local/sra_to_samplesheet/tests/main.nf.test.snap b/modules/local/sra_to_samplesheet/tests/main.nf.test.snap deleted file mode 100644 index 568f3ea7..00000000 --- a/modules/local/sra_to_samplesheet/tests/main.nf.test.snap +++ /dev/null @@ -1,117 +0,0 @@ -{ - "Should run without failures": { - "content": [ - { - "0": [ - [ - { - "id": "ERX1188904_ERR1109373", - "run_accession": "ERR1109373", - "experiment_accession": "ERX1188904", - "sample_accession": "SAMEA3643867", - "experiment_alias": "ena-EXPERIMENT-CAM-03-11-2015-17:01:52:847-7", - "run_alias": "ena-RUN-CAM-03-11-2015-17:01:52:847-7", - "sample_alias": "sample_56", - "study_alias": "ena-STUDY-CAM-02-11-2015-17:42:24:189-13", - "library_layout": "PAIRED", - "experiment_title": "Illumina HiSeq 2500 paired end sequencing", - "sample_title": "RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome", - "sample_description": "RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome", - "fastq_md5": "8d7d7b854d0207d1226477a30103fade;9fd57225d6c07a31843276d6df9b15c0;5a62e8f785687dce890cfb4fe3e607f9", - "fastq_ftp": "ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_2.fastq.gz", - "fastq_1": "./results/fastq/ERX1188904_ERR1109373_1.fastq.gz", - "fastq_2": "./results/fastq/ERX1188904_ERR1109373_2.fastq.gz", - "md5_1": "9fd57225d6c07a31843276d6df9b15c0", - "md5_2": "5a62e8f785687dce890cfb4fe3e607f9", - "single_end": false - }, - "ERX1188904_ERR1109373.samplesheet.csv:md5,e7898191d57258e049ee7129d36f5c08" - ] - ], - "1": [ - [ - { - "id": "ERX1188904_ERR1109373", - "run_accession": "ERR1109373", - "experiment_accession": "ERX1188904", - "sample_accession": "SAMEA3643867", - "experiment_alias": "ena-EXPERIMENT-CAM-03-11-2015-17:01:52:847-7", - "run_alias": "ena-RUN-CAM-03-11-2015-17:01:52:847-7", - "sample_alias": "sample_56", - "study_alias": "ena-STUDY-CAM-02-11-2015-17:42:24:189-13", - "library_layout": "PAIRED", - "experiment_title": "Illumina HiSeq 2500 paired end sequencing", - "sample_title": "RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome", - "sample_description": "RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome", - "fastq_md5": "8d7d7b854d0207d1226477a30103fade;9fd57225d6c07a31843276d6df9b15c0;5a62e8f785687dce890cfb4fe3e607f9", - "fastq_ftp": "ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_2.fastq.gz", - "fastq_1": "./results/fastq/ERX1188904_ERR1109373_1.fastq.gz", - "fastq_2": "./results/fastq/ERX1188904_ERR1109373_2.fastq.gz", - "md5_1": "9fd57225d6c07a31843276d6df9b15c0", - "md5_2": "5a62e8f785687dce890cfb4fe3e607f9", - "single_end": false - }, - "ERX1188904_ERR1109373.mappings.csv:md5,d09ddb4f0709675e5dfe1eadf12c608f" - ] - ], - "mappings": [ - [ - { - "id": "ERX1188904_ERR1109373", - "run_accession": "ERR1109373", - "experiment_accession": "ERX1188904", - "sample_accession": "SAMEA3643867", - "experiment_alias": "ena-EXPERIMENT-CAM-03-11-2015-17:01:52:847-7", - "run_alias": "ena-RUN-CAM-03-11-2015-17:01:52:847-7", - "sample_alias": "sample_56", - "study_alias": "ena-STUDY-CAM-02-11-2015-17:42:24:189-13", - "library_layout": "PAIRED", - "experiment_title": "Illumina HiSeq 2500 paired end sequencing", - "sample_title": "RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome", - "sample_description": "RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome", - "fastq_md5": "8d7d7b854d0207d1226477a30103fade;9fd57225d6c07a31843276d6df9b15c0;5a62e8f785687dce890cfb4fe3e607f9", - "fastq_ftp": "ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_2.fastq.gz", - "fastq_1": "./results/fastq/ERX1188904_ERR1109373_1.fastq.gz", - "fastq_2": "./results/fastq/ERX1188904_ERR1109373_2.fastq.gz", - "md5_1": "9fd57225d6c07a31843276d6df9b15c0", - "md5_2": "5a62e8f785687dce890cfb4fe3e607f9", - "single_end": false - }, - "ERX1188904_ERR1109373.mappings.csv:md5,d09ddb4f0709675e5dfe1eadf12c608f" - ] - ], - "samplesheet": [ - [ - { - "id": "ERX1188904_ERR1109373", - "run_accession": "ERR1109373", - "experiment_accession": "ERX1188904", - "sample_accession": "SAMEA3643867", - "experiment_alias": "ena-EXPERIMENT-CAM-03-11-2015-17:01:52:847-7", - "run_alias": "ena-RUN-CAM-03-11-2015-17:01:52:847-7", - "sample_alias": "sample_56", - "study_alias": "ena-STUDY-CAM-02-11-2015-17:42:24:189-13", - "library_layout": "PAIRED", - "experiment_title": "Illumina HiSeq 2500 paired end sequencing", - "sample_title": "RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome", - "sample_description": "RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome", - "fastq_md5": "8d7d7b854d0207d1226477a30103fade;9fd57225d6c07a31843276d6df9b15c0;5a62e8f785687dce890cfb4fe3e607f9", - "fastq_ftp": "ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_2.fastq.gz", - "fastq_1": "./results/fastq/ERX1188904_ERR1109373_1.fastq.gz", - "fastq_2": "./results/fastq/ERX1188904_ERR1109373_2.fastq.gz", - "md5_1": "9fd57225d6c07a31843276d6df9b15c0", - "md5_2": "5a62e8f785687dce890cfb4fe3e607f9", - "single_end": false - }, - "ERX1188904_ERR1109373.samplesheet.csv:md5,e7898191d57258e049ee7129d36f5c08" - ] - ] - } - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T11:51:38.244046" - } -} \ No newline at end of file diff --git a/modules/nf-core/custom/sratoolsncbisettings/main.nf b/modules/nf-core/custom/sratoolsncbisettings/main.nf index 577117ed..7dcb66e9 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/main.nf +++ b/modules/nf-core/custom/sratoolsncbisettings/main.nf @@ -12,7 +12,7 @@ process CUSTOM_SRATOOLSNCBISETTINGS { output: path('*.mkfg') , emit: ncbi_settings - path 'versions.yml', emit: versions + tuple val("${task.process}"), val('sratools'), eval("vdb-config --version 2>&1 | grep -Eo '[0-9.]+'"), topic: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh b/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh index cfe3a324..b553659b 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh +++ b/modules/nf-core/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh @@ -38,8 +38,3 @@ else fi cp "${NCBI_SETTINGS}" ./ fi - -cat <<-END_VERSIONS > versions.yml -"!{task.process}": - sratools: $(vdb-config --version 2>&1 | grep -Eo '[0-9.]+') -END_VERSIONS diff --git a/modules/nf-core/sratools/fasterqdump/main.nf b/modules/nf-core/sratools/fasterqdump/main.nf index e7cf157a..4fdd07fe 100644 --- a/modules/nf-core/sratools/fasterqdump/main.nf +++ b/modules/nf-core/sratools/fasterqdump/main.nf @@ -14,7 +14,8 @@ process SRATOOLS_FASTERQDUMP { output: tuple val(meta), path('*.fastq.gz'), emit: reads - path "versions.yml" , emit: versions + tuple val("${task.process}"), val('sratools'), eval("fasterq-dump --version 2>&1 | grep -Eo '[0-9.]+'"), topic: versions + tuple val("${task.process}"), val('pigz'), eval("pigz --version 2>&1 | sed 's/pigz //g'"), topic: versions when: task.ext.when == null || task.ext.when @@ -45,11 +46,5 @@ process SRATOOLS_FASTERQDUMP { --no-name \\ --processes $task.cpus \\ *.fastq - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - sratools: \$(fasterq-dump --version 2>&1 | grep -Eo '[0-9.]+') - pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' ) - END_VERSIONS """ } diff --git a/modules/nf-core/sratools/fasterqdump/nextflow.config b/modules/nf-core/sratools/fasterqdump/nextflow.config index f98b140d..7e1649d1 100644 --- a/modules/nf-core/sratools/fasterqdump/nextflow.config +++ b/modules/nf-core/sratools/fasterqdump/nextflow.config @@ -1,10 +1,5 @@ process { withName: SRATOOLS_FASTERQDUMP { ext.args = '--split-files --include-technical' - publishDir = [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ] } } \ No newline at end of file diff --git a/modules/nf-core/sratools/prefetch/main.nf b/modules/nf-core/sratools/prefetch/main.nf index 74838d53..ac9d1e80 100644 --- a/modules/nf-core/sratools/prefetch/main.nf +++ b/modules/nf-core/sratools/prefetch/main.nf @@ -14,7 +14,7 @@ process SRATOOLS_PREFETCH { output: tuple val(meta), path(id, type: 'dir'), emit: sra - path 'versions.yml' , emit: versions + tuple val("${task.process}"), val('sratools'), eval("prefetch --version 2>&1 | grep -Eo '[0-9.]+'"), topic: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/sratools/prefetch/nextflow.config b/modules/nf-core/sratools/prefetch/nextflow.config deleted file mode 100644 index a2ca8848..00000000 --- a/modules/nf-core/sratools/prefetch/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: SRATOOLS_PREFETCH { - publishDir = [ - path: { "${params.outdir}/sra" }, - enabled: false - ] - } -} \ No newline at end of file diff --git a/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh b/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh index bfee6070..395593d6 100755 --- a/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh +++ b/modules/nf-core/sratools/prefetch/templates/retry_with_backoff.sh @@ -57,9 +57,3 @@ if grep -q "checksums missing" vdb-validate_result.txt; then exit 1 fi fi - -cat <<-END_VERSIONS > versions.yml -"!{task.process}": - sratools: $(prefetch --version 2>&1 | grep -Eo '[0-9.]+') - curl: $(curl --version | head -n 1 | sed 's/^curl //; s/ .*$//') -END_VERSIONS diff --git a/modules/nf-core/untar/environment.yml b/modules/nf-core/untar/environment.yml deleted file mode 100644 index 0c9cbb10..00000000 --- a/modules/nf-core/untar/environment.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: untar - -channels: - - conda-forge - - bioconda - - defaults - -dependencies: - - conda-forge::grep=3.11 - - conda-forge::sed=4.7 - - conda-forge::tar=1.34 diff --git a/modules/nf-core/untar/main.nf b/modules/nf-core/untar/main.nf deleted file mode 100644 index 8a75bb95..00000000 --- a/modules/nf-core/untar/main.nf +++ /dev/null @@ -1,63 +0,0 @@ -process UNTAR { - tag "$archive" - label 'process_single' - - conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'nf-core/ubuntu:20.04' }" - - input: - tuple val(meta), path(archive) - - output: - tuple val(meta), path("$prefix"), emit: untar - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' - prefix = task.ext.prefix ?: ( meta.id ? "${meta.id}" : archive.baseName.toString().replaceFirst(/\.tar$/, "")) - - """ - mkdir $prefix - - ## Ensures --strip-components only applied when top level of tar contents is a directory - ## If just files or multiple directories, place all in prefix - if [[ \$(tar -taf ${archive} | grep -o -P "^.*?\\/" | uniq | wc -l) -eq 1 ]]; then - tar \\ - -C $prefix --strip-components 1 \\ - -xavf \\ - $args \\ - $archive \\ - $args2 - else - tar \\ - -C $prefix \\ - -xavf \\ - $args \\ - $archive \\ - $args2 - fi - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - untar: \$(echo \$(tar --version 2>&1) | sed 's/^.*(GNU tar) //; s/ Copyright.*\$//') - END_VERSIONS - """ - - stub: - prefix = task.ext.prefix ?: ( meta.id ? "${meta.id}" : archive.toString().replaceFirst(/\.[^\.]+(.gz)?$/, "")) - """ - mkdir $prefix - touch ${prefix}/file.txt - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - untar: \$(echo \$(tar --version 2>&1) | sed 's/^.*(GNU tar) //; s/ Copyright.*\$//') - END_VERSIONS - """ -} diff --git a/modules/nf-core/untar/meta.yml b/modules/nf-core/untar/meta.yml deleted file mode 100644 index a9a2110f..00000000 --- a/modules/nf-core/untar/meta.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: untar -description: Extract files. -keywords: - - untar - - uncompress - - extract -tools: - - untar: - description: | - Extract tar.gz files. - documentation: https://www.gnu.org/software/tar/manual/ - licence: ["GPL-3.0-or-later"] -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - archive: - type: file - description: File to be untar - pattern: "*.{tar}.{gz}" -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - untar: - type: directory - description: Directory containing contents of archive - pattern: "*/" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@joseespinosa" - - "@drpatelh" - - "@matthdsm" - - "@jfy133" -maintainers: - - "@joseespinosa" - - "@drpatelh" - - "@matthdsm" - - "@jfy133" diff --git a/modules/nf-core/untar/tests/main.nf.test b/modules/nf-core/untar/tests/main.nf.test deleted file mode 100644 index 98b769ad..00000000 --- a/modules/nf-core/untar/tests/main.nf.test +++ /dev/null @@ -1,45 +0,0 @@ -nextflow_process { - - name "Test Process UNTAR" - script "../main.nf" - process "UNTAR" - - test("test_untar") { - - when { - process { - """ - input[0] = [ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kraken2.tar.gz', checkIfExists: true) ] - """ - } - } - - then { - assertAll ( - { assert process.success }, - { assert snapshot(process.out.untar).match("test_untar") }, - ) - } - - } - - test("test_untar_onlyfiles") { - - when { - process { - """ - input[0] = [ [], file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) ] - """ - } - } - - then { - assertAll ( - { assert process.success }, - { assert snapshot(process.out.untar).match("test_untar_onlyfiles") }, - ) - } - - } - -} diff --git a/modules/nf-core/untar/tests/main.nf.test.snap b/modules/nf-core/untar/tests/main.nf.test.snap deleted file mode 100644 index 64550292..00000000 --- a/modules/nf-core/untar/tests/main.nf.test.snap +++ /dev/null @@ -1,42 +0,0 @@ -{ - "test_untar_onlyfiles": { - "content": [ - [ - [ - [ - - ], - [ - "hello.txt:md5,e59ff97941044f85df5297e1c302d260" - ] - ] - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T11:49:41.320643" - }, - "test_untar": { - "content": [ - [ - [ - [ - - ], - [ - "hash.k2d:md5,8b8598468f54a7087c203ad0190555d9", - "opts.k2d:md5,a033d00cf6759407010b21700938f543", - "taxo.k2d:md5,094d5891cdccf2f1468088855c214b2c" - ] - ] - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T11:49:33.795172" - } -} \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index a047ab96..a09178b3 100644 --- a/nextflow.config +++ b/nextflow.config @@ -196,6 +196,10 @@ set -C # No clobber - prevent output redirection from overwriting files. // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false +// workflow outputs +outputDir = params.outdir +workflow.output.mode = params.publish_dir_mode + def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true @@ -220,7 +224,7 @@ manifest { homePage = 'https://github.com/nf-core/fetchngs' description = """Pipeline to fetch metadata and raw FastQ files from public databases""" mainScript = 'main.nf' - nextflowVersion = '!>=24.04.2' + nextflowVersion = '!>=25.04.0' version = '1.13.0dev' doi = '10.5281/zenodo.5070524' } diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf index fbeacf4a..eb5ec390 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf @@ -12,28 +12,22 @@ workflow FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS { main: - ch_versions = Channel.empty() - // // Detect existing NCBI user settings or create new ones. // CUSTOM_SRATOOLSNCBISETTINGS ( ch_sra_ids.collect() ) ch_ncbi_settings = CUSTOM_SRATOOLSNCBISETTINGS.out.ncbi_settings - ch_versions = ch_versions.mix(CUSTOM_SRATOOLSNCBISETTINGS.out.versions) // // Prefetch sequencing reads in SRA format. // SRATOOLS_PREFETCH ( ch_sra_ids, ch_ncbi_settings, ch_dbgap_key ) - ch_versions = ch_versions.mix(SRATOOLS_PREFETCH.out.versions.first()) // // Convert the SRA format into one or more compressed FASTQ files. // SRATOOLS_FASTERQDUMP ( SRATOOLS_PREFETCH.out.sra, ch_ncbi_settings, ch_dbgap_key ) - ch_versions = ch_versions.mix(SRATOOLS_FASTERQDUMP.out.versions.first()) emit: reads = SRATOOLS_FASTERQDUMP.out.reads // channel: [ val(meta), [ reads ] ] - versions = ch_versions // channel: [ versions.yml ] } diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config index de803a38..187faf6d 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config @@ -1,2 +1 @@ -includeConfig '../../../modules/nf-core/sratools/prefetch/nextflow.config' includeConfig '../../../modules/nf-core/sratools/fasterqdump/nextflow.config' diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 5cb7bafe..6a4d6ca1 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -89,31 +89,30 @@ def getWorkflowVersion() { return version_string } -// -// Get software versions for pipeline -// -def processVersionsFromYAML(yaml_file) { - def yaml = new org.yaml.snakeyaml.Yaml() - def versions = yaml.load(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } - return yaml.dumpAsMap(versions).trim() -} - // // Get workflow version for pipeline // def workflowVersionToYAML() { - return """ - Workflow: - ${workflow.manifest.name}: ${getWorkflowVersion()} - Nextflow: ${workflow.nextflow.version} - """.stripIndent().trim() + return Channel.of( + [ 'Workflow', workflow.manifest.name, getWorkflowVersion() ], + [ 'Workflow', 'Nextflow', workflow.nextflow.version ] + ) } // // Get channel of software versions used in pipeline in YAML format // -def softwareVersionsToYAML(ch_versions) { - return ch_versions.unique().map { version -> processVersionsFromYAML(version) }.unique().mix(Channel.of(workflowVersionToYAML())) +def softwareVersionsToYAML() { + return Channel.topic('versions') + .unique() + .mix(workflowVersionToYAML()) + .map { process, name, version -> + [ + (process.tokenize(':').last()): [ + (name): version + ] + ] + } } // diff --git a/workflows/sra/main.nf b/workflows/sra/main.nf index 0c8cac0c..1487b98a 100644 --- a/workflows/sra/main.nf +++ b/workflows/sra/main.nf @@ -4,13 +4,10 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { MULTIQC_MAPPINGS_CONFIG } from '../../modules/local/multiqc_mappings_config' include { SRA_FASTQ_FTP } from '../../modules/local/sra_fastq_ftp' include { SRA_IDS_TO_RUNINFO } from '../../modules/local/sra_ids_to_runinfo' include { SRA_RUNINFO_TO_FTP } from '../../modules/local/sra_runinfo_to_ftp' include { ASPERA_CLI } from '../../modules/local/aspera_cli' -include { SRA_TO_SAMPLESHEET } from '../../modules/local/sra_to_samplesheet' -include { softwareVersionsToYAML } from '../../subworkflows/nf-core/utils_nfcore_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -32,7 +29,6 @@ workflow SRA { ids // channel: [ ids ] main: - ch_versions = Channel.empty() // // MODULE: Get SRA run information for public database ids @@ -41,7 +37,6 @@ workflow SRA { ids, params.ena_metadata_fields ?: '' ) - ch_versions = ch_versions.mix(SRA_IDS_TO_RUNINFO.out.versions.first()) // // MODULE: Parse SRA run information, create file containing FTP links and read into workflow as [ meta, [reads] ] @@ -49,7 +44,6 @@ workflow SRA { SRA_RUNINFO_TO_FTP ( SRA_IDS_TO_RUNINFO.out.tsv ) - ch_versions = ch_versions.mix(SRA_RUNINFO_TO_FTP.out.versions.first()) SRA_RUNINFO_TO_FTP .out @@ -95,7 +89,6 @@ workflow SRA { SRA_FASTQ_FTP ( ch_sra_reads.ftp ) - ch_versions = ch_versions.mix(SRA_FASTQ_FTP.out.versions.first()) // // SUBWORKFLOW: Download sequencing reads without FTP links using sra-tools. @@ -104,7 +97,6 @@ workflow SRA { ch_sra_reads.sratools, params.dbgap_key ? file(params.dbgap_key, checkIfExists: true) : [] ) - ch_versions = ch_versions.mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.versions.first()) // // MODULE: If Aspera link is provided in run information then download FastQ directly via Aspera CLI and validate with md5sums @@ -113,80 +105,39 @@ workflow SRA { ch_sra_reads.aspera, 'era-fasp' ) - ch_versions = ch_versions.mix(ASPERA_CLI.out.versions.first()) // Isolate FASTQ channel which will be added to emit block - SRA_FASTQ_FTP - .out - .fastq + SRA_FASTQ_FTP.out.fastq .mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads) .mix(ASPERA_CLI.out.fastq) - .map { - meta, fastq -> - def reads = fastq instanceof List ? fastq.flatten() : [ fastq ] - def meta_clone = meta.clone() + .set { ch_fastq } - meta_clone.fastq_1 = reads[0] ? "${params.outdir}/fastq/${reads[0].getName()}" : '' - meta_clone.fastq_2 = reads[1] && !meta.single_end ? "${params.outdir}/fastq/${reads[1].getName()}" : '' + SRA_FASTQ_FTP.out.md5 + .mix(ASPERA_CLI.out.md5) + .set { ch_md5 } - return meta_clone + ch_fastq + .join(ch_md5, remainder: true) + .map { + meta, fastq, md5 -> + fastq = fastq instanceof List ? fastq.flatten() : [ fastq ] + md5 = md5 instanceof List ? md5.flatten() : [ md5 ] + meta + [ + fastq_1: fastq[0], + fastq_2: fastq[1] && !meta.single_end ? fastq[1] : null, + md5_1: md5[0], + md5_2: md5[1] && !meta.single_end ? md5[1] : null, + ] } - .set { ch_sra_metadata } + .set { ch_samples } } - - // - // MODULE: Stage FastQ files downloaded by SRA together and auto-create a samplesheet - // - SRA_TO_SAMPLESHEET ( - ch_sra_metadata, - params.nf_core_pipeline ?: '', - params.nf_core_rnaseq_strandedness ?: 'auto', - params.sample_mapping_fields - ) - - // Merge samplesheets and mapping files across all samples - SRA_TO_SAMPLESHEET - .out - .samplesheet - .map { it[1] } - .collectFile(name:'tmp_samplesheet.csv', newLine: true, keepHeader: true, sort: { it.baseName }) - .map { it.text.tokenize('\n').join('\n') } - .collectFile(name:'samplesheet.csv', storeDir: "${params.outdir}/samplesheet") - .set { ch_samplesheet } - - SRA_TO_SAMPLESHEET - .out - .mappings - .map { it[1] } - .collectFile(name:'tmp_id_mappings.csv', newLine: true, keepHeader: true, sort: { it.baseName }) - .map { it.text.tokenize('\n').join('\n') } - .collectFile(name:'id_mappings.csv', storeDir: "${params.outdir}/samplesheet") - .set { ch_mappings } - - // - // MODULE: Create a MutiQC config file with sample name mappings - // - ch_sample_mappings_yml = Channel.empty() - if (params.sample_mapping_fields) { - MULTIQC_MAPPINGS_CONFIG ( - ch_mappings - ) - ch_versions = ch_versions.mix(MULTIQC_MAPPINGS_CONFIG.out.versions) - ch_sample_mappings_yml = MULTIQC_MAPPINGS_CONFIG.out.yml + else { + ch_samples = Channel.empty() } - // - // Collate and save software versions - // - softwareVersionsToYAML(ch_versions) - .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true) - emit: - samplesheet = ch_samplesheet - mappings = ch_mappings - sample_mappings = ch_sample_mappings_yml - sra_metadata = ch_sra_metadata - versions = ch_versions.unique() + samples = ch_samples + metadata = SRA_RUNINFO_TO_FTP.out.tsv } /* diff --git a/workflows/sra/nextflow.config b/workflows/sra/nextflow.config index d242c238..522b05b8 100644 --- a/workflows/sra/nextflow.config +++ b/workflows/sra/nextflow.config @@ -1,8 +1,3 @@ -includeConfig "../../modules/local/multiqc_mappings_config/nextflow.config" includeConfig "../../modules/local/aspera_cli/nextflow.config" includeConfig "../../modules/local/sra_fastq_ftp/nextflow.config" -includeConfig "../../modules/local/sra_ids_to_runinfo/nextflow.config" -includeConfig "../../modules/local/sra_runinfo_to_ftp/nextflow.config" -includeConfig "../../modules/local/sra_to_samplesheet/nextflow.config" -includeConfig "../../modules/nf-core/sratools/prefetch/nextflow.config" includeConfig "../../subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config"