From 76c0d893c36f56658ff3c0d57119c06338e92aec Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 22 Nov 2021 14:33:58 +0100 Subject: [PATCH 01/28] feat: update test data --- .gitattributes | 2 ++ .nf-core.yml | 1 + conf/genomes.config | 2 +- conf/test.config | 18 +++++++++--------- lib/NfcoreSchema.groovy | 3 +-- tests/csv/3.0/fastq_pair.csv | 3 +++ tests/csv/3.0/fastq_single.csv | 2 ++ workflows/sarek.nf | 1 + 8 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 tests/csv/3.0/fastq_pair.csv create mode 100644 tests/csv/3.0/fastq_single.csv diff --git a/.gitattributes b/.gitattributes index 7fe55006f8..050bb12035 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ *.config linguist-language=nextflow +modules/nf-core/** linguist-generated +subworkflows/nf-core/** linguist-generated diff --git a/.nf-core.yml b/.nf-core.yml index 8427fec762..492c26486c 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1,5 +1,6 @@ lint: files_unchanged: + - assets/multiqc_config.yaml - assets/nf-core-sarek_logo.png - docs/images/nf-core-sarek_logo.png - lib/NfcoreTemplate.groovy diff --git a/conf/genomes.config b/conf/genomes.config index 834c8536b8..5abaa370dc 100644 --- a/conf/genomes.config +++ b/conf/genomes.config @@ -86,8 +86,8 @@ params { 'small_hg38' { dbsnp = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz" fasta = "${params.genomes_base}/data/genomics/homo_sapiens/genome/genome.fasta" - fasta_fai = "${params.genomes_base}/data/genomics/homo_sapiens/genome/genome.fasta.fai" germline_resource = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz" + intervals = "${params.genomes_base}/data/genomics/homo_sapiens/genome/genome.interval_list" known_indels = "${params.genomes_base}/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz" snpeff_db = 'GRCh38.99' vep_genome = 'GRCh38' diff --git a/conf/test.config b/conf/test.config index 09659607f8..df2d9fd199 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,12 +20,12 @@ params { max_time = 6.h // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-manta-normal-https.csv' + input = "${baseDir}/tests/csv/3.0/fastq_single.csv" // Small reference genome igenomes_ignore = true - genome = 'smallGRCh37' - genomes_base = "https://raw.githubusercontent.com/nf-core/test-datasets/sarek/reference" + genome = 'small_hg38' + genomes_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules' snpeff_db = 'WBcel235.99' vep_species = 'caenorhabditis_elegans' vep_cache_version = '104' @@ -42,7 +42,7 @@ profiles { params.step = 'annotate' } pair { - params.input = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-manta-https.csv' + params.input = "${baseDir}/tests/csv/3.0/fastq_pair.csv" } prepare_recalibration { params.input = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-mapped-normal-https.csv' @@ -77,22 +77,22 @@ profiles { params.trim_fastq = true } use_gatk_spark { - params.use_gatk_spark = 'markduplicates,bqsr' + params.use_gatk_spark = 'bqsr,markduplicates' } umi_quiaseq { params.genome = 'smallGRCh38' params.genomes_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/reference/chr20_hg38' params.input = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-umi-qiaseq-https.csv' - params.read_structure1 = "12M11S+T" - params.read_structure2 = "12M11S+T" + params.read_structure1 = '12M11S+T' + params.read_structure2 = '12M11S+T' params.umi = true } umi_tso { genome = 'smallGRCh38' genomes_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/reference/chr20_hg38' input = 'https://raw.githubusercontent.com/nf-core/test-datasets/sarek/testdata/csv/tiny-umi-tso-https.csv' - read_structure1 = "7M1S+T" - read_structure2 = "7M1S+T" + read_structure1 = '7M1S+T' + read_structure2 = '7M1S+T' umi = true } } diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index 8d6920dd64..dcb39c839e 100755 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -260,13 +260,12 @@ class NfcoreSchema { // Get pipeline parameters defined in JSON Schema def Map params_summary = [:] - def blacklist = ['hostnames'] def params_map = paramsLoad(getSchemaPath(workflow, schema_filename=schema_filename)) for (group in params_map.keySet()) { def sub_params = new LinkedHashMap() def group_params = params_map.get(group) // This gets the parameters of that particular group for (param in group_params.keySet()) { - if (params.containsKey(param) && !blacklist.contains(param)) { + if (params.containsKey(param)) { def params_value = params.get(param) def schema_value = group_params.get(param).default def param_type = group_params.get(param).type diff --git a/tests/csv/3.0/fastq_pair.csv b/tests/csv/3.0/fastq_pair.csv new file mode 100644 index 0000000000..ce9e72fd97 --- /dev/null +++ b/tests/csv/3.0/fastq_pair.csv @@ -0,0 +1,3 @@ +patient,gender,status,sample,lane,fastq_1,fastq_2 +test,XX,0,test,test_L1,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz +test,XX,1,test2,test2_L1,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test2_1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test2_2.fastq.gz diff --git a/tests/csv/3.0/fastq_single.csv b/tests/csv/3.0/fastq_single.csv new file mode 100644 index 0000000000..7e84c751df --- /dev/null +++ b/tests/csv/3.0/fastq_single.csv @@ -0,0 +1,2 @@ +patient,gender,status,sample,lane,fastq_1,fastq_2 +test,XX,0,test,test_L1,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_1.fastq.gz,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/illumina/fastq/test_2.fastq.gz diff --git a/workflows/sarek.nf b/workflows/sarek.nf index b963697aaa..b722a5d7a4 100644 --- a/workflows/sarek.nf +++ b/workflows/sarek.nf @@ -19,6 +19,7 @@ def checkPathParamList = [ params.cadd_wg_snvs_tbi, params.chr_dir, params.chr_length, + params.dict, params.dbsnp, params.dbsnp_tbi, params.fasta, From dace03e60f9356d4f6daa354bf14fd2a3584fc49 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 22 Nov 2021 14:36:50 +0100 Subject: [PATCH 02/28] fix: using tools dev for linting --- .github/workflows/linting.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 3b448773c4..c5c9fe386c 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -120,7 +120,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install nf-core + pip install nf-core git+https://github.com/nf-core/tools.git@dev - name: Run nf-core lint env: From badee216d32d31ae8462dc5f65a45a70cd072597 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 22 Nov 2021 14:46:49 +0100 Subject: [PATCH 03/28] fix: linting --- .github/ISSUE_TEMPLATE/config.yml | 1 - .nf-core.yml | 1 + lib/NfcoreSchema.groovy | 19 ++++++++++++++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index f390abc544..af28464c99 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,4 +1,3 @@ -blank_issues_enabled: false contact_links: - name: Join nf-core url: https://nf-co.re/join diff --git a/.nf-core.yml b/.nf-core.yml index 492c26486c..375f1bceb1 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1,5 +1,6 @@ lint: files_unchanged: + - .github/workflows/linting.yml - assets/multiqc_config.yaml - assets/nf-core-sarek_logo.png - docs/images/nf-core-sarek_logo.png diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index dcb39c839e..5f681078f2 100755 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -105,9 +105,13 @@ class NfcoreSchema { // Collect expected parameters from the schema def expectedParams = [] + def enums = [:] for (group in schemaParams) { for (p in group.value['properties']) { expectedParams.push(p.key) + if (group.value['properties'][p.key].containsKey('enum')) { + enums[p.key] = group.value['properties'][p.key]['enum'] + } } } @@ -155,7 +159,7 @@ class NfcoreSchema { println '' log.error 'ERROR: Validation of pipeline parameters failed!' JSONObject exceptionJSON = e.toJSON() - printExceptions(exceptionJSON, params_json, log) + printExceptions(exceptionJSON, params_json, log, enums) println '' has_error = true } @@ -329,7 +333,7 @@ class NfcoreSchema { // // Loop over nested exceptions and print the causingException // - private static void printExceptions(ex_json, params_json, log) { + private static void printExceptions(ex_json, params_json, log, enums, limit=5) { def causingExceptions = ex_json['causingExceptions'] if (causingExceptions.length() == 0) { def m = ex_json['message'] =~ /required key \[([^\]]+)\] not found/ @@ -345,7 +349,16 @@ class NfcoreSchema { else { def param = ex_json['pointerToViolation'] - ~/^#\// def param_val = params_json[param].toString() - log.error "* --${param}: ${ex_json['message']} (${param_val})" + if (enums.containsKey(param)) { + def error_msg = "* --${param}: '${param_val}' is not a valid choice (Available choices" + if (enums[param].size() > limit) { + log.error "${error_msg} (${limit} of ${enums[param].size()}): ${enums[param][0..limit-1].join(', ')}, ... )" + } else { + log.error "${error_msg}: ${enums[param].join(', ')})" + } + } else { + log.error "* --${param}: ${ex_json['message']} (${param_val})" + } } } for (ex in causingExceptions) { From de70e688116bc4e8581d620ed830ac589e218f1f Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 22 Nov 2021 14:47:49 +0100 Subject: [PATCH 04/28] fix lint --- .github/ISSUE_TEMPLATE/{bug_report.md => bug_report.yml} | 0 .../ISSUE_TEMPLATE/{feature_request.md => feature_request.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/ISSUE_TEMPLATE/{bug_report.md => bug_report.yml} (100%) rename .github/ISSUE_TEMPLATE/{feature_request.md => feature_request.yml} (100%) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.md rename to .github/ISSUE_TEMPLATE/bug_report.yml diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .github/ISSUE_TEMPLATE/feature_request.yml From 206c3f5ef8937cf78d3b20dba9d7df32e20eb22d Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 22 Nov 2021 14:48:34 +0100 Subject: [PATCH 05/28] sync template for lint --- .github/ISSUE_TEMPLATE/bug_report.yml | 111 ++++++++++----------- .github/ISSUE_TEMPLATE/feature_request.yml | 39 ++------ 2 files changed, 59 insertions(+), 91 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ef2b2e52fc..358b415f77 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,63 +1,52 @@ ---- + name: Bug report -about: Report something that is broken or incorrect +description: Report something that is broken or incorrect labels: bug ---- - - - -## Check Documentation - -I have checked the following places for your error: - -- [ ] [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting) -- [ ] [nf-core/sarek pipeline documentation](https://nf-co.re/sarek/usage) - -## Description of the bug - - - -## Steps to reproduce - -Steps to reproduce the behaviour: - -1. Command line: -2. See error: - -## Expected behaviour - - - -## Log files - -Have you provided the following extra information/files: - -- [ ] The command used to run the pipeline -- [ ] The `.nextflow.log` file - -## System - -- Hardware: -- Executor: -- OS: -- Version - -## Nextflow Installation - -- Version: - -## Container engine - -- Engine: -- version: - -## Additional context - - +body: + + - type: markdown + attributes: + value: | + Before you post this issue, please check the documentation: + + - [nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting) + - [nf-core/sarek pipeline documentation](https://nf-co.re/sarek/usage) + + - type: textarea + id: description + attributes: + label: Description of the bug + description: A clear and concise description of what the bug is. + validations: + required: true + + - type: textarea + id: command_used + attributes: + label: Command used and terminal output + description: Steps to reproduce the behaviour. Please paste the command you used to launch the pipeline and the output from your terminal. + render: console + placeholder: | + $ nextflow run ... + + Some output where something broke + + - type: textarea + id: files + attributes: + label: Relevant files + description: | + Please upload (drag and drop) and relevant files. Make into a `.zip` file if the extension is not allowed. + Your verbose log file `.nextflow.log` is often useful _(this is a hidden file in the directory where you launched the pipeline)_ as well as custom Nextflow configuration files. + + - type: textarea + id: system + attributes: + label: System information + description: | + * Nextflow version _(eg. 21.04.01)_ + * Hardware _(eg. HPC, Desktop, Cloud)_ + * Executor _(eg. slurm, local, awsbatch)_ + * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_ + * OS _(eg. CentOS Linux, macOS, Linux Mint)_ + * Version of nf-core/sarek _(eg. 1.1, 1.5, 1.8.2)_ diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 1794083651..604195a000 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,32 +1,11 @@ ---- name: Feature request -about: Suggest an idea for the nf-core/sarek pipeline +description: Suggest an idea for the nf-core/sarek pipeline labels: enhancement ---- - - - -## Is your feature request related to a problem? Please describe - - - - - -## Describe the solution you'd like - - - -## Describe alternatives you've considered - - - -## Additional context - - +body: + - type: textarea + id: description + attributes: + label: Description of feature + description: Please describe your suggestion for a new feature. It might help to describe a problem or use case, plus any alternatives that you have considered. + validations: + required: true From 72e31df0d3b6e712a8b80b6e970f50d758b59856 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 22 Nov 2021 15:17:44 +0100 Subject: [PATCH 06/28] fix: update tests --- .github/workflows/ci.yml | 4 +- tests/test_aligner.yml | 31 +++++++-------- tests/test_default.yml | 31 +++++++-------- tests/test_gatk_spark.yml | 31 +++++++-------- tests/test_pair.yml | 60 +++++++++++++----------------- tests/test_save_bam_mapped.yml | 35 ++++++++--------- tests/test_skip_markduplicates.yml | 59 ++++++++++++++--------------- 7 files changed, 109 insertions(+), 142 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77a6956a0c..f95baa39e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,9 +32,9 @@ jobs: - 'gatk4_spark' - 'save_bam_mapped' - 'skip_markduplicates' - - 'targeted' + # - 'targeted' - 'tumor_normal_pair' - - 'variant_calling' + # - 'variant_calling' steps: - name: Check out pipeline code uses: actions/checkout@v2 diff --git a/tests/test_aligner.yml b/tests/test_aligner.yml index 2f452eb2e5..5ce45ca8f4 100644 --- a/tests/test_aligner.yml +++ b/tests/test_aligner.yml @@ -6,24 +6,19 @@ - preprocessing files: - path: results/multiqc - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram.crai - - path: results/preprocessing/1234N/recal_table/1234N.recal.table - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram.crai + - path: results/preprocessing/test/markduplicates/test.md.cram + - path: results/preprocessing/test/markduplicates/test.md.cram.crai + - path: results/preprocessing/test/recal_table/test.recal.table + - path: results/preprocessing/test/recalibrated/test.recal.cram + - path: results/preprocessing/test/recalibrated/test.recal.cram.crai - path: results/preprocessing/csv/markduplicates.csv - - path: results/preprocessing/csv/markduplicates_1234N.csv + - path: results/preprocessing/csv/markduplicates_test.csv - path: results/preprocessing/csv/markduplicates_no_table.csv - - path: results/preprocessing/csv/markduplicates_no_table_1234N.csv + - path: results/preprocessing/csv/markduplicates_no_table_test.csv - path: results/preprocessing/csv/recalibrated.csv - - path: results/preprocessing/csv/recalibrated_1234N.csv - - path: results/reports/fastqc/1234N-1234N_M1 - - path: results/reports/fastqc/1234N-1234N_M2 - - path: results/reports/fastqc/1234N-1234N_M4 - - path: results/reports/fastqc/1234N-1234N_M5 - - path: results/reports/fastqc/1234N-1234N_M6 - - path: results/reports/fastqc/1234N-1234N_M7 - - path: results/reports/qualimap/1234N/1234N.mapped - - path: results/reports/qualimap/1234N/1234N.recal - - path: results/reports/samtools_stats/1234N/1234N.md.cram.stats - - path: results/reports/samtools_stats/1234N/1234N.recal.cram.stats + - path: results/preprocessing/csv/recalibrated_test.csv + - path: results/reports/fastqc/test-test_L1 + - path: results/reports/qualimap/test/test.mapped + - path: results/reports/qualimap/test/test.recal + - path: results/reports/samtools_stats/test/test.md.cram.stats + - path: results/reports/samtools_stats/test/test.recal.cram.stats diff --git a/tests/test_default.yml b/tests/test_default.yml index 536d136be5..7d0c821582 100644 --- a/tests/test_default.yml +++ b/tests/test_default.yml @@ -5,24 +5,19 @@ - preprocessing files: - path: results/multiqc - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram.crai - - path: results/preprocessing/1234N/recal_table/1234N.recal.table - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram.crai + - path: results/preprocessing/test/markduplicates/test.md.cram + - path: results/preprocessing/test/markduplicates/test.md.cram.crai + - path: results/preprocessing/test/recal_table/test.recal.table + - path: results/preprocessing/test/recalibrated/test.recal.cram + - path: results/preprocessing/test/recalibrated/test.recal.cram.crai - path: results/preprocessing/csv/markduplicates.csv - - path: results/preprocessing/csv/markduplicates_1234N.csv + - path: results/preprocessing/csv/markduplicates_test.csv - path: results/preprocessing/csv/markduplicates_no_table.csv - - path: results/preprocessing/csv/markduplicates_no_table_1234N.csv + - path: results/preprocessing/csv/markduplicates_no_table_test.csv - path: results/preprocessing/csv/recalibrated.csv - - path: results/preprocessing/csv/recalibrated_1234N.csv - - path: results/reports/fastqc/1234N-1234N_M1 - - path: results/reports/fastqc/1234N-1234N_M2 - - path: results/reports/fastqc/1234N-1234N_M4 - - path: results/reports/fastqc/1234N-1234N_M5 - - path: results/reports/fastqc/1234N-1234N_M6 - - path: results/reports/fastqc/1234N-1234N_M7 - - path: results/reports/qualimap/1234N/1234N.mapped - - path: results/reports/qualimap/1234N/1234N.recal - - path: results/reports/samtools_stats/1234N/1234N.md.cram.stats - - path: results/reports/samtools_stats/1234N/1234N.recal.cram.stats + - path: results/preprocessing/csv/recalibrated_test.csv + - path: results/reports/fastqc/test-test_L1 + - path: results/reports/qualimap/test/test.mapped + - path: results/reports/qualimap/test/test.recal + - path: results/reports/samtools_stats/test/test.md.cram.stats + - path: results/reports/samtools_stats/test/test.recal.cram.stats diff --git a/tests/test_gatk_spark.yml b/tests/test_gatk_spark.yml index 50a7c6518f..769e6eaa74 100644 --- a/tests/test_gatk_spark.yml +++ b/tests/test_gatk_spark.yml @@ -6,24 +6,19 @@ - preprocessing files: - path: results/multiqc - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram.crai - - path: results/preprocessing/1234N/recal_table/1234N.recal.table - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram.crai + - path: results/preprocessing/test/markduplicates/test.md.cram + - path: results/preprocessing/test/markduplicates/test.md.cram.crai + - path: results/preprocessing/test/recal_table/test.recal.table + - path: results/preprocessing/test/recalibrated/test.recal.cram + - path: results/preprocessing/test/recalibrated/test.recal.cram.crai - path: results/preprocessing/csv/markduplicates.csv - - path: results/preprocessing/csv/markduplicates_1234N.csv + - path: results/preprocessing/csv/markduplicates_test.csv - path: results/preprocessing/csv/markduplicates_no_table.csv - - path: results/preprocessing/csv/markduplicates_no_table_1234N.csv + - path: results/preprocessing/csv/markduplicates_no_table_test.csv - path: results/preprocessing/csv/recalibrated.csv - - path: results/preprocessing/csv/recalibrated_1234N.csv - - path: results/reports/fastqc/1234N-1234N_M1 - - path: results/reports/fastqc/1234N-1234N_M2 - - path: results/reports/fastqc/1234N-1234N_M4 - - path: results/reports/fastqc/1234N-1234N_M5 - - path: results/reports/fastqc/1234N-1234N_M6 - - path: results/reports/fastqc/1234N-1234N_M7 - - path: results/reports/qualimap/1234N/1234N.mapped - - path: results/reports/qualimap/1234N/1234N.recal - - path: results/reports/samtools_stats/1234N/1234N.md.cram.stats - - path: results/reports/samtools_stats/1234N/1234N.recal.cram.stats + - path: results/preprocessing/csv/recalibrated_test.csv + - path: results/reports/fastqc/test-test_L1 + - path: results/reports/qualimap/test/test.mapped + - path: results/reports/qualimap/test/test.recal + - path: results/reports/samtools_stats/test/test.md.cram.stats + - path: results/reports/samtools_stats/test/test.recal.cram.stats diff --git a/tests/test_pair.yml b/tests/test_pair.yml index 90913b0cc8..d66f477734 100644 --- a/tests/test_pair.yml +++ b/tests/test_pair.yml @@ -5,40 +5,32 @@ - tumor_normal_pair files: - path: results/multiqc - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram.crai - - path: results/preprocessing/1234N/recal_table/1234N.recal.table - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram.crai - - path: results/preprocessing/9876T/markduplicates/9876T.md.cram - - path: results/preprocessing/9876T/markduplicates/9876T.md.cram.crai - - path: results/preprocessing/9876T/recal_table/9876T.recal.table - - path: results/preprocessing/9876T/recalibrated/9876T.recal.cram - - path: results/preprocessing/9876T/recalibrated/9876T.recal.cram.crai + - path: results/preprocessing/test/markduplicates/test.md.cram + - path: results/preprocessing/test/markduplicates/test.md.cram.crai + - path: results/preprocessing/test/recal_table/test.recal.table + - path: results/preprocessing/test/recalibrated/test.recal.cram + - path: results/preprocessing/test/recalibrated/test.recal.cram.crai + - path: results/preprocessing/test2/markduplicates/test2.md.cram + - path: results/preprocessing/test2/markduplicates/test2.md.cram.crai + - path: results/preprocessing/test2/recal_table/test2.recal.table + - path: results/preprocessing/test2/recalibrated/test2.recal.cram + - path: results/preprocessing/test2/recalibrated/test2.recal.cram.crai - path: results/preprocessing/csv/markduplicates.csv - - path: results/preprocessing/csv/markduplicates_1234N.csv - - path: results/preprocessing/csv/markduplicates_9876T.csv + - path: results/preprocessing/csv/markduplicates_test.csv + - path: results/preprocessing/csv/markduplicates_test2.csv - path: results/preprocessing/csv/markduplicates_no_table.csv - - path: results/preprocessing/csv/markduplicates_no_table_1234N.csv - - path: results/preprocessing/csv/markduplicates_no_table_9876T.csv + - path: results/preprocessing/csv/markduplicates_no_table_test.csv + - path: results/preprocessing/csv/markduplicates_no_table_test2.csv - path: results/preprocessing/csv/recalibrated.csv - - path: results/preprocessing/csv/recalibrated_1234N.csv - - path: results/preprocessing/csv/recalibrated_9876T.csv - - path: results/reports/fastqc/1234N-1234N_M1 - - path: results/reports/fastqc/1234N-1234N_M2 - - path: results/reports/fastqc/1234N-1234N_M4 - - path: results/reports/fastqc/1234N-1234N_M5 - - path: results/reports/fastqc/1234N-1234N_M6 - - path: results/reports/fastqc/1234N-1234N_M7 - - path: results/reports/fastqc/9876T-9876T_M1 - - path: results/reports/fastqc/9876T-9876T_M2 - - path: results/reports/fastqc/9876T-9876T_M4 - - path: results/reports/fastqc/9876T-9876T_M5 - - path: results/reports/qualimap/1234N/1234N.mapped - - path: results/reports/qualimap/1234N/1234N.recal - - path: results/reports/qualimap/9876T/9876T.mapped - - path: results/reports/qualimap/9876T/9876T.recal - - path: results/reports/samtools_stats/1234N/1234N.md.cram.stats - - path: results/reports/samtools_stats/1234N/1234N.recal.cram.stats - - path: results/reports/samtools_stats/9876T/9876T.md.cram.stats - - path: results/reports/samtools_stats/9876T/9876T.recal.cram.stats + - path: results/preprocessing/csv/recalibrated_test.csv + - path: results/preprocessing/csv/recalibrated_test2.csv + - path: results/reports/fastqc/test-test_L1 + - path: results/reports/fastqc/test2-test2_L1 + - path: results/reports/qualimap/test/test.mapped + - path: results/reports/qualimap/test/test.recal + - path: results/reports/qualimap/test2/test2.mapped + - path: results/reports/qualimap/test2/test2.recal + - path: results/reports/samtools_stats/test/test.md.cram.stats + - path: results/reports/samtools_stats/test/test.recal.cram.stats + - path: results/reports/samtools_stats/test2/test2.md.cram.stats + - path: results/reports/samtools_stats/test2/test2.recal.cram.stats diff --git a/tests/test_save_bam_mapped.yml b/tests/test_save_bam_mapped.yml index ce576aa825..9e1a2fc5fd 100644 --- a/tests/test_save_bam_mapped.yml +++ b/tests/test_save_bam_mapped.yml @@ -5,26 +5,21 @@ - save_bam_mapped files: - path: results/multiqc - - path: results/preprocessing/1234N/mapped/1234N.bam - - path: results/preprocessing/1234N/mapped/1234N.bam.bai - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram.crai - - path: results/preprocessing/1234N/recal_table/1234N.recal.table - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram.crai + - path: results/preprocessing/test/mapped/test.bam + - path: results/preprocessing/test/mapped/test.bam.bai + - path: results/preprocessing/test/markduplicates/test.md.cram + - path: results/preprocessing/test/markduplicates/test.md.cram.crai + - path: results/preprocessing/test/recal_table/test.recal.table + - path: results/preprocessing/test/recalibrated/test.recal.cram + - path: results/preprocessing/test/recalibrated/test.recal.cram.crai - path: results/preprocessing/csv/markduplicates.csv - - path: results/preprocessing/csv/markduplicates_1234N.csv + - path: results/preprocessing/csv/markduplicates_test.csv - path: results/preprocessing/csv/markduplicates_no_table.csv - - path: results/preprocessing/csv/markduplicates_no_table_1234N.csv + - path: results/preprocessing/csv/markduplicates_no_table_test.csv - path: results/preprocessing/csv/recalibrated.csv - - path: results/preprocessing/csv/recalibrated_1234N.csv - - path: results/reports/fastqc/1234N-1234N_M1 - - path: results/reports/fastqc/1234N-1234N_M2 - - path: results/reports/fastqc/1234N-1234N_M4 - - path: results/reports/fastqc/1234N-1234N_M5 - - path: results/reports/fastqc/1234N-1234N_M6 - - path: results/reports/fastqc/1234N-1234N_M7 - - path: results/reports/qualimap/1234N/1234N.mapped - - path: results/reports/qualimap/1234N/1234N.recal - - path: results/reports/samtools_stats/1234N/1234N.md.cram.stats - - path: results/reports/samtools_stats/1234N/1234N.recal.cram.stats + - path: results/preprocessing/csv/recalibrated_test.csv + - path: results/reports/fastqc/test-test_L1 + - path: results/reports/qualimap/test/test.mapped + - path: results/reports/qualimap/test/test.recal + - path: results/reports/samtools_stats/test/test.md.cram.stats + - path: results/reports/samtools_stats/test/test.recal.cram.stats diff --git a/tests/test_skip_markduplicates.yml b/tests/test_skip_markduplicates.yml index d9525f49b8..66d66b396d 100644 --- a/tests/test_skip_markduplicates.yml +++ b/tests/test_skip_markduplicates.yml @@ -6,29 +6,24 @@ - skip_markduplicates files: - path: results/multiqc - - path: results/preprocessing/1234N/mapped/1234N.bam - - path: results/preprocessing/1234N/mapped/1234N.bam.bai - - path: results/preprocessing/1234N/mapped/1234N.recal.table - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram.crai - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram.crai + - path: results/preprocessing/test/mapped/test.bam + - path: results/preprocessing/test/mapped/test.bam.bai + - path: results/preprocessing/test/mapped/test.recal.table + - path: results/preprocessing/test/markduplicates/test.md.cram + - path: results/preprocessing/test/markduplicates/test.md.cram.crai + - path: results/preprocessing/test/recalibrated/test.recal.cram + - path: results/preprocessing/test/recalibrated/test.recal.cram.crai - path: results/preprocessing/csv/markduplicates.csv - - path: results/preprocessing/csv/markduplicates_1234N.csv + - path: results/preprocessing/csv/markduplicates_test.csv - path: results/preprocessing/csv/markduplicates_no_table.csv - - path: results/preprocessing/csv/markduplicates_no_table_1234N.csv + - path: results/preprocessing/csv/markduplicates_no_table_test.csv - path: results/preprocessing/csv/recalibrated.csv - - path: results/preprocessing/csv/recalibrated_1234N.csv - - path: results/reports/fastqc/1234N-1234N_M1 - - path: results/reports/fastqc/1234N-1234N_M2 - - path: results/reports/fastqc/1234N-1234N_M4 - - path: results/reports/fastqc/1234N-1234N_M5 - - path: results/reports/fastqc/1234N-1234N_M6 - - path: results/reports/fastqc/1234N-1234N_M7 - - path: results/reports/qualimap/1234N/1234N.mapped - - path: results/reports/qualimap/1234N/1234N.recal - - path: results/reports/samtools_stats/1234N/1234N.md.cram.stats - - path: results/reports/samtools_stats/1234N/1234N.recal.cram.stats + - path: results/preprocessing/csv/recalibrated_test.csv + - path: results/reports/fastqc/test-test_L1 + - path: results/reports/qualimap/test/test.mapped + - path: results/reports/qualimap/test/test.recal + - path: results/reports/samtools_stats/test/test.md.cram.stats + - path: results/reports/samtools_stats/test/test.recal.cram.stats - name: Run default pipeline while skipping MarkDuplicates, starting with prepare_recalibration command: nextflow run main.nf -profile test,docker,prepare_recalibration,skip_markduplicates tags: @@ -38,18 +33,18 @@ - skip_markduplicates files: - path: results/multiqc - - path: results/preprocessing/1234N/mapped/1234N.recal.table - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram - - path: results/preprocessing/1234N/markduplicates/1234N.md.cram.crai - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram - - path: results/preprocessing/1234N/recalibrated/1234N.recal.cram.crai + - path: results/preprocessing/test/mapped/test.recal.table + - path: results/preprocessing/test/markduplicates/test.md.cram + - path: results/preprocessing/test/markduplicates/test.md.cram.crai + - path: results/preprocessing/test/recalibrated/test.recal.cram + - path: results/preprocessing/test/recalibrated/test.recal.cram.crai - path: results/preprocessing/csv/markduplicates.csv - - path: results/preprocessing/csv/markduplicates_1234N.csv + - path: results/preprocessing/csv/markduplicates_test.csv - path: results/preprocessing/csv/markduplicates_no_table.csv - - path: results/preprocessing/csv/markduplicates_no_table_1234N.csv + - path: results/preprocessing/csv/markduplicates_no_table_test.csv - path: results/preprocessing/csv/recalibrated.csv - - path: results/preprocessing/csv/recalibrated_1234N.csv - - path: results/reports/qualimap/1234N/1234N.mapped - - path: results/reports/qualimap/1234N/1234N.recal - - path: results/reports/samtools_stats/1234N/1234N.md.cram.stats - - path: results/reports/samtools_stats/1234N/1234N.recal.cram.stats + - path: results/preprocessing/csv/recalibrated_test.csv + - path: results/reports/qualimap/test/test.mapped + - path: results/reports/qualimap/test/test.recal + - path: results/reports/samtools_stats/test/test.md.cram.stats + - path: results/reports/samtools_stats/test/test.recal.cram.stats From f84a2941bac35445b66d1f55556b1fc4e2e50a07 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 22 Nov 2021 16:46:00 +0100 Subject: [PATCH 07/28] fix tests --- tests/test_save_bam_mapped.yml | 6 ++++-- tests/test_skip_markduplicates.yml | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_save_bam_mapped.yml b/tests/test_save_bam_mapped.yml index 9e1a2fc5fd..8acefd914f 100644 --- a/tests/test_save_bam_mapped.yml +++ b/tests/test_save_bam_mapped.yml @@ -5,13 +5,15 @@ - save_bam_mapped files: - path: results/multiqc - - path: results/preprocessing/test/mapped/test.bam - - path: results/preprocessing/test/mapped/test.bam.bai + # - path: results/preprocessing/test/mapped/test-test_L1.bam + - path: results/preprocessing/test/mapped/test-test_L1.bam.bai - path: results/preprocessing/test/markduplicates/test.md.cram - path: results/preprocessing/test/markduplicates/test.md.cram.crai - path: results/preprocessing/test/recal_table/test.recal.table - path: results/preprocessing/test/recalibrated/test.recal.cram - path: results/preprocessing/test/recalibrated/test.recal.cram.crai + - path: results/preprocessing/csv/mapped.csv + - path: results/preprocessing/csv/mapped_test.csv - path: results/preprocessing/csv/markduplicates.csv - path: results/preprocessing/csv/markduplicates_test.csv - path: results/preprocessing/csv/markduplicates_no_table.csv diff --git a/tests/test_skip_markduplicates.yml b/tests/test_skip_markduplicates.yml index 66d66b396d..f0e38dbc2f 100644 --- a/tests/test_skip_markduplicates.yml +++ b/tests/test_skip_markduplicates.yml @@ -6,8 +6,8 @@ - skip_markduplicates files: - path: results/multiqc - - path: results/preprocessing/test/mapped/test.bam - - path: results/preprocessing/test/mapped/test.bam.bai + # - path: results/preprocessing/test/mapped/test-test_L1.bam + - path: results/preprocessing/test/mapped/test-test_L1.bam.bai - path: results/preprocessing/test/mapped/test.recal.table - path: results/preprocessing/test/markduplicates/test.md.cram - path: results/preprocessing/test/markduplicates/test.md.cram.crai From f8bfab0c18f3acdab9e889c0f7d06c536eb39be3 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 22 Nov 2021 16:53:43 +0100 Subject: [PATCH 08/28] remove part of tests for now --- tests/test_skip_markduplicates.yml | 48 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_skip_markduplicates.yml b/tests/test_skip_markduplicates.yml index f0e38dbc2f..41e44d9803 100644 --- a/tests/test_skip_markduplicates.yml +++ b/tests/test_skip_markduplicates.yml @@ -24,27 +24,27 @@ - path: results/reports/qualimap/test/test.recal - path: results/reports/samtools_stats/test/test.md.cram.stats - path: results/reports/samtools_stats/test/test.recal.cram.stats -- name: Run default pipeline while skipping MarkDuplicates, starting with prepare_recalibration - command: nextflow run main.nf -profile test,docker,prepare_recalibration,skip_markduplicates - tags: - - markduplicates - - prepare_recalibration - - preprocessing - - skip_markduplicates - files: - - path: results/multiqc - - path: results/preprocessing/test/mapped/test.recal.table - - path: results/preprocessing/test/markduplicates/test.md.cram - - path: results/preprocessing/test/markduplicates/test.md.cram.crai - - path: results/preprocessing/test/recalibrated/test.recal.cram - - path: results/preprocessing/test/recalibrated/test.recal.cram.crai - - path: results/preprocessing/csv/markduplicates.csv - - path: results/preprocessing/csv/markduplicates_test.csv - - path: results/preprocessing/csv/markduplicates_no_table.csv - - path: results/preprocessing/csv/markduplicates_no_table_test.csv - - path: results/preprocessing/csv/recalibrated.csv - - path: results/preprocessing/csv/recalibrated_test.csv - - path: results/reports/qualimap/test/test.mapped - - path: results/reports/qualimap/test/test.recal - - path: results/reports/samtools_stats/test/test.md.cram.stats - - path: results/reports/samtools_stats/test/test.recal.cram.stats +# - name: Run default pipeline while skipping MarkDuplicates, starting with prepare_recalibration +# command: nextflow run main.nf -profile test,docker,prepare_recalibration,skip_markduplicates +# tags: +# - markduplicates +# - prepare_recalibration +# - preprocessing +# - skip_markduplicates +# files: +# - path: results/multiqc +# - path: results/preprocessing/test/mapped/test.recal.table +# - path: results/preprocessing/test/markduplicates/test.md.cram +# - path: results/preprocessing/test/markduplicates/test.md.cram.crai +# - path: results/preprocessing/test/recalibrated/test.recal.cram +# - path: results/preprocessing/test/recalibrated/test.recal.cram.crai +# - path: results/preprocessing/csv/markduplicates.csv +# - path: results/preprocessing/csv/markduplicates_test.csv +# - path: results/preprocessing/csv/markduplicates_no_table.csv +# - path: results/preprocessing/csv/markduplicates_no_table_test.csv +# - path: results/preprocessing/csv/recalibrated.csv +# - path: results/preprocessing/csv/recalibrated_test.csv +# - path: results/reports/qualimap/test/test.mapped +# - path: results/reports/qualimap/test/test.recal +# - path: results/reports/samtools_stats/test/test.md.cram.stats +# - path: results/reports/samtools_stats/test/test.recal.cram.stats From 4302f13545cbbb719f7bbf1c777f6d96a5a90a67 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 23 Nov 2021 15:40:58 +0100 Subject: [PATCH 09/28] fix: save bam and bai --- conf/modules.config | 6 ++---- modules/local/samtools/index/main.nf | 8 ++++---- subworkflows/nf-core/mapping.nf | 2 +- subworkflows/nf-core/recalibrate.nf | 4 ++-- tests/test_save_bam_mapped.yml | 2 +- tests/test_skip_markduplicates.yml | 2 +- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index c10aae17f2..a35be6a55e 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -94,13 +94,11 @@ params { } 'samtools_index_mapping' { publish_by_meta = true - publish_files = ['bai':'mapped'] + publish_files = ['bam': 'mapped', 'bai': 'mapped'] publish_dir = 'preprocessing' } 'merge_bam_mapping' { - publish_by_meta = true - publish_files = ['bam':'mapped'] - publish_dir = 'preprocessing' + publish_files = false } 'seqkit_split2' { args = "--by-size ${params.split_fastq}" diff --git a/modules/local/samtools/index/main.nf b/modules/local/samtools/index/main.nf index 26f12ebda4..b1fe410d69 100644 --- a/modules/local/samtools/index/main.nf +++ b/modules/local/samtools/index/main.nf @@ -22,10 +22,10 @@ process SAMTOOLS_INDEX { tuple val(meta), path(input) output: - tuple val(meta), path("*.bai") , optional:true, emit: bai - tuple val(meta), path("*.csi") , optional:true, emit: csi - tuple val(meta), path("*.crai"), optional:true, emit: crai - path "*.version.txt" , emit: version + tuple val(meta), path("*.bam", includeInputs:true), path("*.bai") , optional:true, emit: bam_bai + tuple val(meta), path("*.bam", includeInputs:true), path("*.csi") , optional:true, emit: bam_csi + tuple val(meta), path("*.cram", includeInputs:true), path("*.crai"), optional:true, emit: cram_crai + path "*.version.txt" , emit: version script: def software = getSoftwareName(task.process) diff --git a/subworkflows/nf-core/mapping.nf b/subworkflows/nf-core/mapping.nf index 5dbd1e834d..cc48ed4f24 100644 --- a/subworkflows/nf-core/mapping.nf +++ b/subworkflows/nf-core/mapping.nf @@ -115,7 +115,7 @@ workflow MAPPING { bam_merged = bam_to_merge.single.mix(SAMTOOLS_MERGE.out.bam) SAMTOOLS_INDEX(bam_merged) - bam_indexed = bam_merged.join(SAMTOOLS_INDEX.out.bai) + bam_indexed = SAMTOOLS_INDEX.out.bam_bai } emit: diff --git a/subworkflows/nf-core/recalibrate.nf b/subworkflows/nf-core/recalibrate.nf index 285ab29bdc..81f7923c92 100644 --- a/subworkflows/nf-core/recalibrate.nf +++ b/subworkflows/nf-core/recalibrate.nf @@ -62,13 +62,13 @@ workflow RECALIBRATE { cram_recalibrated = SAMTOOLS_MERGE_CRAM.out.cram SAMTOOLS_INDEX(cram_recalibrated) - cram_recalibrated_index = cram_recalibrated.join(SAMTOOLS_INDEX.out.crai) + cram_recalibrated_index = SAMTOOLS_INDEX.out.cram_crai qualimap_bamqc = Channel.empty() samtools_stats = Channel.empty() if (!skip_bamqc) { - QUALIMAP_BAMQC_CRAM(cram_recalibrated_index,target_bed, params.target_bed,fasta, fai) + QUALIMAP_BAMQC_CRAM(cram_recalibrated_index, target_bed, params.target_bed,fasta, fai) qualimap_bamqc = QUALIMAP_BAMQC_CRAM.out.results } diff --git a/tests/test_save_bam_mapped.yml b/tests/test_save_bam_mapped.yml index 8acefd914f..45cdd7d410 100644 --- a/tests/test_save_bam_mapped.yml +++ b/tests/test_save_bam_mapped.yml @@ -5,7 +5,7 @@ - save_bam_mapped files: - path: results/multiqc - # - path: results/preprocessing/test/mapped/test-test_L1.bam + - path: results/preprocessing/test/mapped/test-test_L1.bam - path: results/preprocessing/test/mapped/test-test_L1.bam.bai - path: results/preprocessing/test/markduplicates/test.md.cram - path: results/preprocessing/test/markduplicates/test.md.cram.crai diff --git a/tests/test_skip_markduplicates.yml b/tests/test_skip_markduplicates.yml index 41e44d9803..dc9fd69d0d 100644 --- a/tests/test_skip_markduplicates.yml +++ b/tests/test_skip_markduplicates.yml @@ -6,7 +6,7 @@ - skip_markduplicates files: - path: results/multiqc - # - path: results/preprocessing/test/mapped/test-test_L1.bam + - path: results/preprocessing/test/mapped/test-test_L1.bam - path: results/preprocessing/test/mapped/test-test_L1.bam.bai - path: results/preprocessing/test/mapped/test.recal.table - path: results/preprocessing/test/markduplicates/test.md.cram From 0024e06bc19691a64f61b51f27867b9d20dace49 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 23 Nov 2021 16:08:34 +0100 Subject: [PATCH 10/28] feat: update markduplicates --- conf/modules.config | 2 +- modules.json | 3 + .../local/gatk4/markduplicates/functions.nf | 54 +++++---- modules/local/gatk4/markduplicates/main.nf | 42 ++++--- modules/local/gatk4/markduplicates/meta.yml | 9 +- .../custom/dumpsoftwareversions/functions.nf | 78 +++++++++++++ .../custom/dumpsoftwareversions/main.nf | 106 ++++++++++++++++++ .../custom/dumpsoftwareversions/meta.yml | 33 ++++++ subworkflows/nf-core/markduplicates.nf | 20 ++-- 9 files changed, 288 insertions(+), 59 deletions(-) create mode 100644 modules/nf-core/modules/custom/dumpsoftwareversions/functions.nf create mode 100644 modules/nf-core/modules/custom/dumpsoftwareversions/main.nf create mode 100644 modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml diff --git a/conf/modules.config b/conf/modules.config index a35be6a55e..cfd1c7736c 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -113,7 +113,7 @@ params { publish_files = ['metrics': 'markduplicates'] } 'markduplicates' { - args = 'REMOVE_DUPLICATES=false VALIDATION_STRINGENCY=LENIENT' + args = '-REMOVE_DUPLICATES false -VALIDATION_STRINGENCY LENIENT' suffix = '.md' publish_by_meta = true publish_dir = 'preprocessing' diff --git a/modules.json b/modules.json index 6dffdac102..b0ba83a2aa 100644 --- a/modules.json +++ b/modules.json @@ -9,6 +9,9 @@ "bwamem2/index": { "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" }, + "custom/dumpsoftwareversions": { + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" + }, "ensemblvep": { "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" }, diff --git a/modules/local/gatk4/markduplicates/functions.nf b/modules/local/gatk4/markduplicates/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/markduplicates/functions.nf +++ b/modules/local/gatk4/markduplicates/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/markduplicates/main.nf b/modules/local/gatk4/markduplicates/main.nf index a0ec0f2f07..e30ff27e85 100644 --- a/modules/local/gatk4/markduplicates/main.nf +++ b/modules/local/gatk4/markduplicates/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -11,44 +11,42 @@ process GATK4_MARKDUPLICATES { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - conda (params.enable_conda ? "bioconda::gatk4=4.2.0.0" : null) + conda (params.enable_conda ? "bioconda::gatk4=4.2.3.0" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/gatk4:4.2.0.0--0" + container "https://depot.galaxyproject.org/singularity/gatk4:4.2.3.0--hdfd78af_0" } else { - container "quay.io/biocontainers/gatk4:4.2.0.0--0" + container "quay.io/biocontainers/gatk4:4.2.3.0--hdfd78af_0" } input: - tuple val(meta), path(bam) - val use_metrics + tuple val(meta), path(bams) output: - tuple val(meta), path("*.bam"), path("*.bai") , emit: bam - tuple val(meta), path("*.metrics"), optional : true, emit: metrics - path "*.version.txt" , emit: version + tuple val(meta), path("*.bam"), path("*.bai"), emit: bam_bai + tuple val(meta), path("*.metrics") , emit: metrics + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" - def metrics = use_metrics ? "M=${prefix}.metrics" :'' - def bams = bam.collect(){ x -> "INPUT=".concat(x.toString()) }.join(" ") + def bam_list = bams.collect(){ bam -> "--INPUT ".concat(bam.toString()) }.join(" ") + def avail_mem = 3 if (!task.memory) { - log.info '[GATK MarkDuplicates] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' + log.info '[GATK HaplotypeCaller] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { avail_mem = task.memory.giga } """ gatk MarkDuplicates \\ - $bams \\ - $metrics \\ - TMP_DIR=. \\ - ASSUME_SORT_ORDER=coordinate \\ - CREATE_INDEX=true \\ - O=${prefix}.bam \\ + $bam_list \\ + --METRICS_FILE ${prefix}.metrics \\ + --TMP_DIR . \\ + --CREATE_INDEX true \\ + --OUTPUT ${prefix}.bam \\ $options.args - mv ${prefix}.bai ${prefix}.bam.bai - - echo \$(gatk MarkDuplicates --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/markduplicates/meta.yml b/modules/local/gatk4/markduplicates/meta.yml index abe61e023f..5777067a4f 100644 --- a/modules/local/gatk4/markduplicates/meta.yml +++ b/modules/local/gatk4/markduplicates/meta.yml @@ -13,7 +13,7 @@ tools: documentation: https://gatk.broadinstitute.org/hc/en-us/articles/360037052812-MarkDuplicates-Picard- tool_dev_url: https://github.com/broadinstitute/gatk doi: 10.1158/1538-7445.AM2017-3590 - licence: ['BSD-3-clause'] + licence: ['MIT'] input: - meta: @@ -32,10 +32,10 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" - bam: type: file description: Marked duplicates BAM file @@ -47,3 +47,4 @@ output: authors: - "@ajodeh-juma" + - "@FriederikeHanssen" diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/functions.nf b/modules/nf-core/modules/custom/dumpsoftwareversions/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/custom/dumpsoftwareversions/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf b/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf new file mode 100644 index 0000000000..faf2073f75 --- /dev/null +++ b/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf @@ -0,0 +1,106 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process CUSTOM_DUMPSOFTWAREVERSIONS { + label 'process_low' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) } + + // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container + conda (params.enable_conda ? "bioconda::multiqc=1.11" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/multiqc:1.11--pyhdfd78af_0" + } else { + container "quay.io/biocontainers/multiqc:1.11--pyhdfd78af_0" + } + + input: + path versions + + output: + path "software_versions.yml" , emit: yml + path "software_versions_mqc.yml", emit: mqc_yml + path "versions.yml" , emit: versions + + script: + """ + #!/usr/bin/env python + + import yaml + import platform + from textwrap import dedent + + def _make_versions_html(versions): + html = [ + dedent( + '''\\ + + + + + + + + + + ''' + ) + ] + for process, tmp_versions in sorted(versions.items()): + html.append("") + for i, (tool, version) in enumerate(sorted(tmp_versions.items())): + html.append( + dedent( + f'''\\ + + + + + + ''' + ) + ) + html.append("") + html.append("
Process Name Software Version
{process if (i == 0) else ''}{tool}{version}
") + return "\\n".join(html) + + module_versions = {} + module_versions["${getProcessName(task.process)}"] = { + 'python': platform.python_version(), + 'yaml': yaml.__version__ + } + + with open("$versions") as f: + workflow_versions = yaml.load(f, Loader=yaml.BaseLoader) | module_versions + + workflow_versions["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version" + } + + versions_mqc = { + 'id': 'software_versions', + 'section_name': '${workflow.manifest.name} Software Versions', + 'section_href': 'https://github.com/${workflow.manifest.name}', + 'plot_type': 'html', + 'description': 'are collected at run time from the software output.', + 'data': _make_versions_html(workflow_versions) + } + + with open("software_versions.yml", 'w') as f: + yaml.dump(workflow_versions, f, default_flow_style=False) + with open("software_versions_mqc.yml", 'w') as f: + yaml.dump(versions_mqc, f, default_flow_style=False) + + with open('versions.yml', 'w') as f: + yaml.dump(module_versions, f, default_flow_style=False) + """ +} diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml new file mode 100644 index 0000000000..c8310e3560 --- /dev/null +++ b/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml @@ -0,0 +1,33 @@ +name: custom_dumpsoftwareversions +description: Custom module used to dump software versions within the nf-core pipeline template +keywords: + - custom + - version +tools: + - custom: + description: Custom module used to dump software versions within the nf-core pipeline template + homepage: https://github.com/nf-core/tools + documentation: https://github.com/nf-core/tools + licence: ['MIT'] +input: + - versions: + type: file + description: YML file containing software versions + pattern: "*.yml" + +output: + - yml: + type: file + description: Standard YML file containing software versions + pattern: "software_versions.yml" + - mqc_yml: + type: file + description: MultiQC custom content YML file containing software versions + pattern: "software_versions_mqc.yml" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@drpatelh" diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index 1582a1b36d..9dff6b0994 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -11,14 +11,14 @@ params.samtools_index_options = [:] params.samtools_stats_options = [:] params.samtools_view_options = [:] -include { GATK4_ESTIMATELIBRARYCOMPLEXITY } from '../../modules/local/gatk4/estimatelibrarycomplexity/main' addParams(options: params.estimatelibrarycomplexity_options) -include { GATK4_MARKDUPLICATES } from '../../modules/local/gatk4/markduplicates/main' addParams(options: params.markduplicates_options) -include { GATK4_MARKDUPLICATES_SPARK } from '../../modules/local/gatk4/markduplicatesspark/main' addParams(options: params.markduplicatesspark_options) -include { QUALIMAP_BAMQC } from '../../modules/local/qualimap/bamqc/main' addParams(options: params.qualimap_bamqc_options) -include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) -include { SAMTOOLS_STATS } from '../../modules/local/samtools/stats/main' addParams(options: params.samtools_stats_options) -include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM } from '../../modules/local/samtools/view/main.nf' addParams(options: params.samtools_view_options) -include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM_SPARK } from '../../modules/local/samtools/view/main.nf' addParams(options: params.samtools_view_options) +include { GATK4_ESTIMATELIBRARYCOMPLEXITY } from '../../modules/local/gatk4/estimatelibrarycomplexity/main' addParams(options: params.estimatelibrarycomplexity_options) +include { GATK4_MARKDUPLICATES } from '../../modules/local/gatk4/markduplicates/main' addParams(options: params.markduplicates_options) +include { GATK4_MARKDUPLICATES_SPARK } from '../../modules/local/gatk4/markduplicatesspark/main' addParams(options: params.markduplicatesspark_options) +include { QUALIMAP_BAMQC } from '../../modules/local/qualimap/bamqc/main' addParams(options: params.qualimap_bamqc_options) +include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) +include { SAMTOOLS_STATS } from '../../modules/local/samtools/stats/main' addParams(options: params.samtools_stats_options) +include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM } from '../../modules/local/samtools/view/main' addParams(options: params.samtools_view_options) +include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM_SPARK } from '../../modules/local/samtools/view/main' addParams(options: params.samtools_view_options) workflow MARKDUPLICATES { take: @@ -64,9 +64,9 @@ workflow MARKDUPLICATES { } } else { - GATK4_MARKDUPLICATES(bam_mapped, save_metrics) + GATK4_MARKDUPLICATES(bam_mapped) report_markduplicates = GATK4_MARKDUPLICATES.out.metrics - bam_markduplicates = GATK4_MARKDUPLICATES.out.bam + bam_markduplicates = GATK4_MARKDUPLICATES.out.bam_bai SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fai) cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram From 67751f69285946b5e8176ab5ae12386955100d3c Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 23 Nov 2021 17:03:20 +0100 Subject: [PATCH 11/28] feat: update modules bwa/index --- modules.json | 2 +- .../nf-core/modules/bwa/index/functions.nf | 54 +++++++++++-------- modules/nf-core/modules/bwa/index/main.nf | 19 ++++--- modules/nf-core/modules/bwa/index/meta.yml | 7 +-- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/modules.json b/modules.json index b0ba83a2aa..40a70b140b 100644 --- a/modules.json +++ b/modules.json @@ -4,7 +4,7 @@ "repos": { "nf-core/modules": { "bwa/index": { - "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" + "git_sha": "9475960928a3ba49624b49ef2b48438a4696ed0f" }, "bwamem2/index": { "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" diff --git a/modules/nf-core/modules/bwa/index/functions.nf b/modules/nf-core/modules/bwa/index/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/bwa/index/functions.nf +++ b/modules/nf-core/modules/bwa/index/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/bwa/index/main.nf b/modules/nf-core/modules/bwa/index/main.nf index aabd187c91..db1911cb9b 100644 --- a/modules/nf-core/modules/bwa/index/main.nf +++ b/modules/nf-core/modules/bwa/index/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -22,14 +22,21 @@ process BWA_INDEX { path fasta output: - path "bwa" , emit: index - path "*.version.txt", emit: version + path "bwa" , emit: index + path "versions.yml", emit: versions script: - def software = getSoftwareName(task.process) """ mkdir bwa - bwa index $options.args $fasta -p bwa/${fasta.baseName} - echo \$(bwa 2>&1) | sed 's/^.*Version: //; s/Contact:.*\$//' > ${software}.version.txt + bwa \\ + index \\ + $options.args \\ + -p bwa/${fasta.baseName} \\ + $fasta + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(bwa 2>&1) | sed 's/^.*Version: //; s/Contact:.*\$//') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/bwa/index/meta.yml b/modules/nf-core/modules/bwa/index/meta.yml index 181204c3da..11d62df374 100644 --- a/modules/nf-core/modules/bwa/index/meta.yml +++ b/modules/nf-core/modules/bwa/index/meta.yml @@ -13,6 +13,7 @@ tools: homepage: http://bio-bwa.sourceforge.net/ documentation: http://www.htslib.org/doc/samtools.html arxiv: arXiv:1303.3997 + licence: ['GPL-3.0-or-later'] input: - fasta: type: file @@ -22,10 +23,10 @@ output: type: file description: BWA genome index files pattern: "*.{amb,ann,bwt,pac,sa}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@maxulysse" From db42a4d18d551add84199b6ad9c8ee77d3f3e364 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 23 Nov 2021 17:08:48 +0100 Subject: [PATCH 12/28] feat: update modules bwamem2/index --- modules.json | 2 +- .../modules/bwamem2/index/functions.nf | 54 +++++++++++-------- modules/nf-core/modules/bwamem2/index/main.nf | 16 ++++-- .../nf-core/modules/bwamem2/index/meta.yml | 7 +-- 4 files changed, 48 insertions(+), 31 deletions(-) diff --git a/modules.json b/modules.json index 40a70b140b..fd28160b48 100644 --- a/modules.json +++ b/modules.json @@ -7,7 +7,7 @@ "git_sha": "9475960928a3ba49624b49ef2b48438a4696ed0f" }, "bwamem2/index": { - "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "custom/dumpsoftwareversions": { "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" diff --git a/modules/nf-core/modules/bwamem2/index/functions.nf b/modules/nf-core/modules/bwamem2/index/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/bwamem2/index/functions.nf +++ b/modules/nf-core/modules/bwamem2/index/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/bwamem2/index/main.nf b/modules/nf-core/modules/bwamem2/index/main.nf index b667f266cf..5732017f0b 100644 --- a/modules/nf-core/modules/bwamem2/index/main.nf +++ b/modules/nf-core/modules/bwamem2/index/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -23,13 +23,19 @@ process BWAMEM2_INDEX { output: path "bwamem2" , emit: index - path "*.version.txt", emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) """ mkdir bwamem2 - bwa-mem2 index $options.args $fasta -p bwamem2/${fasta} - echo \$(bwa-mem2 version 2>&1) > ${software}.version.txt + bwa-mem2 \\ + index \\ + $options.args \\ + $fasta -p bwamem2/${fasta} + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(bwa-mem2 version 2>&1) | sed 's/.* //') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/bwamem2/index/meta.yml b/modules/nf-core/modules/bwamem2/index/meta.yml index 9d717f739e..e0f6014cd7 100644 --- a/modules/nf-core/modules/bwamem2/index/meta.yml +++ b/modules/nf-core/modules/bwamem2/index/meta.yml @@ -12,6 +12,7 @@ tools: a large reference genome, such as the human genome. homepage: https://github.com/bwa-mem2/bwa-mem2 documentation: https://github.com/bwa-mem2/bwa-mem2#usage + licence: ['MIT'] input: - fasta: type: file @@ -21,9 +22,9 @@ output: type: file description: BWA genome index files pattern: "*.{0132,amb,ann,bwt.2bit.64,pac}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@maxulysse" From 76f575a53bfee10f689791124d0aa318017c478b Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 23 Nov 2021 17:12:14 +0100 Subject: [PATCH 13/28] feat: update modules gatk4/createsequencedictionary --- modules.json | 2 +- .../createsequencedictionary/functions.nf | 54 +++++++++++-------- .../gatk4/createsequencedictionary/main.nf | 16 +++--- .../gatk4/createsequencedictionary/meta.yml | 8 +-- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/modules.json b/modules.json index fd28160b48..130baa0958 100644 --- a/modules.json +++ b/modules.json @@ -19,7 +19,7 @@ "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" }, "gatk4/createsequencedictionary": { - "git_sha": "c8877835c288eb32b2ad1810d38a78e83e791f3d" + "git_sha": "3b600af50eae8264960df817277cfe303d2acd47" }, "multiqc": { "git_sha": "c5235a983d454787fa0c3247b02086969217163b" diff --git a/modules/nf-core/modules/gatk4/createsequencedictionary/functions.nf b/modules/nf-core/modules/gatk4/createsequencedictionary/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/gatk4/createsequencedictionary/functions.nf +++ b/modules/nf-core/modules/gatk4/createsequencedictionary/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/gatk4/createsequencedictionary/main.nf b/modules/nf-core/modules/gatk4/createsequencedictionary/main.nf index 0276e8b246..db28e244c4 100644 --- a/modules/nf-core/modules/gatk4/createsequencedictionary/main.nf +++ b/modules/nf-core/modules/gatk4/createsequencedictionary/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -11,11 +11,11 @@ process GATK4_CREATESEQUENCEDICTIONARY { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:[:], publish_by_meta:[]) } - conda (params.enable_conda ? "bioconda::gatk4=4.2.0.0" : null) + conda (params.enable_conda ? "bioconda::gatk4=4.2.3.0" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/gatk4:4.2.0.0--0" + container "https://depot.galaxyproject.org/singularity/gatk4:4.2.3.0--hdfd78af_0" } else { - container "quay.io/biocontainers/gatk4:4.2.0.0--0" + container "quay.io/biocontainers/gatk4:4.2.3.0--hdfd78af_0" } input: @@ -23,10 +23,9 @@ process GATK4_CREATESEQUENCEDICTIONARY { output: path "*.dict" , emit: dict - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) def avail_mem = 6 if (!task.memory) { log.info '[GATK] Available memory not known - defaulting to 6GB. Specify process memory requirements to change this.' @@ -40,6 +39,9 @@ process GATK4_CREATESEQUENCEDICTIONARY { --URI $fasta \\ $options.args - echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/gatk4/createsequencedictionary/meta.yml b/modules/nf-core/modules/gatk4/createsequencedictionary/meta.yml index d0cc59801f..54f479b393 100644 --- a/modules/nf-core/modules/gatk4/createsequencedictionary/meta.yml +++ b/modules/nf-core/modules/gatk4/createsequencedictionary/meta.yml @@ -12,6 +12,8 @@ tools: homepage: https://gatk.broadinstitute.org/hc/en-us documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s doi: 10.1158/1538-7445.AM2017-3590 + licence: ['Apache-2.0'] + input: - fasta: type: file @@ -22,9 +24,9 @@ output: type: file description: gatk dictionary file pattern: "*.{dict}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@maxulysse" From b91d9af9c73c5196a34492a0ca641dc039ed2fb2 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 10:16:13 +0100 Subject: [PATCH 14/28] feat: update more modules --- modules.json | 20 +++--- .../nf-core/modules/ensemblvep/functions.nf | 54 +++++++++------- modules/nf-core/modules/ensemblvep/main.nf | 10 +-- modules/nf-core/modules/ensemblvep/meta.yml | 7 ++- modules/nf-core/modules/fastqc/functions.nf | 54 +++++++++------- modules/nf-core/modules/fastqc/main.nf | 19 ++++-- modules/nf-core/modules/fastqc/meta.yml | 7 ++- modules/nf-core/modules/multiqc/functions.nf | 54 +++++++++------- modules/nf-core/modules/multiqc/main.nf | 11 ++-- modules/nf-core/modules/multiqc/meta.yml | 7 ++- .../modules/samtools/faidx/functions.nf | 54 +++++++++------- .../nf-core/modules/samtools/faidx/main.nf | 18 +++--- .../nf-core/modules/samtools/faidx/meta.yml | 7 ++- .../modules/samtools/merge/functions.nf | 54 +++++++++------- .../nf-core/modules/samtools/merge/main.nf | 27 +++++--- .../nf-core/modules/samtools/merge/meta.yml | 24 +++++--- .../modules/seqkit/split2/functions.nf | 54 +++++++++------- modules/nf-core/modules/seqkit/split2/main.nf | 61 ++++++++++--------- .../nf-core/modules/seqkit/split2/meta.yml | 7 ++- modules/nf-core/modules/snpeff/functions.nf | 54 +++++++++------- modules/nf-core/modules/snpeff/main.nf | 10 +-- modules/nf-core/modules/snpeff/meta.yml | 7 ++- .../modules/tabix/bgziptabix/functions.nf | 54 +++++++++------- .../nf-core/modules/tabix/bgziptabix/main.nf | 16 ++--- .../nf-core/modules/tabix/bgziptabix/meta.yml | 7 ++- .../nf-core/modules/tabix/tabix/functions.nf | 54 +++++++++------- modules/nf-core/modules/tabix/tabix/main.nf | 16 ++--- modules/nf-core/modules/tabix/tabix/meta.yml | 7 ++- .../nf-core/modules/trimgalore/functions.nf | 54 +++++++++------- modules/nf-core/modules/trimgalore/main.nf | 17 ++++-- modules/nf-core/modules/trimgalore/meta.yml | 7 ++- subworkflows/nf-core/fastqc_trimgalore.nf | 8 +-- 32 files changed, 507 insertions(+), 353 deletions(-) diff --git a/modules.json b/modules.json index 130baa0958..638b5823c3 100644 --- a/modules.json +++ b/modules.json @@ -13,37 +13,37 @@ "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "ensemblvep": { - "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "fastqc": { - "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "gatk4/createsequencedictionary": { "git_sha": "3b600af50eae8264960df817277cfe303d2acd47" }, "multiqc": { - "git_sha": "c5235a983d454787fa0c3247b02086969217163b" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "samtools/faidx": { - "git_sha": "c5235a983d454787fa0c3247b02086969217163b" + "git_sha": "19035c99d1cd8412b5046c6bb4af787658b84e5b" }, "samtools/merge": { - "git_sha": "c5235a983d454787fa0c3247b02086969217163b" + "git_sha": "19035c99d1cd8412b5046c6bb4af787658b84e5b" }, "seqkit/split2": { - "git_sha": "c5235a983d454787fa0c3247b02086969217163b" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "snpeff": { - "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "tabix/bgziptabix": { - "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "tabix/tabix": { - "git_sha": "e937c7950af70930d1f34bb961403d9d2aa81c7d" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, "trimgalore": { - "git_sha": "c5235a983d454787fa0c3247b02086969217163b" + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" } } } diff --git a/modules/nf-core/modules/ensemblvep/functions.nf b/modules/nf-core/modules/ensemblvep/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/ensemblvep/functions.nf +++ b/modules/nf-core/modules/ensemblvep/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/ensemblvep/main.nf b/modules/nf-core/modules/ensemblvep/main.nf index c2194e77b7..ad9c38a668 100644 --- a/modules/nf-core/modules/ensemblvep/main.nf +++ b/modules/nf-core/modules/ensemblvep/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -33,10 +33,9 @@ process ENSEMBLVEP { output: tuple val(meta), path("*.ann.vcf"), emit: vcf path "*.summary.html" , emit: report - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" dir_cache = params.use_cache ? "\${PWD}/${cache}" : "/.vep" """ @@ -57,6 +56,9 @@ process ENSEMBLVEP { rm -rf $prefix - echo \$(vep --help 2>&1) > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( echo \$(vep --help 2>&1) | sed 's/^.*Versions:.*ensembl-vep : //;s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/ensemblvep/meta.yml b/modules/nf-core/modules/ensemblvep/meta.yml index 5eb111e96a..1b8192279a 100644 --- a/modules/nf-core/modules/ensemblvep/meta.yml +++ b/modules/nf-core/modules/ensemblvep/meta.yml @@ -9,6 +9,7 @@ tools: or structural variants) on genes, transcripts, and protein sequence, as well as regulatory regions. homepage: https://www.ensembl.org/info/docs/tools/vep/index.html documentation: https://www.ensembl.org/info/docs/tools/vep/script/index.html + licence: ['Apache-2.0'] params: - use_cache: type: boolean @@ -56,9 +57,9 @@ output: type: file description: VEP report file pattern: "*.html" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@maxulysse" diff --git a/modules/nf-core/modules/fastqc/functions.nf b/modules/nf-core/modules/fastqc/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/fastqc/functions.nf +++ b/modules/nf-core/modules/fastqc/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/fastqc/main.nf b/modules/nf-core/modules/fastqc/main.nf index 39c327b261..9f6cfc5538 100644 --- a/modules/nf-core/modules/fastqc/main.nf +++ b/modules/nf-core/modules/fastqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,24 +24,31 @@ process FASTQC { output: tuple val(meta), path("*.html"), emit: html tuple val(meta), path("*.zip") , emit: zip - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: // Add soft-links to original FastQs for consistent naming in pipeline - def software = getSoftwareName(task.process) - def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" if (meta.single_end) { """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz - fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS """ } else { """ [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS """ } } diff --git a/modules/nf-core/modules/fastqc/meta.yml b/modules/nf-core/modules/fastqc/meta.yml index 8eb9953dce..b09553a3c3 100644 --- a/modules/nf-core/modules/fastqc/meta.yml +++ b/modules/nf-core/modules/fastqc/meta.yml @@ -15,6 +15,7 @@ tools: overrepresented sequences. homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ + licence: ['GPL-2.0-only'] input: - meta: type: map @@ -40,10 +41,10 @@ output: type: file description: FastQC report archive pattern: "*_{fastqc.zip}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@grst" diff --git a/modules/nf-core/modules/multiqc/functions.nf b/modules/nf-core/modules/multiqc/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/multiqc/functions.nf +++ b/modules/nf-core/modules/multiqc/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/multiqc/main.nf b/modules/nf-core/modules/multiqc/main.nf index 8b6d6f0c5e..0861aa5934 100644 --- a/modules/nf-core/modules/multiqc/main.nf +++ b/modules/nf-core/modules/multiqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,12 +24,15 @@ process MULTIQC { path "*multiqc_report.html", emit: report path "*_data" , emit: data path "*_plots" , optional:true, emit: plots - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) """ multiqc -f $options.args . - multiqc --version | sed -e "s/multiqc, version //g" > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( multiqc --version | sed -e "s/multiqc, version //g" ) + END_VERSIONS """ } diff --git a/modules/nf-core/modules/multiqc/meta.yml b/modules/nf-core/modules/multiqc/meta.yml index 532a8bb1ef..63c75a450a 100644 --- a/modules/nf-core/modules/multiqc/meta.yml +++ b/modules/nf-core/modules/multiqc/meta.yml @@ -11,6 +11,7 @@ tools: It's a general use tool, perfect for summarising the output from numerous bioinformatics tools. homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ + licence: ['GPL-3.0-or-later'] input: - multiqc_files: type: file @@ -29,10 +30,10 @@ output: type: file description: Plots created by MultiQC pattern: "*_data" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@abhi18av" - "@bunop" diff --git a/modules/nf-core/modules/samtools/faidx/functions.nf b/modules/nf-core/modules/samtools/faidx/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/samtools/faidx/functions.nf +++ b/modules/nf-core/modules/samtools/faidx/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/samtools/faidx/main.nf b/modules/nf-core/modules/samtools/faidx/main.nf index a89ff2bbb6..8070808426 100644 --- a/modules/nf-core/modules/samtools/faidx/main.nf +++ b/modules/nf-core/modules/samtools/faidx/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -11,24 +11,26 @@ process SAMTOOLS_FAIDX { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:[:], publish_by_meta:[]) } - conda (params.enable_conda ? 'bioconda::samtools=1.13' : null) + conda (params.enable_conda ? "bioconda::samtools=1.14" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/samtools:1.13--h8c37831_0" + container "https://depot.galaxyproject.org/singularity/samtools:1.14--hb421002_0" } else { - container "quay.io/biocontainers/samtools:1.13--h8c37831_0" + container "quay.io/biocontainers/samtools:1.14--hb421002_0" } input: path fasta output: - path "*.fai" , emit: fai - path "*.version.txt", emit: version + path "*.fai" , emit: fai + path "versions.yml", emit: versions script: - def software = getSoftwareName(task.process) """ samtools faidx $fasta - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/samtools/faidx/meta.yml b/modules/nf-core/modules/samtools/faidx/meta.yml index f92234d02a..16c0b33471 100644 --- a/modules/nf-core/modules/samtools/faidx/meta.yml +++ b/modules/nf-core/modules/samtools/faidx/meta.yml @@ -12,6 +12,7 @@ tools: homepage: http://www.htslib.org/ documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 + licence: ['MIT'] input: - fasta: type: file @@ -22,10 +23,10 @@ output: type: file description: FASTA index file pattern: "*.{fai}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@ewels" diff --git a/modules/nf-core/modules/samtools/merge/functions.nf b/modules/nf-core/modules/samtools/merge/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/samtools/merge/functions.nf +++ b/modules/nf-core/modules/samtools/merge/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/samtools/merge/main.nf b/modules/nf-core/modules/samtools/merge/main.nf index 0182b9fd52..ab641bb90b 100644 --- a/modules/nf-core/modules/samtools/merge/main.nf +++ b/modules/nf-core/modules/samtools/merge/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -11,25 +11,32 @@ process SAMTOOLS_MERGE { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - conda (params.enable_conda ? 'bioconda::samtools=1.13' : null) + conda (params.enable_conda ? "bioconda::samtools=1.14" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/samtools:1.13--h8c37831_0" + container "https://depot.galaxyproject.org/singularity/samtools:1.14--hb421002_0" } else { - container "quay.io/biocontainers/samtools:1.13--h8c37831_0" + container "quay.io/biocontainers/samtools:1.14--hb421002_0" } input: - tuple val(meta), path(bams) + tuple val(meta), path(input_files) + path fasta output: - tuple val(meta), path("${prefix}.bam"), emit: bam - path "*.version.txt" , emit: version + tuple val(meta), path("${prefix}.bam"), optional:true, emit: bam + tuple val(meta), path("${prefix}.cram"), optional:true, emit: cram + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def file_type = input_files[0].getExtension() + def reference = fasta ? "--reference ${fasta}" : "" """ - samtools merge ${prefix}.bam $bams - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt + samtools merge --threads ${task.cpus-1} $options.args ${reference} ${prefix}.${file_type} $input_files + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/samtools/merge/meta.yml b/modules/nf-core/modules/samtools/merge/meta.yml index 9092f22e50..2576a3a355 100644 --- a/modules/nf-core/modules/samtools/merge/meta.yml +++ b/modules/nf-core/modules/samtools/merge/meta.yml @@ -1,5 +1,5 @@ name: samtools_merge -description: Merge BAM file +description: Merge BAM or CRAM file keywords: - merge - bam @@ -14,31 +14,41 @@ tools: homepage: http://www.htslib.org/ documentation: hhttp://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 + licence: ['MIT'] input: - meta: type: map description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - bam: + - input_files: type: file - description: BAM file + description: BAM/CRAM file pattern: "*.{bam,cram,sam}" + - fasta: + type: optional file + description: Reference file the CRAM was created with + pattern: "*.{fasta,fa}" output: - meta: type: map description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - merged_bam: + - bam: type: file description: BAM file pattern: "*.{bam}" - - version: + - cram: + type: file + description: CRAM file + pattern: "*.{cram}" + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@yuukiiwa " - "@maxulysse" + - "@FriederikeHanssen" diff --git a/modules/nf-core/modules/seqkit/split2/functions.nf b/modules/nf-core/modules/seqkit/split2/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/seqkit/split2/functions.nf +++ b/modules/nf-core/modules/seqkit/split2/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/seqkit/split2/main.nf b/modules/nf-core/modules/seqkit/split2/main.nf index 5eeb0ad08d..80f55bb633 100644 --- a/modules/nf-core/modules/seqkit/split2/main.nf +++ b/modules/nf-core/modules/seqkit/split2/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -7,13 +7,11 @@ options = initOptions(params.options) process SEQKIT_SPLIT2 { tag "$meta.id" label 'process_medium' - publishDir "${params.outdir}", mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } conda (params.enable_conda ? 'bioconda::seqkit=0.16.1' : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { container "https://depot.galaxyproject.org/singularity/seqkit:0.16.1--h9ee0642_0" } else { @@ -24,36 +22,39 @@ process SEQKIT_SPLIT2 { tuple val(meta), path(reads) output: - tuple val(meta), path("*.split/*.gz"), emit: reads - path("*.version.txt") , emit: version - + tuple val(meta), path("*${prefix}/*.gz"), emit: reads + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) - def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" - + prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" if(meta.single_end){ - """ - seqkit \ - split2 \ - $options.args \ - --threads $task.cpus \ - -1 ${reads} \ - --out-dir ${prefix}.split - - echo \$(seqkit --version 2>&1) | sed 's/^.*seqkit //; s/Using.*\$//' > ${software}.version.txt - """ + """ + seqkit \\ + split2 \\ + $options.args \\ + --threads $task.cpus \\ + -1 $reads \\ + --out-dir $prefix + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(seqkit 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + END_VERSIONS + """ } else { - """ - seqkit \ - split2 \ - $options.args \ - --threads $task.cpus \ - -1 ${reads[0]} \ - -2 ${reads[1]} \ - --out-dir ${prefix}.split - - echo \$(seqkit --version 2>&1) | sed 's/^.*seqkit //; s/Using.*\$//' > ${software}.version.txt - """ + """ + seqkit \\ + split2 \\ + $options.args \\ + --threads $task.cpus \\ + -1 ${reads[0]} \\ + -2 ${reads[1]} \\ + --out-dir $prefix + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(seqkit 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + END_VERSIONS + """ } } diff --git a/modules/nf-core/modules/seqkit/split2/meta.yml b/modules/nf-core/modules/seqkit/split2/meta.yml index 44ae4ea79e..90eec7f9a5 100644 --- a/modules/nf-core/modules/seqkit/split2/meta.yml +++ b/modules/nf-core/modules/seqkit/split2/meta.yml @@ -10,6 +10,7 @@ tools: homepage: https://github.com/shenwei356/seqkit documentation: https://bioinf.shenwei.me/seqkit/ doi: 10.1371/journal.pone.0163962 + licence: ['MIT'] input: - meta: type: map @@ -30,9 +31,9 @@ output: type: file description: Split fastq files pattern: "*.{fq.gz/fastq.gz}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@FriederikeHanssen" diff --git a/modules/nf-core/modules/snpeff/functions.nf b/modules/nf-core/modules/snpeff/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/snpeff/functions.nf +++ b/modules/nf-core/modules/snpeff/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/snpeff/main.nf b/modules/nf-core/modules/snpeff/main.nf index af0fd816f7..3a1f6a52ae 100644 --- a/modules/nf-core/modules/snpeff/main.nf +++ b/modules/nf-core/modules/snpeff/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -31,10 +31,9 @@ process SNPEFF { output: tuple val(meta), path("*.ann.vcf"), emit: vcf path "*.csv" , emit: report - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) def avail_mem = 6 if (!task.memory) { log.info '[snpEff] Available memory not known - defaulting to 6GB. Specify process memory requirements to change this.' @@ -52,6 +51,9 @@ process SNPEFF { $vcf \\ > ${prefix}.ann.vcf - echo \$(snpEff -version 2>&1) > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(snpEff -version 2>&1) | cut -f 2 -d ' ') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/snpeff/meta.yml b/modules/nf-core/modules/snpeff/meta.yml index 7ba62cde08..8ba1868364 100644 --- a/modules/nf-core/modules/snpeff/meta.yml +++ b/modules/nf-core/modules/snpeff/meta.yml @@ -9,6 +9,7 @@ tools: It annotates and predicts the effects of genetic variants on genes and proteins (such as amino acid changes). homepage: https://pcingola.github.io/SnpEff/ documentation: https://pcingola.github.io/SnpEff/se_introduction/ + licence: ['MIT'] params: - use_cache: type: boolean @@ -49,9 +50,9 @@ output: type: file description: snpEff report file pattern: "*.html" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@maxulysse" diff --git a/modules/nf-core/modules/tabix/bgziptabix/functions.nf b/modules/nf-core/modules/tabix/bgziptabix/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/tabix/bgziptabix/functions.nf +++ b/modules/nf-core/modules/tabix/bgziptabix/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/tabix/bgziptabix/main.nf b/modules/nf-core/modules/tabix/bgziptabix/main.nf index 866a8bf85b..e44a722671 100644 --- a/modules/nf-core/modules/tabix/bgziptabix/main.nf +++ b/modules/nf-core/modules/tabix/bgziptabix/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -11,11 +11,11 @@ process TABIX_BGZIPTABIX { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - conda (params.enable_conda ? "bioconda::tabix=0.2.6" : null) + conda (params.enable_conda ? 'bioconda::tabix=1.11' : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/tabix:0.2.6--ha92aebf_0" + container "https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0" } else { - container "quay.io/biocontainers/tabix:0.2.6--ha92aebf_0" + container "quay.io/biocontainers/tabix:1.11--hdfd78af_0" } input: @@ -23,15 +23,17 @@ process TABIX_BGZIPTABIX { output: tuple val(meta), path("*.gz"), path("*.tbi"), emit: tbi - path "*.version.txt", emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" """ bgzip -c $options.args $input > ${prefix}.gz tabix $options.args2 ${prefix}.gz - echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/(.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/tabix/bgziptabix/meta.yml b/modules/nf-core/modules/tabix/bgziptabix/meta.yml index 5bef23502c..f2aed84d13 100644 --- a/modules/nf-core/modules/tabix/bgziptabix/meta.yml +++ b/modules/nf-core/modules/tabix/bgziptabix/meta.yml @@ -12,6 +12,7 @@ tools: homepage: https://www.htslib.org/doc/tabix.html documentation: https://www.htslib.org/doc/tabix.1.html doi: 10.1093/bioinformatics/btq671 + licence: ['MIT'] input: - meta: type: map @@ -36,9 +37,9 @@ output: type: file description: tabix index file pattern: "*.{gz.tbi}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@maxulysse" diff --git a/modules/nf-core/modules/tabix/tabix/functions.nf b/modules/nf-core/modules/tabix/tabix/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/tabix/tabix/functions.nf +++ b/modules/nf-core/modules/tabix/tabix/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/tabix/tabix/main.nf b/modules/nf-core/modules/tabix/tabix/main.nf index da23f535bc..1574c0b558 100644 --- a/modules/nf-core/modules/tabix/tabix/main.nf +++ b/modules/nf-core/modules/tabix/tabix/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -11,11 +11,11 @@ process TABIX_TABIX { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - conda (params.enable_conda ? "bioconda::tabix=0.2.6" : null) + conda (params.enable_conda ? 'bioconda::tabix=1.11' : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/tabix:0.2.6--ha92aebf_0" + container "https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0" } else { - container "quay.io/biocontainers/tabix:0.2.6--ha92aebf_0" + container "quay.io/biocontainers/tabix:1.11--hdfd78af_0" } input: @@ -23,13 +23,15 @@ process TABIX_TABIX { output: tuple val(meta), path("*.tbi"), emit: tbi - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) """ tabix $options.args $tab - echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/(.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/nf-core/modules/tabix/tabix/meta.yml b/modules/nf-core/modules/tabix/tabix/meta.yml index f66270dba5..2e37c4ff90 100644 --- a/modules/nf-core/modules/tabix/tabix/meta.yml +++ b/modules/nf-core/modules/tabix/tabix/meta.yml @@ -10,6 +10,7 @@ tools: homepage: https://www.htslib.org/doc/tabix.html documentation: https://www.htslib.org/doc/tabix.1.html doi: 10.1093/bioinformatics/btq671 + licence: ['MIT'] input: - meta: type: map @@ -30,10 +31,10 @@ output: type: file description: tabix index file pattern: "*.{tbi}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@joseespinosa" - "@drpatelh" diff --git a/modules/nf-core/modules/trimgalore/functions.nf b/modules/nf-core/modules/trimgalore/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/nf-core/modules/trimgalore/functions.nf +++ b/modules/nf-core/modules/trimgalore/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/nf-core/modules/trimgalore/main.nf b/modules/nf-core/modules/trimgalore/main.nf index 3c16d66fee..8e77f1f77f 100644 --- a/modules/nf-core/modules/trimgalore/main.nf +++ b/modules/nf-core/modules/trimgalore/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,7 +24,7 @@ process TRIMGALORE { output: tuple val(meta), path("*.fq.gz") , emit: reads tuple val(meta), path("*report.txt"), emit: log - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions tuple val(meta), path("*.html"), emit: html optional true tuple val(meta), path("*.zip") , emit: zip optional true @@ -48,7 +48,6 @@ process TRIMGALORE { def tpc_r2 = params.three_prime_clip_r2 > 0 ? "--three_prime_clip_r2 ${params.three_prime_clip_r2}" : '' // Added soft-links to original fastqs for consistent naming in MultiQC - def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" if (meta.single_end) { """ @@ -60,7 +59,11 @@ process TRIMGALORE { $c_r1 \\ $tpc_r1 \\ ${prefix}.fastq.gz - echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//') + cutadapt: \$(cutadapt --version) + END_VERSIONS """ } else { """ @@ -77,7 +80,11 @@ process TRIMGALORE { $tpc_r2 \\ ${prefix}_1.fastq.gz \\ ${prefix}_2.fastq.gz - echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//') + cutadapt: \$(cutadapt --version) + END_VERSIONS """ } } diff --git a/modules/nf-core/modules/trimgalore/meta.yml b/modules/nf-core/modules/trimgalore/meta.yml index 735387076c..c7e1df1de7 100644 --- a/modules/nf-core/modules/trimgalore/meta.yml +++ b/modules/nf-core/modules/trimgalore/meta.yml @@ -13,6 +13,7 @@ tools: MspI-digested RRBS-type (Reduced Representation Bisufite-Seq) libraries. homepage: https://www.bioinformatics.babraham.ac.uk/projects/trim_galore/ documentation: https://github.com/FelixKrueger/TrimGalore/blob/master/Docs/Trim_Galore_User_Guide.md + licence: ['GPL-3.0-or-later'] input: - meta: type: map @@ -48,10 +49,10 @@ output: type: file description: Trim Galore! trimming report pattern: "*_{report.txt}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@ewels" diff --git a/subworkflows/nf-core/fastqc_trimgalore.nf b/subworkflows/nf-core/fastqc_trimgalore.nf index 12360ab980..576f5f4157 100644 --- a/subworkflows/nf-core/fastqc_trimgalore.nf +++ b/subworkflows/nf-core/fastqc_trimgalore.nf @@ -22,7 +22,7 @@ workflow FASTQC_TRIMGALORE { FASTQC(reads) fastqc_html = FASTQC.out.html fastqc_zip = FASTQC.out.zip - fastqc_version = FASTQC.out.version.first() + fastqc_version = FASTQC.out.versions.first() } trim_reads = reads @@ -36,7 +36,7 @@ workflow FASTQC_TRIMGALORE { trim_html = TRIMGALORE.out.html trim_zip = TRIMGALORE.out.zip trim_log = TRIMGALORE.out.log - trimgalore_version = TRIMGALORE.out.version.first() + trimgalore_version = TRIMGALORE.out.versions.first() } emit: @@ -44,10 +44,10 @@ workflow FASTQC_TRIMGALORE { fastqc_html // channel: [ val(meta), [ html ] ] fastqc_zip // channel: [ val(meta), [ zip ] ] - fastqc_version // path: *.version.txt + fastqc_version // path: *.versions.txt trim_html // channel: [ val(meta), [ html ] ] trim_zip // channel: [ val(meta), [ zip ] ] trim_log // channel: [ val(meta), [ txt ] ] - trimgalore_version // path: *.version.txt + trimgalore_version // path: *.versions.txt } From a105909d7b55e8ce32953666dd605a93cfee27e1 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 10:21:27 +0100 Subject: [PATCH 15/28] fix: update syntax for samtools/merge --- subworkflows/nf-core/mapping.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/mapping.nf b/subworkflows/nf-core/mapping.nf index cc48ed4f24..ff6dde33a8 100644 --- a/subworkflows/nf-core/mapping.nf +++ b/subworkflows/nf-core/mapping.nf @@ -111,7 +111,7 @@ workflow MAPPING { multiple: it[1].size() > 1 }.set{bam_to_merge} - SAMTOOLS_MERGE(bam_to_merge.multiple) + SAMTOOLS_MERGE(bam_to_merge.multiple, []) bam_merged = bam_to_merge.single.mix(SAMTOOLS_MERGE.out.bam) SAMTOOLS_INDEX(bam_merged) From 8d37904a8f38d1a793b5cbebfd4411fbe98a5477 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 10:27:31 +0100 Subject: [PATCH 16/28] fix: update output for annotation subworkflows --- subworkflows/nf-core/ensemblvep_annotate.nf | 2 +- subworkflows/nf-core/snpeff_annotate.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/ensemblvep_annotate.nf b/subworkflows/nf-core/ensemblvep_annotate.nf index a5c043a818..a1e45771f9 100644 --- a/subworkflows/nf-core/ensemblvep_annotate.nf +++ b/subworkflows/nf-core/ensemblvep_annotate.nf @@ -31,5 +31,5 @@ workflow ENSEMBLVEP_ANNOTATE { emit: vcf = TABIX_BGZIPTABIX.out.tbi // channel: [ val(meta), vcf, tbi ] vep_report = ENSEMBLVEP.out.report // path: *.html - vep_version = ENSEMBLVEP.out.version // path: *.version.txt + vep_version = ENSEMBLVEP.out.versions // path: *.version.txt } diff --git a/subworkflows/nf-core/snpeff_annotate.nf b/subworkflows/nf-core/snpeff_annotate.nf index 577cdefb69..dd42b5e1e5 100644 --- a/subworkflows/nf-core/snpeff_annotate.nf +++ b/subworkflows/nf-core/snpeff_annotate.nf @@ -29,5 +29,5 @@ workflow SNPEFF_ANNOTATE { emit: vcf = TABIX_BGZIPTABIX.out.tbi // channel: [ val(meta), vcf, tbi ] snpeff_report = SNPEFF.out.report // path: *.html - snpeff_version = SNPEFF.out.version // path: *.version.txt + snpeff_version = SNPEFF.out.versions // path: *.version.txt } From 9aec3fff63e27de44ee63aef373aac876f6b920b Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 11:50:13 +0100 Subject: [PATCH 17/28] feat: update subworkflows --- .../local/gatk4/baserecalibrator/functions.nf | 54 ++++--- modules/local/gatk4/baserecalibrator/main.nf | 2 +- modules/local/samtools/view/main.nf | 2 +- .../{build_indices.nf => prepare_genome.nf} | 153 +++++++++--------- subworkflows/nf-core/markduplicates.nf | 6 +- subworkflows/nf-core/prepare_recalibration.nf | 15 +- workflows/sarek.nf | 44 +++-- 7 files changed, 145 insertions(+), 131 deletions(-) rename subworkflows/local/{build_indices.nf => prepare_genome.nf} (53%) diff --git a/modules/local/gatk4/baserecalibrator/functions.nf b/modules/local/gatk4/baserecalibrator/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/baserecalibrator/functions.nf +++ b/modules/local/gatk4/baserecalibrator/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/baserecalibrator/main.nf b/modules/local/gatk4/baserecalibrator/main.nf index b81d81a7a7..2844fdc344 100644 --- a/modules/local/gatk4/baserecalibrator/main.nf +++ b/modules/local/gatk4/baserecalibrator/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) diff --git a/modules/local/samtools/view/main.nf b/modules/local/samtools/view/main.nf index f97c939060..8f3ad0e0be 100644 --- a/modules/local/samtools/view/main.nf +++ b/modules/local/samtools/view/main.nf @@ -25,7 +25,7 @@ process SAMTOOLS_VIEW { path(fai) output: - tuple val(meta), path("*.cram"), path("*.crai"), emit: cram + tuple val(meta), path("*.cram"), path("*.crai"), emit: cram_crai path "*.version.txt" , emit: version script: diff --git a/subworkflows/local/build_indices.nf b/subworkflows/local/prepare_genome.nf similarity index 53% rename from subworkflows/local/build_indices.nf rename to subworkflows/local/prepare_genome.nf index 007ee8cf90..9fe8ec28c4 100644 --- a/subworkflows/local/build_indices.nf +++ b/subworkflows/local/prepare_genome.nf @@ -30,7 +30,7 @@ include { TABIX_TABIX as TABIX_GERMLINE_RESOURCE } from '../../modules/nf-core/m include { TABIX_TABIX as TABIX_KNOWN_INDELS } from '../../modules/nf-core/modules/tabix/tabix/main' addParams(options: params.tabix_known_indels_options) include { TABIX_TABIX as TABIX_PON } from '../../modules/nf-core/modules/tabix/tabix/main' addParams(options: params.tabix_pon_options) -workflow BUILD_INDICES { +workflow PREPARE_GENOME { take: dbsnp // channel: [optional] dbsnp fasta // channel: [mandatory] fasta @@ -38,87 +38,95 @@ workflow BUILD_INDICES { germline_resource // channel: [optional] germline_resource known_indels // channel: [optional] known_indels pon // channel: [optional] pon - target_bed // channel: [optionnal] target_bed - tools - step + target_bed // channel: [optional] target_bed + tools // value: [mandatory] tools + step // value: [mandatory] step main: - result_bwa = Channel.empty() - version_bwa = Channel.empty() - if (!(params.bwa) && 'mapping' in step) - if (params.aligner == "bwa-mem") (result_bwa, version_bwa) = BWAMEM1_INDEX(fasta) - else (result_bwa, version_bwa) = BWAMEM2_INDEX(fasta) - - result_dict = Channel.empty() - version_gatk = Channel.empty() - if (!(params.dict) && !('annotate' in step) && !('controlfreec' in step)) - (result_dict, version_gatk) = GATK4_CREATESEQUENCEDICTIONARY(fasta) - - result_fai = Channel.empty() - version_samtools = Channel.empty() - if (fasta_fai) result_fai = fasta_fai - if (!(params.fasta_fai) && !('annotate' in step)) - (result_fai, version_samtools) = SAMTOOLS_FAIDX(fasta) - - result_target_bed = Channel.empty() - version_target_bed = Channel.empty() + ch_versions = Channel.empty() + + ch_bwa = Channel.empty() + if (!(params.bwa) && 'mapping' in step) { + if (params.aligner == "bwa-mem") { + BWAMEM1_INDEX(fasta) + ch_bwa = BWAMEM1_INDEX.out.index + ch_versions = ch_versions.mix(BWAMEM1_INDEX.out.versions) + } else if (params.aligner == "bwa-mem2") { + BWAMEM2_INDEX(fasta) + ch_bwa = BWAMEM2_INDEX.out.index + ch_versions = ch_versions.mix(BWAMEM2_INDEX.out.versions) + } + } + + ch_dict = Channel.empty() + if (!(params.dict) && !('annotate' in step) && !('controlfreec' in step)) { + GATK4_CREATESEQUENCEDICTIONARY(fasta) + ch_dict = GATK4_CREATESEQUENCEDICTIONARY.out.dict + ch_versions = ch_versions.mix(GATK4_CREATESEQUENCEDICTIONARY.out.versions) + } + + ch_fasta_fai = Channel.empty() + if (fasta_fai) ch_fasta_fai = fasta_fai + if (!(params.fasta_fai) && !('annotate' in step)) { + SAMTOOLS_FAIDX(fasta) + ch_fasta_fai = SAMTOOLS_FAIDX.out.fai + ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) + } + + ch_target_bed = Channel.empty() if ((params.target_bed) && ('manta' in tools || 'strelka' in tools)) { - target_bed_id = target_bed.map{ it -> [[id:it[0].getName()], it] } - (result_target_bed, version_target_bed) = TABIX_BGZIPTABIX(target_bed_id) - result_target_bed = result_target_bed.map{ meta, bed, tbi -> [bed, tbi] } + TABIX_BGZIPTABIX(target_bed.map{ it -> [[id:it[0].getName()], it] }) + ch_target_bed = TABIX_BGZIPTABIX.out.tbi.map{ meta, bed, tbi -> [bed, tbi] } + ch_versions = ch_versions.mix(TABIX_BGZIPTABIX.out.versions) } - result_dbsnp_tbi = Channel.empty() - version_dbsnp_tbi = Channel.empty() + ch_dbsnp_tbi = Channel.empty() if (!(params.dbsnp_tbi) && params.dbsnp && ('mapping' in step || 'preparerecalibration' in step || 'controlfreec' in tools || 'haplotypecaller' in tools|| 'mutect2' in tools || 'tnscope' in tools)) { - dbsnp_id = dbsnp.map{ it -> [[id:it[0].baseName], it] } - (result_dbsnp_tbi, version_dbsnp_tbi) = TABIX_DBSNP(dbsnp_id) - result_dbsnp_tbi = result_dbsnp_tbi.map{ meta, tbi -> [tbi] } + TABIX_DBSNP(dbsnp.map{ it -> [[id:it[0].baseName], it] }) + ch_dbsnp_tbi = TABIX_DBSNP.out.tbi.map{ meta, tbi -> [tbi] } + ch_versions = ch_versions.mix(TABIX_DBSNP.out.versions) } - result_germline_resource_tbi = Channel.empty() - version_germline_resource_tbi = Channel.empty() + ch_germline_resource_tbi = Channel.empty() if (!(params.germline_resource_tbi) && params.germline_resource && 'mutect2' in tools) { - germline_resource_id = germline_resource.map{ it -> [[id:it[0].baseName], it] } - (result_germline_resource_tbi, version_germline_resource_tbi) = TABIX_GERMLINE_RESOURCE(germline_resource_id) + TABIX_GERMLINE_RESOURCE(germline_resource.map{ it -> [[id:it[0].baseName], it] }) + ch_germline_resource_tbi = TABIX_GERMLINE_RESOURCE.out.tbi.map{ meta, tbi -> [tbi] } + ch_versions = ch_versions.mix(TABIX_GERMLINE_RESOURCE.out.versions) } - result_known_indels_tbi = Channel.empty() - version_known_indels_tbi = Channel.empty() + ch_known_indels_tbi = Channel.empty() if (!(params.known_indels_tbi) && params.known_indels && ('mapping' in step || 'preparerecalibration' in step)) { - known_indels_id = known_indels.map{ it -> [[id:it[0].baseName], it] } - (result_known_indels_tbi, version_known_indels_tbi) = TABIX_KNOWN_INDELS(known_indels_id) - result_known_indels_tbi = result_known_indels_tbi.map{ meta, tbi -> [tbi] } + TABIX_KNOWN_INDELS(known_indels.map{ it -> [[id:it[0].baseName], it] }) + ch_known_indels_tbi = TABIX_KNOWN_INDELS.out.tbi.map{ meta, tbi -> [tbi] } + ch_versions = ch_versions.mix(TABIX_KNOWN_INDELS.out.versions) } - result_pon_tbi = Channel.empty() - version_pon_tbi = Channel.empty() + ch_pon_tbi = Channel.empty() if (!(params.pon_tbi) && params.pon && ('tnscope' in tools || 'mutect2' in tools)) { - pon_id = pon.map{ it -> [[id:it[0].baseName], it] } - (result_pon_tbi, version_pon_tbi) = TABIX_PON(pon_id) + TABIX_PON(pon.map{ it -> [[id:it[0].baseName], it] }) + ch_pon_tbi = TABIX_PON.out.tbi.map{ meta, tbi -> [tbi] } + ch_versions = ch_versions.mix(TABIX_PON.out.versions) } - version_tabix = Channel.empty() - version_tabix = version_tabix.mix(version_target_bed, version_dbsnp_tbi, version_germline_resource_tbi, version_known_indels_tbi, version_pon_tbi).first() - - result_msisensorpro_scan = Channel.empty() - version_msisensorpro_scan = Channel.empty() - if ('msisensorpro' in tools) - (result_msisensorpro_scan, version_msisensorpro_scan) = MSISENSORPRO_SCAN(fasta) + ch_msisensorpro_scan = Channel.empty() + if ('msisensorpro' in tools) { + MSISENSORPRO_SCAN(fasta) + ch_msisensorpro_scan = MSISENSORPRO_SCAN.out.list + ch_versions = ch_versions.mix(MSISENSORPRO_SCAN.out.versions) + } - result_intervals = Channel.empty() + ch_intervals = Channel.empty() if (params.no_intervals) { file("${params.outdir}/no_intervals.bed").text = "no_intervals\n" - result_intervals = Channel.fromPath(file("${params.outdir}/no_intervals.bed")) - } else if (!('annotate' in step) && !('controlfreec' in step)) - if (!params.intervals) - result_intervals = CREATE_INTERVALS_BED(BUILD_INTERVALS(result_fai)) - else - result_intervals = CREATE_INTERVALS_BED(file(params.intervals)) + ch_intervals = Channel.fromPath(file("${params.outdir}/no_intervals.bed")) + } else if (!('annotate' in step) && !('controlfreec' in step)) { + if (!params.intervals) ch_intervals = CREATE_INTERVALS_BED(BUILD_INTERVALS(ch_fasta_fai)) + else ch_intervals = CREATE_INTERVALS_BED(file(params.intervals)) + } if (!params.no_intervals) { - result_intervals = result_intervals.flatten() + ch_intervals = ch_intervals.flatten() .map{ intervalFile -> def duration = 0.0 for (line in intervalFile.readLines()) { @@ -137,19 +145,16 @@ workflow BUILD_INDICES { } emit: - bwa = result_bwa - bwa_version = version_bwa - dbsnp_tbi = result_dbsnp_tbi - dict = result_dict - fai = result_fai - gatk_version = version_gatk - germline_resource_tbi = result_germline_resource_tbi - intervals = result_intervals - known_indels_tbi = result_known_indels_tbi.collect() - msisensorpro_scan = result_msisensorpro_scan - msisensorpro_scan_version = version_msisensorpro_scan - pon_tbi = result_pon_tbi - samtools_version = version_samtools - tabix_version = version_tabix - target_bed_gz_tbi = result_target_bed + bwa = ch_bwa // path: {bwa,bwamem2}/index + dbsnp_tbi = ch_dbsnp_tbi // path: dbsnb.vcf.gz.tbi + dict = ch_dict // path: genome.fasta.dict + fasta_fai = ch_fasta_fai // path: genome.fasta.fai + germline_resource_tbi = ch_germline_resource_tbi // path: germline_resource.vcf.gz.tbi + intervals = ch_intervals // path: intervals.bed + known_indels_tbi = ch_known_indels_tbi.collect() // path: {known_indels*}.vcf.gz.tbi + msisensorpro_scan = ch_msisensorpro_scan // path: genome_msi.list + pon_tbi = ch_pon_tbi // path: pon.vcf.gz.tbi + target_bed_gz_tbi = ch_target_bed // path: target.bed.gz.tbi + + versions = ch_versions.ifEmpty(null) // channel: [ versions.yml ] } diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index 9dff6b0994..6d8c2b1c67 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -41,7 +41,7 @@ workflow MARKDUPLICATES { if (skip_markduplicates) { bam_markduplicates = bam_indexed SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fai) - cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram + cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram_crai } else { if (use_gatk_spark) { //If BAMQC should be run on MD output, then don't use MDSpark to convert to cram, but use bam output instead @@ -51,7 +51,7 @@ workflow MARKDUPLICATES { bam_markduplicates = GATK4_MARKDUPLICATES_SPARK.out.output.join(SAMTOOLS_INDEX.out.bai) SAMTOOLS_BAM_TO_CRAM_SPARK(bam_markduplicates, fasta, fai) - cram_markduplicates = SAMTOOLS_BAM_TO_CRAM_SPARK.out.cram + cram_markduplicates = SAMTOOLS_BAM_TO_CRAM_SPARK.out.cram_crai } else { GATK4_MARKDUPLICATES_SPARK(bam_mapped, fasta, fai, dict, "cram") SAMTOOLS_INDEX(GATK4_MARKDUPLICATES_SPARK.out.output) @@ -69,7 +69,7 @@ workflow MARKDUPLICATES { bam_markduplicates = GATK4_MARKDUPLICATES.out.bam_bai SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fai) - cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram + cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram_crai } } diff --git a/subworkflows/nf-core/prepare_recalibration.nf b/subworkflows/nf-core/prepare_recalibration.nf index 18183f8e18..343d988699 100644 --- a/subworkflows/nf-core/prepare_recalibration.nf +++ b/subworkflows/nf-core/prepare_recalibration.nf @@ -6,17 +6,17 @@ params.baserecalibrator_options = [:] params.gatherbqsrreports_options = [:] params.baserecalibrator_spark_options = [:] -include { GATK4_BASERECALIBRATOR as BASERECALIBRATOR } from '../../modules/local/gatk4/baserecalibrator/main' addParams(options: params.baserecalibrator_options) -include { GATK4_BASERECALIBRATOR_SPARK as BASERECALIBRATOR_SPARK } from '../../modules/local/gatk4/baserecalibratorspark/main' addParams(options: params.baserecalibrator_spark_options) -include { GATK4_GATHERBQSRREPORTS as GATHERBQSRREPORTS } from '../../modules/local/gatk4/gatherbqsrreports/main' addParams(options: params.gatherbqsrreports_options) +include { GATK4_BASERECALIBRATOR as BASERECALIBRATOR } from '../../modules/local/gatk4/baserecalibrator/main' addParams(options: params.baserecalibrator_options) +include { GATK4_BASERECALIBRATOR_SPARK as BASERECALIBRATOR_SPARK } from '../../modules/local/gatk4/baserecalibratorspark/main' addParams(options: params.baserecalibrator_spark_options) +include { GATK4_GATHERBQSRREPORTS as GATHERBQSRREPORTS } from '../../modules/local/gatk4/gatherbqsrreports/main' addParams(options: params.gatherbqsrreports_options) workflow PREPARE_RECALIBRATION { take: cram_markduplicates // channel: [mandatory] cram_markduplicates use_gatk_spark // value: [mandatory] use gatk spark dict // channel: [mandatory] dict - fai // channel: [mandatory] fai fasta // channel: [mandatory] fasta + fasta_fai // channel: [mandatory] fasta_fai intervals // channel: [mandatory] intervals num_intervals known_sites // channel: [optional] known_sites @@ -25,6 +25,8 @@ workflow PREPARE_RECALIBRATION { main: + ch_versions = Channel.empty() + cram_markduplicates.combine(intervals) .map{ meta, cram, crai, intervals -> new_meta = meta.clone() @@ -33,10 +35,10 @@ workflow PREPARE_RECALIBRATION { }.set{cram_markduplicates_intervals} if (use_gatk_spark) { - BASERECALIBRATOR_SPARK(cram_markduplicates_intervals, fasta, fai, dict, known_sites, known_sites_tbi) + BASERECALIBRATOR_SPARK(cram_markduplicates_intervals, fasta, fasta_fai, dict, known_sites, known_sites_tbi) table_baserecalibrator = BASERECALIBRATOR_SPARK.out.table } else { - BASERECALIBRATOR(cram_markduplicates_intervals, fasta, fai, dict, known_sites, known_sites_tbi) + BASERECALIBRATOR(cram_markduplicates_intervals, fasta, fasta_fai, dict, known_sites, known_sites_tbi) table_baserecalibrator = BASERECALIBRATOR.out.table } @@ -58,4 +60,5 @@ workflow PREPARE_RECALIBRATION { emit: table_bqsr = table_bqsr + versions = ch_versions // channel: [versions.yml] } diff --git a/workflows/sarek.nf b/workflows/sarek.nf index b722a5d7a4..9baf952307 100644 --- a/workflows/sarek.nf +++ b/workflows/sarek.nf @@ -145,7 +145,7 @@ include { PREPARE_RECALIBRATION_CSV } from '../subworkflows/local/prepare_recali include { RECALIBRATE_CSV } from '../subworkflows/local/recalibrate_csv' // Build indices if needed -include { BUILD_INDICES } from '../subworkflows/local/build_indices' addParams( +include { PREPARE_GENOME } from '../subworkflows/local/prepare_genome' addParams( bgziptabix_target_bed_options: modules['bgziptabix_target_bed'], build_intervals_options: modules['build_intervals'], bwa_index_options: modules['bwa_index'], @@ -268,7 +268,7 @@ workflow SAREK { qc_reports = Channel.empty() // Build indices if needed - BUILD_INDICES( + PREPARE_GENOME( dbsnp, fasta, params.fasta_fai, @@ -280,15 +280,15 @@ workflow SAREK { step) // Gather built indices or get them from the params - bwa = params.fasta ? params.bwa ? Channel.fromPath(params.bwa).collect() : BUILD_INDICES.out.bwa : ch_dummy_file - dict = params.fasta ? params.dict ? Channel.fromPath(params.dict).collect() : BUILD_INDICES.out.dict : ch_dummy_file - fai = params.fasta ? params.fasta_fai ? Channel.fromPath(params.fasta_fai).collect() : BUILD_INDICES.out.fai : ch_dummy_file - dbsnp_tbi = params.dbsnp ? params.dbsnp_tbi ? Channel.fromPath(params.dbsnp_tbi).collect() : BUILD_INDICES.out.dbsnp_tbi : ch_dummy_file - germline_resource_tbi = params.germline_resource ? params.germline_resource_tbi ? Channel.fromPath(params.germline_resource_tbi).collect() : BUILD_INDICES.out.germline_resource_tbi : ch_dummy_file - known_indels_tbi = params.known_indels ? params.known_indels_tbi ? Channel.fromPath(params.known_indels_tbi).collect() : BUILD_INDICES.out.known_indels_tbi : ch_dummy_file - pon_tbi = params.pon ? params.pon_tbi ? Channel.fromPath(params.pon_tbi).collect() : BUILD_INDICES.out.pon_tbi : ch_dummy_file - msisensorpro_scan = BUILD_INDICES.out.msisensorpro_scan - target_bed_gz_tbi = BUILD_INDICES.out.target_bed_gz_tbi + bwa = params.fasta ? params.bwa ? Channel.fromPath(params.bwa).collect() : PREPARE_GENOME.out.bwa : ch_dummy_file + dict = params.fasta ? params.dict ? Channel.fromPath(params.dict).collect() : PREPARE_GENOME.out.dict : ch_dummy_file + fasta_fai = params.fasta ? params.fasta_fai ? Channel.fromPath(params.fasta_fai).collect() : PREPARE_GENOME.out.fasta_fai : ch_dummy_file + dbsnp_tbi = params.dbsnp ? params.dbsnp_tbi ? Channel.fromPath(params.dbsnp_tbi).collect() : PREPARE_GENOME.out.dbsnp_tbi : ch_dummy_file + germline_resource_tbi = params.germline_resource ? params.germline_resource_tbi ? Channel.fromPath(params.germline_resource_tbi).collect() : PREPARE_GENOME.out.germline_resource_tbi : ch_dummy_file + known_indels_tbi = params.known_indels ? params.known_indels_tbi ? Channel.fromPath(params.known_indels_tbi).collect() : PREPARE_GENOME.out.known_indels_tbi : ch_dummy_file + pon_tbi = params.pon ? params.pon_tbi ? Channel.fromPath(params.pon_tbi).collect() : PREPARE_GENOME.out.pon_tbi : ch_dummy_file + msisensorpro_scan = PREPARE_GENOME.out.msisensorpro_scan + target_bed_gz_tbi = PREPARE_GENOME.out.target_bed_gz_tbi //TODO @Rike, is this working for you? // known_sites is made by grouping both the dbsnp and the known indels ressources @@ -297,16 +297,12 @@ workflow SAREK { known_sites_tbi = dbsnp_tbi.concat(known_indels_tbi).collect() // Intervals for speed up preprocessing/variant calling by spread/gather - intervals = BUILD_INDICES.out.intervals + intervals = PREPARE_GENOME.out.intervals num_intervals = 0 intervals.count().map{ num_intervals = it } // Get versions from all software used - ch_software_versions = ch_software_versions.mix(BUILD_INDICES.out.bwa_version.ifEmpty(null)) - ch_software_versions = ch_software_versions.mix(BUILD_INDICES.out.gatk_version.ifEmpty(null)) - ch_software_versions = ch_software_versions.mix(BUILD_INDICES.out.samtools_version.ifEmpty(null)) - ch_software_versions = ch_software_versions.mix(BUILD_INDICES.out.msisensorpro_scan_version.ifEmpty(null)) - ch_software_versions = ch_software_versions.mix(BUILD_INDICES.out.tabix_version.ifEmpty(null)) + ch_software_versions = ch_software_versions.mix(PREPARE_GENOME.out.versions.ifEmpty(null)) // PREPROCESSING @@ -342,7 +338,7 @@ workflow SAREK { MAPPING( params.aligner, bwa, - fai, + fasta_fai, fasta, reads_input, params.skip_markduplicates, @@ -369,7 +365,7 @@ workflow SAREK { ('markduplicates' in params.use_gatk_spark), !('markduplicates' in skip_qc), fasta, - fai, + fasta_fai, dict, params.skip_markduplicates, ('bamqc' in skip_qc), @@ -388,8 +384,8 @@ workflow SAREK { cram_markduplicates, ('bqsr' in params.use_gatk_spark), dict, - fai, fasta, + fasta_fai, intervals, num_intervals, known_sites, @@ -412,7 +408,7 @@ workflow SAREK { ('samtools' in skip_qc), cram_applybqsr, dict, - fai, + fasta_fai, fasta, intervals, num_intervals, @@ -441,7 +437,7 @@ workflow SAREK { dbsnp, dbsnp_tbi, dict, - fai, + fasta_fai, fasta, intervals, num_intervals, @@ -459,7 +455,7 @@ workflow SAREK { // dbsnp, // dbsnp_tbi, // dict, - // fai, + // fasta_fai, // fasta, // intervals, // target_bed, @@ -472,7 +468,7 @@ workflow SAREK { // dbsnp, // dbsnp_tbi, // dict, - // fai, + // fasta_fai, // fasta, // intervals, // msisensorpro_scan, From df8fa7e9cb9a75e9f8d8932ed7228fa905cdfed3 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 12:15:30 +0100 Subject: [PATCH 18/28] fix: input files order more consistent --- subworkflows/local/germline_variant_calling.nf | 14 +++++++------- subworkflows/local/pair_variant_calling.nf | 10 +++++----- subworkflows/nf-core/mapping.nf | 2 +- subworkflows/nf-core/markduplicates.nf | 16 ++++++++-------- subworkflows/nf-core/recalibrate.nf | 8 ++++---- workflows/sarek.nf | 8 ++++---- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/subworkflows/local/germline_variant_calling.nf b/subworkflows/local/germline_variant_calling.nf index 6cc692c924..806e45ab87 100644 --- a/subworkflows/local/germline_variant_calling.nf +++ b/subworkflows/local/germline_variant_calling.nf @@ -23,8 +23,8 @@ workflow GERMLINE_VARIANT_CALLING { dbsnp // channel: [mandatory] dbsnp dbsnp_tbi // channel: [mandatory] dbsnp_tbi dict // channel: [mandatory] dict - fai // channel: [mandatory] fai fasta // channel: [mandatory] fasta + fasta_fai // channel: [mandatory] fasta_fai intervals // channel: [mandatory] intervals num_intervals target_bed // channel: [optional] target_bed @@ -54,7 +54,7 @@ workflow GERMLINE_VARIANT_CALLING { dbsnp_tbi, dict, fasta, - fai, + fasta_fai, no_intervals) haplotypecaller_raw = HAPLOTYPECALLER.out.vcf.map{ meta,vcf -> @@ -64,7 +64,7 @@ workflow GERMLINE_VARIANT_CALLING { CONCAT_GVCF( haplotypecaller_raw, - fai, + fasta_fai, target_bed) haplotypecaller_gvcf = CONCAT_GVCF.out.vcf @@ -77,7 +77,7 @@ workflow GERMLINE_VARIANT_CALLING { dbsnp_tbi, dict, fasta, - fai, + fasta_fai, no_intervals) haplotypecaller_results = GENOTYPEGVCF.out.vcf.map{ meta, vcf -> @@ -87,7 +87,7 @@ workflow GERMLINE_VARIANT_CALLING { CONCAT_HAPLOTYPECALLER( haplotypecaller_results, - fai, + fasta_fai, target_bed) haplotypecaller_vcf = CONCAT_HAPLOTYPECALLER.out.vcf @@ -98,7 +98,7 @@ workflow GERMLINE_VARIANT_CALLING { DEEPVARIANT( bam, fasta, - fai) + fasta_fai) deepvariant_vcf = DEEPVARIANT.out.vcf deepvariant_gvcf = DEEPVARIANT.out.gvcf @@ -108,7 +108,7 @@ workflow GERMLINE_VARIANT_CALLING { STRELKA( cram, fasta, - fai, + fasta_fai, target_bed_gz_tbi) strelka_vcf = STRELKA.out.vcf diff --git a/subworkflows/local/pair_variant_calling.nf b/subworkflows/local/pair_variant_calling.nf index 4dc4ab112f..dc14143d15 100644 --- a/subworkflows/local/pair_variant_calling.nf +++ b/subworkflows/local/pair_variant_calling.nf @@ -21,8 +21,8 @@ workflow PAIR_VARIANT_CALLING { dbsnp // channel: [mandatory] dbsnp dbsnp_tbi // channel: [mandatory] dbsnp_tbi dict // channel: [mandatory] dict - fai // channel: [mandatory] fai fasta // channel: [mandatory] fasta + fasta_fai // channel: [mandatory] fasta_fai intervals // channel: [mandatory] intervals msisensorpro_scan // channel: [optional] msisensorpro_scan target_bed // channel: [optional] target_bed @@ -73,7 +73,7 @@ workflow PAIR_VARIANT_CALLING { MANTA( cram_pair, fasta, - fai, + fasta_fai, target_bed_gz_tbi) manta_candidate_small_indels_vcf = MANTA.out.candidate_small_indels_vcf @@ -88,7 +88,7 @@ workflow PAIR_VARIANT_CALLING { STRELKA_BP( manta_csi_for_strelka_bp, fasta, - fai, + fasta_fai, target_bed_gz_tbi) strelka_indels_vcf = STRELKA_BP.out.indels_vcf @@ -108,7 +108,7 @@ workflow PAIR_VARIANT_CALLING { STRELKA( cram_pair, fasta, - fai, + fasta_fai, target_bed_gz_tbi) strelka_indels_vcf = STRELKA.out.indels_vcf @@ -125,7 +125,7 @@ workflow PAIR_VARIANT_CALLING { panel_of_normals_tbi, dict, fasta, - fai, + fasta_fai, no_intervals, germline_resource, germline_resource_tbi diff --git a/subworkflows/nf-core/mapping.nf b/subworkflows/nf-core/mapping.nf index ff6dde33a8..69d8099d0a 100644 --- a/subworkflows/nf-core/mapping.nf +++ b/subworkflows/nf-core/mapping.nf @@ -22,8 +22,8 @@ workflow MAPPING { take: aligner // string: [mandatory] "bwa-mem" or "bwa-mem2" bwa // channel: [mandatory] bwa - fai // channel: [mandatory] fai fasta // channel: [mandatory] fasta + fasta_fai // channel: [mandatory] fasta_fai reads_input // channel: [mandatory] meta, reads_input skip_markduplicates // boolean: true/false save_bam_mapped // boolean: true/false diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index 6d8c2b1c67..b3ee59d78a 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -26,9 +26,9 @@ workflow MARKDUPLICATES { bam_indexed // channel: [mandatory] meta, bam, bai use_gatk_spark // value: [mandatory] use gatk spark save_metrics // value: [mandatory] save metrics - fasta // channel: [mandatory] fasta - fai // channel: [mandatory] fai dict // channel: [mandatory] dict + fasta // channel: [mandatory] fasta + fasta_fai // channel: [mandatory] fasta_fai skip_markduplicates // boolean: true/false skip_bamqc // boolean: true/false skip_samtools // boolean: true/false @@ -40,26 +40,26 @@ workflow MARKDUPLICATES { if (skip_markduplicates) { bam_markduplicates = bam_indexed - SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fai) + SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fasta_fai) cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram_crai } else { if (use_gatk_spark) { //If BAMQC should be run on MD output, then don't use MDSpark to convert to cram, but use bam output instead if (!skip_bamqc) { - GATK4_MARKDUPLICATES_SPARK(bam_mapped, fasta, fai, dict, "bam") + GATK4_MARKDUPLICATES_SPARK(bam_mapped, fasta, fasta_fai, dict, "bam") SAMTOOLS_INDEX(GATK4_MARKDUPLICATES_SPARK.out.output) bam_markduplicates = GATK4_MARKDUPLICATES_SPARK.out.output.join(SAMTOOLS_INDEX.out.bai) - SAMTOOLS_BAM_TO_CRAM_SPARK(bam_markduplicates, fasta, fai) + SAMTOOLS_BAM_TO_CRAM_SPARK(bam_markduplicates, fasta, fasta_fai) cram_markduplicates = SAMTOOLS_BAM_TO_CRAM_SPARK.out.cram_crai } else { - GATK4_MARKDUPLICATES_SPARK(bam_mapped, fasta, fai, dict, "cram") + GATK4_MARKDUPLICATES_SPARK(bam_mapped, fasta, fasta_fai, dict, "cram") SAMTOOLS_INDEX(GATK4_MARKDUPLICATES_SPARK.out.output) cram_markduplicates = GATK4_MARKDUPLICATES_SPARK.out.output.join(SAMTOOLS_INDEX.out.crai) } if (save_metrics) { - GATK4_ESTIMATELIBRARYCOMPLEXITY(bam_mapped, fasta, fai, dict) + GATK4_ESTIMATELIBRARYCOMPLEXITY(bam_mapped, fasta, fasta_fai, dict) report_markduplicates = GATK4_ESTIMATELIBRARYCOMPLEXITY.out.metrics } @@ -68,7 +68,7 @@ workflow MARKDUPLICATES { report_markduplicates = GATK4_MARKDUPLICATES.out.metrics bam_markduplicates = GATK4_MARKDUPLICATES.out.bam_bai - SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fai) + SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fasta_fai) cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram_crai } } diff --git a/subworkflows/nf-core/recalibrate.nf b/subworkflows/nf-core/recalibrate.nf index 81f7923c92..f6a71111cc 100644 --- a/subworkflows/nf-core/recalibrate.nf +++ b/subworkflows/nf-core/recalibrate.nf @@ -23,8 +23,8 @@ workflow RECALIBRATE { skip_samtools // boolean: true/false cram // channel: [mandatory] cram dict // channel: [mandatory] dict - fai // channel: [mandatory] fai fasta // channel: [mandatory] fasta + fasta_fai // channel: [mandatory] fasta_fai intervals // channel: [mandatory] intervals num_intervals target_bed // channel: [optional] target_bed @@ -42,10 +42,10 @@ workflow RECALIBRATE { }.set{cram_intervals} if(use_gatk_spark){ - APPLYBQSR_SPARK(cram_intervals, fasta, fai, dict) + APPLYBQSR_SPARK(cram_intervals, fasta, fasta_fai, dict) cram_applybqsr = APPLYBQSR_SPARK.out.cram }else{ - APPLYBQSR(cram_intervals, fasta, fai, dict) + APPLYBQSR(cram_intervals, fasta, fasta_fai, dict) cram_applybqsr = APPLYBQSR.out.cram } @@ -68,7 +68,7 @@ workflow RECALIBRATE { samtools_stats = Channel.empty() if (!skip_bamqc) { - QUALIMAP_BAMQC_CRAM(cram_recalibrated_index, target_bed, params.target_bed,fasta, fai) + QUALIMAP_BAMQC_CRAM(cram_recalibrated_index, target_bed, params.target_bed,fasta, fasta_fai) qualimap_bamqc = QUALIMAP_BAMQC_CRAM.out.results } diff --git a/workflows/sarek.nf b/workflows/sarek.nf index fe5af06481..4160bc7563 100644 --- a/workflows/sarek.nf +++ b/workflows/sarek.nf @@ -338,8 +338,8 @@ workflow SAREK { MAPPING( params.aligner, bwa, - fasta_fai, fasta, + fasta_fai, reads_input, params.skip_markduplicates, save_bam_mapped) @@ -364,9 +364,9 @@ workflow SAREK { bam_indexed, ('markduplicates' in params.use_gatk_spark), !('markduplicates' in skip_qc), + dict, fasta, fasta_fai, - dict, params.skip_markduplicates, ('bamqc' in skip_qc), ('samtools' in skip_qc), @@ -444,8 +444,8 @@ workflow SAREK { dbsnp, dbsnp_tbi, dict, - fasta_fai, fasta, + fasta_fai, intervals, num_intervals, target_bed, @@ -462,8 +462,8 @@ workflow SAREK { // dbsnp, // dbsnp_tbi, // dict, - // fasta_fai, // fasta, + // fasta_fai, // intervals, // target_bed, // target_bed_gz_tbi) From e750ec228ae4964b23fa8f970f0ad6e1de69d2c9 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 12:32:05 +0100 Subject: [PATCH 19/28] install modules qualimap/bamqc --- modules.json | 3 + modules/local/qualimap/bamqc/functions.nf | 68 ---------------- .../modules/qualimap/bamqc/functions.nf | 78 +++++++++++++++++++ .../modules}/qualimap/bamqc/main.nf | 12 +-- .../modules}/qualimap/bamqc/meta.yml | 7 +- subworkflows/nf-core/markduplicates.nf | 7 +- 6 files changed, 96 insertions(+), 79 deletions(-) delete mode 100644 modules/local/qualimap/bamqc/functions.nf create mode 100644 modules/nf-core/modules/qualimap/bamqc/functions.nf rename modules/{local => nf-core/modules}/qualimap/bamqc/main.nf (81%) rename modules/{local => nf-core/modules}/qualimap/bamqc/meta.yml (91%) diff --git a/modules.json b/modules.json index 638b5823c3..e0a9b5c838 100644 --- a/modules.json +++ b/modules.json @@ -24,6 +24,9 @@ "multiqc": { "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, + "qualimap/bamqc": { + "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" + }, "samtools/faidx": { "git_sha": "19035c99d1cd8412b5046c6bb4af787658b84e5b" }, diff --git a/modules/local/qualimap/bamqc/functions.nf b/modules/local/qualimap/bamqc/functions.nf deleted file mode 100644 index da9da093d3..0000000000 --- a/modules/local/qualimap/bamqc/functions.nf +++ /dev/null @@ -1,68 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/modules/nf-core/modules/qualimap/bamqc/functions.nf b/modules/nf-core/modules/qualimap/bamqc/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/qualimap/bamqc/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/local/qualimap/bamqc/main.nf b/modules/nf-core/modules/qualimap/bamqc/main.nf similarity index 81% rename from modules/local/qualimap/bamqc/main.nf rename to modules/nf-core/modules/qualimap/bamqc/main.nf index 93cc76f6e1..d33f1e67bf 100644 --- a/modules/local/qualimap/bamqc/main.nf +++ b/modules/nf-core/modules/qualimap/bamqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -19,16 +19,15 @@ process QUALIMAP_BAMQC { } input: - tuple val(meta), path(bam), path(bai) + tuple val(meta), path(bam) path gff val use_gff output: tuple val(meta), path("${prefix}"), emit: results - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" def collect_pairs = meta.single_end ? '' : '--collect-overlap-pairs' @@ -56,6 +55,9 @@ process QUALIMAP_BAMQC { -outdir $prefix \\ -nt $task.cpus - echo \$(qualimap 2>&1) | sed 's/^.*QualiMap v.//; s/Built.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(qualimap 2>&1) | sed 's/^.*QualiMap v.//; s/Built.*\$//') + END_VERSIONS """ } diff --git a/modules/local/qualimap/bamqc/meta.yml b/modules/nf-core/modules/qualimap/bamqc/meta.yml similarity index 91% rename from modules/local/qualimap/bamqc/meta.yml rename to modules/nf-core/modules/qualimap/bamqc/meta.yml index 3c608f31a2..6888d30e8d 100644 --- a/modules/local/qualimap/bamqc/meta.yml +++ b/modules/nf-core/modules/qualimap/bamqc/meta.yml @@ -14,6 +14,7 @@ tools: homepage: http://qualimap.bioinfo.cipf.es/ documentation: http://qualimap.conesalab.org/doc_html/index.html doi: 10.1093/bioinformatics/bts503 + licence: ['GPL-2.0-only'] input: - meta: type: map @@ -41,9 +42,9 @@ output: type: dir description: Qualimap results dir pattern: "*/*" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@phue" diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index b3ee59d78a..a798748dde 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -14,7 +14,7 @@ params.samtools_view_options = [:] include { GATK4_ESTIMATELIBRARYCOMPLEXITY } from '../../modules/local/gatk4/estimatelibrarycomplexity/main' addParams(options: params.estimatelibrarycomplexity_options) include { GATK4_MARKDUPLICATES } from '../../modules/local/gatk4/markduplicates/main' addParams(options: params.markduplicates_options) include { GATK4_MARKDUPLICATES_SPARK } from '../../modules/local/gatk4/markduplicatesspark/main' addParams(options: params.markduplicatesspark_options) -include { QUALIMAP_BAMQC } from '../../modules/local/qualimap/bamqc/main' addParams(options: params.qualimap_bamqc_options) +include { QUALIMAP_BAMQC } from '../../modules/nf-core/modules/qualimap/bamqc/main' addParams(options: params.qualimap_bamqc_options) include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) include { SAMTOOLS_STATS } from '../../modules/local/samtools/stats/main' addParams(options: params.samtools_stats_options) include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM } from '../../modules/local/samtools/view/main' addParams(options: params.samtools_view_options) @@ -48,7 +48,7 @@ workflow MARKDUPLICATES { if (!skip_bamqc) { GATK4_MARKDUPLICATES_SPARK(bam_mapped, fasta, fasta_fai, dict, "bam") SAMTOOLS_INDEX(GATK4_MARKDUPLICATES_SPARK.out.output) - bam_markduplicates = GATK4_MARKDUPLICATES_SPARK.out.output.join(SAMTOOLS_INDEX.out.bai) + bam_markduplicates = GATK4_MARKDUPLICATES_SPARK.out.output.join(SAMTOOLS_INDEX.out.bai) SAMTOOLS_BAM_TO_CRAM_SPARK(bam_markduplicates, fasta, fasta_fai) cram_markduplicates = SAMTOOLS_BAM_TO_CRAM_SPARK.out.cram_crai @@ -84,7 +84,8 @@ workflow MARKDUPLICATES { qualimap_bamqc = Channel.empty() if (!skip_bamqc) { - QUALIMAP_BAMQC(bam_markduplicates, target_bed, params.target_bed) + bam_markduplicates_for_bamqc = bam_markduplicates.map { meta, bam, bai -> [meta, bam] } + QUALIMAP_BAMQC(bam_markduplicates_for_bamqc, target_bed, params.target_bed) qualimap_bamqc = QUALIMAP_BAMQC.out.results } From ad9683b658f6fec49b2ce854bfe1ccb099d58b40 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 12:38:36 +0100 Subject: [PATCH 20/28] feat: install modules samtools/stats --- modules.json | 3 + modules/local/samtools/stats/functions.nf | 68 ---------------- modules/local/samtools/stats/main.nf | 35 --------- .../modules/samtools/stats/functions.nf | 78 +++++++++++++++++++ .../nf-core/modules/samtools/stats/main.nf | 39 ++++++++++ .../modules}/samtools/stats/meta.yml | 28 ++++--- subworkflows/nf-core/markduplicates.nf | 2 +- subworkflows/nf-core/recalibrate.nf | 12 +-- 8 files changed, 144 insertions(+), 121 deletions(-) delete mode 100644 modules/local/samtools/stats/functions.nf delete mode 100644 modules/local/samtools/stats/main.nf create mode 100644 modules/nf-core/modules/samtools/stats/functions.nf create mode 100644 modules/nf-core/modules/samtools/stats/main.nf rename modules/{local => nf-core/modules}/samtools/stats/meta.yml (68%) diff --git a/modules.json b/modules.json index e0a9b5c838..7796312cb7 100644 --- a/modules.json +++ b/modules.json @@ -33,6 +33,9 @@ "samtools/merge": { "git_sha": "19035c99d1cd8412b5046c6bb4af787658b84e5b" }, + "samtools/stats": { + "git_sha": "19035c99d1cd8412b5046c6bb4af787658b84e5b" + }, "seqkit/split2": { "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, diff --git a/modules/local/samtools/stats/functions.nf b/modules/local/samtools/stats/functions.nf deleted file mode 100644 index da9da093d3..0000000000 --- a/modules/local/samtools/stats/functions.nf +++ /dev/null @@ -1,68 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/modules/local/samtools/stats/main.nf b/modules/local/samtools/stats/main.nf deleted file mode 100644 index ee600dc161..0000000000 --- a/modules/local/samtools/stats/main.nf +++ /dev/null @@ -1,35 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process SAMTOOLS_STATS { - tag "$meta.id" - label 'process_low' - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - - conda (params.enable_conda ? "bioconda::samtools=1.12" : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/samtools:1.12--hd5e65b6_0" - } else { - container "quay.io/biocontainers/samtools:1.12--hd5e65b6_0" - } - - input: - tuple val(meta), path(cram), path(crai) - path(reference) - - output: - tuple val(meta), path("*.stats"), emit: stats - path "*.version.txt" , emit: version - - script: - def software = getSoftwareName(task.process) - """ - samtools stats -@${task.cpus} --reference ${reference} $cram > ${cram}.stats - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt - """ -} diff --git a/modules/nf-core/modules/samtools/stats/functions.nf b/modules/nf-core/modules/samtools/stats/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/samtools/stats/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/samtools/stats/main.nf b/modules/nf-core/modules/samtools/stats/main.nf new file mode 100644 index 0000000000..e0a2b50d5c --- /dev/null +++ b/modules/nf-core/modules/samtools/stats/main.nf @@ -0,0 +1,39 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process SAMTOOLS_STATS { + tag "$meta.id" + label 'process_low' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? "bioconda::samtools=1.14" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/samtools:1.14--hb421002_0" + } else { + container "quay.io/biocontainers/samtools:1.14--hb421002_0" + } + + input: + tuple val(meta), path(input), path(input_index) + path fasta + + output: + tuple val(meta), path("*.stats"), emit: stats + path "versions.yml" , emit: versions + + script: + def reference = fasta ? "--reference ${fasta}" : "" + """ + samtools stats --threads ${task.cpus-1} ${reference} ${input} > ${input}.stats + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS + """ +} diff --git a/modules/local/samtools/stats/meta.yml b/modules/nf-core/modules/samtools/stats/meta.yml similarity index 68% rename from modules/local/samtools/stats/meta.yml rename to modules/nf-core/modules/samtools/stats/meta.yml index b549ff5c8f..869e62e373 100644 --- a/modules/local/samtools/stats/meta.yml +++ b/modules/nf-core/modules/samtools/stats/meta.yml @@ -15,20 +15,25 @@ tools: homepage: http://www.htslib.org/ documentation: hhttp://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 + licence: ['MIT'] input: - meta: type: map description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - bam: - type: file - description: BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" - - bai: - type: file - description: Index for BAM/CRAM/SAM file - pattern: "*.{bai,crai,sai}" + - input: + type: file + description: BAM/CRAM file from alignment + pattern: "*.{bam,cram}" + - input_index: + type: file + description: BAI/CRAI file from alignment + pattern: "*.{bai,crai}" + - fasta: + type: optional file + description: Reference file the CRAM was created with + pattern: "*.{fasta,fa}" output: - meta: type: map @@ -39,9 +44,10 @@ output: type: file description: File containing samtools stats output pattern: "*.{stats}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" + - "@FriederikeHanssen" diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index a798748dde..370809ef89 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -16,7 +16,7 @@ include { GATK4_MARKDUPLICATES } from '../../modules/loca include { GATK4_MARKDUPLICATES_SPARK } from '../../modules/local/gatk4/markduplicatesspark/main' addParams(options: params.markduplicatesspark_options) include { QUALIMAP_BAMQC } from '../../modules/nf-core/modules/qualimap/bamqc/main' addParams(options: params.qualimap_bamqc_options) include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) -include { SAMTOOLS_STATS } from '../../modules/local/samtools/stats/main' addParams(options: params.samtools_stats_options) +include { SAMTOOLS_STATS } from '../../modules/nf-core/modules/samtools/stats/main' addParams(options: params.samtools_stats_options) include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM } from '../../modules/local/samtools/view/main' addParams(options: params.samtools_view_options) include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM_SPARK } from '../../modules/local/samtools/view/main' addParams(options: params.samtools_view_options) diff --git a/subworkflows/nf-core/recalibrate.nf b/subworkflows/nf-core/recalibrate.nf index f6a71111cc..9d5c00a268 100644 --- a/subworkflows/nf-core/recalibrate.nf +++ b/subworkflows/nf-core/recalibrate.nf @@ -9,12 +9,12 @@ params.qualimap_bamqc_options = [:] params.samtools_index_options = [:] params.samtools_stats_options = [:] -include { GATK4_APPLYBQSR as APPLYBQSR } from '../../modules/local/gatk4/applybqsr/main' addParams(options: params.applybqsr_options) -include { GATK4_APPLYBQSR_SPARK as APPLYBQSR_SPARK } from '../../modules/local/gatk4/applybqsrspark/main' addParams(options: params.applybqsr_spark_options) -include { QUALIMAP_BAMQC_CRAM } from '../../modules/local/qualimap/bamqccram/main' addParams(options: params.qualimap_bamqc_options) -include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) -include { SAMTOOLS_MERGE_CRAM } from '../../modules/local/samtools/mergecram/main' addParams(options: params.merge_cram_options) -include { SAMTOOLS_STATS } from '../../modules/local/samtools/stats/main' addParams(options: params.samtools_stats_options) +include { GATK4_APPLYBQSR as APPLYBQSR } from '../../modules/local/gatk4/applybqsr/main' addParams(options: params.applybqsr_options) +include { GATK4_APPLYBQSR_SPARK as APPLYBQSR_SPARK } from '../../modules/local/gatk4/applybqsrspark/main' addParams(options: params.applybqsr_spark_options) +include { QUALIMAP_BAMQC_CRAM } from '../../modules/local/qualimap/bamqccram/main' addParams(options: params.qualimap_bamqc_options) +include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) +include { SAMTOOLS_MERGE_CRAM } from '../../modules/local/samtools/mergecram/main' addParams(options: params.merge_cram_options) +include { SAMTOOLS_STATS } from '../../modules/nf-core/modules/samtools/stats/main' addParams(options: params.samtools_stats_options) workflow RECALIBRATE { take: From 8f65bc5cfd553ee4f578259656531a04e93b5264 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 13:24:10 +0100 Subject: [PATCH 21/28] update samtools/view --- modules/local/samtools/view/functions.nf | 54 ++++++++++++++---------- modules/local/samtools/view/main.nf | 31 ++++++++------ 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/modules/local/samtools/view/functions.nf b/modules/local/samtools/view/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/samtools/view/functions.nf +++ b/modules/local/samtools/view/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/samtools/view/main.nf b/modules/local/samtools/view/main.nf index 8f3ad0e0be..3078a1e051 100644 --- a/modules/local/samtools/view/main.nf +++ b/modules/local/samtools/view/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -12,28 +12,33 @@ process SAMTOOLS_VIEW { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - conda (params.enable_conda ? "bioconda::samtools=1.12" : null) + conda (params.enable_conda ? "bioconda::samtools=1.14" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/samtools:1.12--hd5e65b6_0" + container "https://depot.galaxyproject.org/singularity/samtools:1.14--hb421002_0" } else { - container "quay.io/biocontainers/samtools:1.12--hd5e65b6_0" + container "quay.io/biocontainers/samtools:1.14--hb421002_0" } input: - tuple val(meta), path(bam), path(bai) - path(fasta) - path(fai) + tuple val(meta), path(input), path(index) + path fasta + path fai output: - tuple val(meta), path("*.cram"), path("*.crai"), emit: cram_crai - path "*.version.txt" , emit: version + tuple val(meta), path("*.bam"), path("*.bai") , optional: true, emit: bam_bai + tuple val(meta), path("*.cram"), path("*.crai"), optional: true, emit: cram_crai + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) - def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def reference = fasta ? "--reference ${fasta} -C" : "" """ - samtools view --threads ${task.cpus} -T ${fasta} -C $options.args -o ${prefix}.cram $bam + samtools view --threads ${task.cpus-1} ${reference} $options.args $input > ${prefix}.cram samtools index -@${task.cpus} ${prefix}.cram - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS """ } From da8453e2eef704174c4607bc56e0ef821fe4f700 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 15:42:48 +0100 Subject: [PATCH 22/28] feat: update samtools/index --- modules/local/samtools/index/functions.nf | 54 ++++++++++++++--------- modules/local/samtools/index/main.nf | 19 ++++---- modules/local/samtools/index/meta.yml | 12 +++-- subworkflows/nf-core/mapping.nf | 14 +++--- 4 files changed, 59 insertions(+), 40 deletions(-) diff --git a/modules/local/samtools/index/functions.nf b/modules/local/samtools/index/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/samtools/index/functions.nf +++ b/modules/local/samtools/index/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/samtools/index/main.nf b/modules/local/samtools/index/main.nf index b1fe410d69..50b9f581e2 100644 --- a/modules/local/samtools/index/main.nf +++ b/modules/local/samtools/index/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -11,11 +11,11 @@ process SAMTOOLS_INDEX { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - conda (params.enable_conda ? "bioconda::samtools=1.12" : null) + conda (params.enable_conda ? "bioconda::samtools=1.14" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/samtools:1.12--hd5e65b6_0" + container "https://depot.galaxyproject.org/singularity/samtools:1.14--hb421002_0" } else { - container "quay.io/biocontainers/samtools:1.12--hd5e65b6_0" + container "quay.io/biocontainers/samtools:1.14--hb421002_0" } input: @@ -25,12 +25,15 @@ process SAMTOOLS_INDEX { tuple val(meta), path("*.bam", includeInputs:true), path("*.bai") , optional:true, emit: bam_bai tuple val(meta), path("*.bam", includeInputs:true), path("*.csi") , optional:true, emit: bam_csi tuple val(meta), path("*.cram", includeInputs:true), path("*.crai"), optional:true, emit: cram_crai - path "*.version.txt" , emit: version + path "versions.yml" , emit: version script: - def software = getSoftwareName(task.process) """ - samtools index $options.args $input - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt + samtools index -@ ${task.cpus-1} $options.args $input + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS """ } diff --git a/modules/local/samtools/index/meta.yml b/modules/local/samtools/index/meta.yml index 5d076e3be5..0905b3cd69 100644 --- a/modules/local/samtools/index/meta.yml +++ b/modules/local/samtools/index/meta.yml @@ -14,6 +14,7 @@ tools: homepage: http://www.htslib.org/ documentation: hhttp://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 + licence: ['MIT'] input: - meta: type: map @@ -34,14 +35,19 @@ output: type: file description: BAM/CRAM/SAM index file pattern: "*.{bai,crai,sai}" + - crai: + type: file + description: BAM/CRAM/SAM index file + pattern: "*.{bai,crai,sai}" - csi: type: file description: CSI index file pattern: "*.{csi}" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@ewels" + - "@maxulysse" diff --git a/subworkflows/nf-core/mapping.nf b/subworkflows/nf-core/mapping.nf index 69d8099d0a..6993fea9c4 100644 --- a/subworkflows/nf-core/mapping.nf +++ b/subworkflows/nf-core/mapping.nf @@ -10,13 +10,13 @@ params.merge_bam_options = [:] params.samtools_index_options = [:] params.seqkit_split2_options = [:] -include { BWAMEM2_MEM as BWAMEM2_MEM_T } from '../../modules/local/bwamem2/mem/main' addParams(options: params.bwamem2_mem_tumor_options) -include { BWAMEM2_MEM } from '../../modules/local/bwamem2/mem/main' addParams(options: params.bwamem2_mem_options) -include { BWA_MEM as BWAMEM1_MEM } from '../../modules/local/bwa/mem/main' addParams(options: params.bwamem1_mem_options) -include { BWA_MEM as BWAMEM1_MEM_T } from '../../modules/local/bwa/mem/main' addParams(options: params.bwamem1_mem_tumor_options) -include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) -include { SAMTOOLS_MERGE } from '../../modules/nf-core/modules/samtools/merge/main' addParams(options: params.merge_bam_options) -include { SEQKIT_SPLIT2 } from '../../modules/nf-core/modules/seqkit/split2/main.nf' addParams(options: params.seqkit_split2_options) +include { BWAMEM2_MEM as BWAMEM2_MEM_T } from '../../modules/local/bwamem2/mem/main' addParams(options: params.bwamem2_mem_tumor_options) +include { BWAMEM2_MEM } from '../../modules/local/bwamem2/mem/main' addParams(options: params.bwamem2_mem_options) +include { BWA_MEM as BWAMEM1_MEM } from '../../modules/local/bwa/mem/main' addParams(options: params.bwamem1_mem_options) +include { BWA_MEM as BWAMEM1_MEM_T } from '../../modules/local/bwa/mem/main' addParams(options: params.bwamem1_mem_tumor_options) +include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) +include { SAMTOOLS_MERGE } from '../../modules/nf-core/modules/samtools/merge/main' addParams(options: params.merge_bam_options) +include { SEQKIT_SPLIT2 } from '../../modules/nf-core/modules/seqkit/split2/main' addParams(options: params.seqkit_split2_options) workflow MAPPING { take: From 1602c68f9536f8b59176f931b3abc8ae2368389d Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 16:13:20 +0100 Subject: [PATCH 23/28] chores: update local modules --- modules.json | 18 +++ modules/local/build_intervals/functions.nf | 54 +++++---- modules/local/bwa/mem/functions.nf | 54 +++++---- modules/local/bwa/mem/main.nf | 10 +- modules/local/bwamem2/mem/functions.nf | 54 +++++---- modules/local/bwamem2/mem/main.nf | 10 +- modules/local/concat_vcf/functions.nf | 54 +++++---- .../local/create_intervals_bed/functions.nf | 54 +++++---- modules/local/deepvariant/functions.nf | 86 ++++++++------- modules/local/deepvariant/main.nf | 16 +-- modules/local/freebayes/functions.nf | 68 ------------ modules/local/freebayes/main.nf | 38 ------- modules/local/functions.nf | 54 +++++---- modules/local/gatk4/applybqsr/functions.nf | 54 +++++---- modules/local/gatk4/applybqsr/main.nf | 7 +- .../local/gatk4/applybqsrspark/functions.nf | 54 +++++---- modules/local/gatk4/applybqsrspark/main.nf | 8 +- modules/local/gatk4/baserecalibrator/main.nf | 5 +- .../gatk4/baserecalibratorspark/functions.nf | 54 +++++---- .../local/gatk4/baserecalibratorspark/main.nf | 7 +- .../estimatelibrarycomplexity/functions.nf | 54 +++++---- .../gatk4/estimatelibrarycomplexity/main.nf | 7 +- .../gatk4/gatherbqsrreports/functions.nf | 54 +++++---- modules/local/gatk4/gatherbqsrreports/main.nf | 10 +- modules/local/gatk4/genotypegvcf/functions.nf | 54 +++++---- modules/local/gatk4/genotypegvcf/main.nf | 10 +- .../local/gatk4/haplotypecaller/functions.nf | 54 +++++---- modules/local/gatk4/haplotypecaller/main.nf | 8 +- .../gatk4/markduplicatesspark/functions.nf | 54 +++++---- .../local/gatk4/markduplicatesspark/main.nf | 9 +- .../gatk4/mutect2/merge_stats/functions.nf | 54 +++++---- .../local/gatk4/mutect2/merge_stats/main.nf | 7 +- .../local/gatk4/mutect2/somatic/functions.nf | 54 +++++---- modules/local/gatk4/mutect2/somatic/main.nf | 7 +- .../local/gatk4/mutect2/tumor/functions.nf | 54 +++++---- modules/local/gatk4/mutect2/tumor/main.nf | 7 +- modules/local/get_software_versions.nf | 3 +- modules/local/index_target_bed/functions.nf | 78 +++++++++++++ modules/local/index_target_bed/main.nf | 2 +- modules/local/manta/somatic/functions.nf | 68 ------------ modules/local/manta/somatic/main.nf | 64 ----------- modules/local/msisensorpro/msi/functions.nf | 54 +++++---- modules/local/msisensorpro/msi/main.nf | 25 +++-- modules/local/msisensorpro/scan/functions.nf | 54 +++++---- modules/local/msisensorpro/scan/main.nf | 11 +- modules/local/qualimap/bamqccram/functions.nf | 54 +++++---- modules/local/qualimap/bamqccram/main.nf | 2 +- modules/local/samtools/mergecram/functions.nf | 54 +++++---- modules/local/samtools/mergecram/main.nf | 10 +- modules/local/strelka/germline/functions.nf | 68 ------------ modules/local/strelka/somatic/functions.nf | 68 ------------ modules/local/strelka/somatic/main.nf | 55 ---------- modules/local/strelka/somatic/meta.yml | 55 ---------- modules/local/strelka/somaticbp/functions.nf | 68 ------------ modules/local/strelka/somaticbp/main.nf | 56 ---------- modules/local/strelka/somaticbp/meta.yml | 55 ---------- .../nf-core/modules/freebayes/functions.nf | 78 +++++++++++++ modules/nf-core/modules/freebayes/main.nf | 81 ++++++++++++++ modules/nf-core/modules/freebayes/meta.yml | 83 ++++++++++++++ .../modules/manta/germline/functions.nf | 78 +++++++++++++ .../nf-core/modules/manta/germline/main.nf | 68 ++++++++++++ .../nf-core/modules/manta/germline/meta.yml | 87 +++++++++++++++ .../modules/manta/somatic/functions.nf | 78 +++++++++++++ modules/nf-core/modules/manta/somatic/main.nf | 67 ++++++++++++ .../nf-core/modules/manta/somatic/meta.yml | 103 ++++++++++++++++++ .../modules/manta/tumoronly/functions.nf | 78 +++++++++++++ .../nf-core/modules/manta/tumoronly/main.nf | 68 ++++++++++++ .../nf-core/modules/manta/tumoronly/meta.yml | 87 +++++++++++++++ .../modules/strelka/germline/functions.nf | 78 +++++++++++++ .../modules}/strelka/germline/main.nf | 24 ++-- .../modules}/strelka/germline/meta.yml | 18 +-- .../modules/strelka/somatic/functions.nf | 78 +++++++++++++ .../nf-core/modules/strelka/somatic/main.nf | 61 +++++++++++ .../nf-core/modules/strelka/somatic/meta.yml | 93 ++++++++++++++++ 74 files changed, 2227 insertions(+), 1241 deletions(-) delete mode 100644 modules/local/freebayes/functions.nf delete mode 100644 modules/local/freebayes/main.nf create mode 100644 modules/local/index_target_bed/functions.nf delete mode 100644 modules/local/manta/somatic/functions.nf delete mode 100644 modules/local/manta/somatic/main.nf delete mode 100644 modules/local/strelka/germline/functions.nf delete mode 100644 modules/local/strelka/somatic/functions.nf delete mode 100644 modules/local/strelka/somatic/main.nf delete mode 100644 modules/local/strelka/somatic/meta.yml delete mode 100644 modules/local/strelka/somaticbp/functions.nf delete mode 100644 modules/local/strelka/somaticbp/main.nf delete mode 100644 modules/local/strelka/somaticbp/meta.yml create mode 100644 modules/nf-core/modules/freebayes/functions.nf create mode 100644 modules/nf-core/modules/freebayes/main.nf create mode 100644 modules/nf-core/modules/freebayes/meta.yml create mode 100644 modules/nf-core/modules/manta/germline/functions.nf create mode 100644 modules/nf-core/modules/manta/germline/main.nf create mode 100644 modules/nf-core/modules/manta/germline/meta.yml create mode 100644 modules/nf-core/modules/manta/somatic/functions.nf create mode 100644 modules/nf-core/modules/manta/somatic/main.nf create mode 100644 modules/nf-core/modules/manta/somatic/meta.yml create mode 100644 modules/nf-core/modules/manta/tumoronly/functions.nf create mode 100644 modules/nf-core/modules/manta/tumoronly/main.nf create mode 100644 modules/nf-core/modules/manta/tumoronly/meta.yml create mode 100644 modules/nf-core/modules/strelka/germline/functions.nf rename modules/{local => nf-core/modules}/strelka/germline/main.nf (74%) rename modules/{local => nf-core/modules}/strelka/germline/meta.yml (85%) create mode 100644 modules/nf-core/modules/strelka/somatic/functions.nf create mode 100644 modules/nf-core/modules/strelka/somatic/main.nf create mode 100644 modules/nf-core/modules/strelka/somatic/meta.yml diff --git a/modules.json b/modules.json index 7796312cb7..faa90c9170 100644 --- a/modules.json +++ b/modules.json @@ -18,9 +18,21 @@ "fastqc": { "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, + "freebayes": { + "git_sha": "9767b081b9d808d7b6671d34d48c70afb436a5f1" + }, "gatk4/createsequencedictionary": { "git_sha": "3b600af50eae8264960df817277cfe303d2acd47" }, + "manta/germline": { + "git_sha": "ac1e6df076195cec553a2079c9cebd94026a0d47" + }, + "manta/somatic": { + "git_sha": "ac1e6df076195cec553a2079c9cebd94026a0d47" + }, + "manta/tumoronly": { + "git_sha": "ac1e6df076195cec553a2079c9cebd94026a0d47" + }, "multiqc": { "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, @@ -42,6 +54,12 @@ "snpeff": { "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, + "strelka/germline": { + "git_sha": "ac1e6df076195cec553a2079c9cebd94026a0d47" + }, + "strelka/somatic": { + "git_sha": "ac1e6df076195cec553a2079c9cebd94026a0d47" + }, "tabix/bgziptabix": { "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, diff --git a/modules/local/build_intervals/functions.nf b/modules/local/build_intervals/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/build_intervals/functions.nf +++ b/modules/local/build_intervals/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/bwa/mem/functions.nf b/modules/local/bwa/mem/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/bwa/mem/functions.nf +++ b/modules/local/bwa/mem/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/bwa/mem/main.nf b/modules/local/bwa/mem/main.nf index f0b61e4407..bcdfbcb6e0 100644 --- a/modules/local/bwa/mem/main.nf +++ b/modules/local/bwa/mem/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,7 +24,7 @@ process BWA_MEM { output: tuple val(meta), path("*.bam"), emit: bam - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def split_cpus = Math.floor(task.cpus/2) @@ -47,6 +47,10 @@ process BWA_MEM { $reads \\ | samtools $options.args2 $sort_order --threads ${split_cpus} -o ${prefix}bam - - echo \$(bwa 2>&1) | sed 's/^.*Version: //; s/Contact:.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(bwa 2>&1) | sed 's/^.*Version: //; s/Contact:.*\$//') + samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS """ } diff --git a/modules/local/bwamem2/mem/functions.nf b/modules/local/bwamem2/mem/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/bwamem2/mem/functions.nf +++ b/modules/local/bwamem2/mem/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/bwamem2/mem/main.nf b/modules/local/bwamem2/mem/main.nf index 8add7094d0..351c32a05c 100644 --- a/modules/local/bwamem2/mem/main.nf +++ b/modules/local/bwamem2/mem/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,7 +24,7 @@ process BWAMEM2_MEM { output: tuple val(meta), path("*.bam"), emit: bam - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def split_cpus = Math.floor(task.cpus/2) @@ -47,6 +47,10 @@ process BWAMEM2_MEM { $reads \\ | samtools $options.args2 $sort_order -@ ${split_cpus} -o ${prefix}bam - - echo \$(bwa-mem2 version 2>&1) > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(bwa-mem2 version 2>&1) | sed 's/.* //') + samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS """ } diff --git a/modules/local/concat_vcf/functions.nf b/modules/local/concat_vcf/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/concat_vcf/functions.nf +++ b/modules/local/concat_vcf/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/create_intervals_bed/functions.nf b/modules/local/create_intervals_bed/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/create_intervals_bed/functions.nf +++ b/modules/local/create_intervals_bed/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/deepvariant/functions.nf b/modules/local/deepvariant/functions.nf index 9d0137e374..85628ee0eb 100644 --- a/modules/local/deepvariant/functions.nf +++ b/modules/local/deepvariant/functions.nf @@ -1,19 +1,24 @@ -/* - * ----------------------------------------------------- - * Utility functions used in nf-core DSL2 module files - * ----------------------------------------------------- - */ +// +// Utility functions used in nf-core DSL2 module files +// -/* - * Extract name of software tool from process name using $task.process - */ +// +// Extract name of software tool from process name using $task.process +// def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } -/* - * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules - */ +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// def initOptions(Map args) { def Map options = [:] options.args = args.args ?: '' @@ -26,45 +31,48 @@ def initOptions(Map args) { return options } -/* - * Tidy up and join elements of a list to return a path string - */ +// +// Tidy up and join elements of a list to return a path string +// def getPathFromList(path_list) { def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes return paths.join('/') } -/* - * Function to save/publish module results - */ +// +// Function to save/publish module results +// def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/deepvariant/main.nf b/modules/local/deepvariant/main.nf index c685c18f06..8d0cdc8872 100644 --- a/modules/local/deepvariant/main.nf +++ b/modules/local/deepvariant/main.nf @@ -1,4 +1,5 @@ -include { initOptions; saveFiles; getSoftwareName } from './functions' +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,13 +25,11 @@ process DEEPVARIANT { output: tuple val(meta), path("*.vcf.gz"), emit: vcf - tuple val(meta), path("*g.vcf.gz"), emit: gvcf - path "*.version.txt" , emit: version + tuple val(meta), path("*g.vcf.gz"), emit: gvcf + path "versions.yml", emit: versions script: - def software = getSoftwareName(task.process) - def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" - + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" """ /opt/deepvariant/bin/run_deepvariant \\ --ref=${fasta} \\ @@ -40,7 +39,10 @@ process DEEPVARIANT { ${options.args} \\ --num_shards=${task.cpus} - echo \$(/opt/deepvariant/bin/run_deepvariant --version) | sed 's/^.*version //; s/ .*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + deepvariant: \$(echo \$(/opt/deepvariant/bin/run_deepvariant --version) | sed 's/^.*version //; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/freebayes/functions.nf b/modules/local/freebayes/functions.nf deleted file mode 100644 index da9da093d3..0000000000 --- a/modules/local/freebayes/functions.nf +++ /dev/null @@ -1,68 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/modules/local/freebayes/main.nf b/modules/local/freebayes/main.nf deleted file mode 100644 index 1948a2df5a..0000000000 --- a/modules/local/freebayes/main.nf +++ /dev/null @@ -1,38 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process FREEBAYES { - tag "$meta.id" - label 'process_low' - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - - conda (params.enable_conda ? "bioconda::freebayes=1.3.5" : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/freebayes:1.3.5--py38ha193a2f_3" - } else { - container "quay.io/biocontainers/freebayes:1.3.5--py38ha193a2f_3" - } - - input: - tuple val(meta), path(cram), path(crai) - - output: - // TODO nf-core: Named file extensions MUST be emitted for ALL output channels - tuple val(meta), path("*.bam"), emit: bam - // TODO nf-core: List additional required output channels/values here - path "*.version.txt" , emit: version - - script: - def software = getSoftwareName(task.process) - def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" - """ - - - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt - """ -} diff --git a/modules/local/functions.nf b/modules/local/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/functions.nf +++ b/modules/local/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/applybqsr/functions.nf b/modules/local/gatk4/applybqsr/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/applybqsr/functions.nf +++ b/modules/local/gatk4/applybqsr/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/applybqsr/main.nf b/modules/local/gatk4/applybqsr/main.nf index 5ffdeda474..9d275ad0a7 100644 --- a/modules/local/gatk4/applybqsr/main.nf +++ b/modules/local/gatk4/applybqsr/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -47,6 +47,9 @@ process GATK4_APPLYBQSR { -O ${prefix}.cram \\ $options.args - echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/applybqsrspark/functions.nf b/modules/local/gatk4/applybqsrspark/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/applybqsrspark/functions.nf +++ b/modules/local/gatk4/applybqsrspark/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/applybqsrspark/main.nf b/modules/local/gatk4/applybqsrspark/main.nf index 865a35245f..f429e88b14 100644 --- a/modules/local/gatk4/applybqsrspark/main.nf +++ b/modules/local/gatk4/applybqsrspark/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -48,7 +48,9 @@ process GATK4_APPLYBQSR_SPARK { $options.args \ --spark-master local[${task.cpus}] - - gatk --version | grep Picard | sed "s/Picard Version: //g" > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/baserecalibrator/main.nf b/modules/local/gatk4/baserecalibrator/main.nf index 2844fdc344..e84913ba55 100644 --- a/modules/local/gatk4/baserecalibrator/main.nf +++ b/modules/local/gatk4/baserecalibrator/main.nf @@ -51,6 +51,9 @@ process GATK4_BASERECALIBRATOR { $options.args \ -O ${prefix}.table - gatk --version | grep Picard | sed "s/Picard Version: //g" > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/baserecalibratorspark/functions.nf b/modules/local/gatk4/baserecalibratorspark/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/baserecalibratorspark/functions.nf +++ b/modules/local/gatk4/baserecalibratorspark/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/baserecalibratorspark/main.nf b/modules/local/gatk4/baserecalibratorspark/main.nf index 4a3928bba2..5d02d04980 100644 --- a/modules/local/gatk4/baserecalibratorspark/main.nf +++ b/modules/local/gatk4/baserecalibratorspark/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -51,6 +51,9 @@ process GATK4_BASERECALIBRATOR_SPARK { -O ${prefix}.table \ --spark-master local[${task.cpus}] - gatk --version | grep Picard | sed "s/Picard Version: //g" > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/estimatelibrarycomplexity/functions.nf b/modules/local/gatk4/estimatelibrarycomplexity/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/estimatelibrarycomplexity/functions.nf +++ b/modules/local/gatk4/estimatelibrarycomplexity/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/estimatelibrarycomplexity/main.nf b/modules/local/gatk4/estimatelibrarycomplexity/main.nf index 9838b112ad..b09bae29f8 100644 --- a/modules/local/gatk4/estimatelibrarycomplexity/main.nf +++ b/modules/local/gatk4/estimatelibrarycomplexity/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -45,6 +45,9 @@ process GATK4_ESTIMATELIBRARYCOMPLEXITY { --VALIDATION_STRINGENCY SILENT \ --TMP_DIR . $options.args - echo \$(gatk EstimateLibraryComplexity --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/gatherbqsrreports/functions.nf b/modules/local/gatk4/gatherbqsrreports/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/gatherbqsrreports/functions.nf +++ b/modules/local/gatk4/gatherbqsrreports/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/gatherbqsrreports/main.nf b/modules/local/gatk4/gatherbqsrreports/main.nf index fb78fb4563..1e69c75a20 100644 --- a/modules/local/gatk4/gatherbqsrreports/main.nf +++ b/modules/local/gatk4/gatherbqsrreports/main.nf @@ -1,7 +1,8 @@ -include { initOptions; saveFiles; getSoftwareName } from './functions' +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] -def options = initOptions(params.options) +options = initOptions(params.options) process GATK4_GATHERBQSRREPORTS { tag "$meta.id" @@ -40,6 +41,9 @@ process GATK4_GATHERBQSRREPORTS { --tmp-dir . \ -O ${meta.sample}.recal.table - echo \$(gatk GatherBQSRReports --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/genotypegvcf/functions.nf b/modules/local/gatk4/genotypegvcf/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/genotypegvcf/functions.nf +++ b/modules/local/gatk4/genotypegvcf/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/genotypegvcf/main.nf b/modules/local/gatk4/genotypegvcf/main.nf index ffb93bc4b0..261a9ca47a 100644 --- a/modules/local/gatk4/genotypegvcf/main.nf +++ b/modules/local/gatk4/genotypegvcf/main.nf @@ -1,7 +1,8 @@ -include { initOptions; saveFiles; getSoftwareName } from './functions' +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] -def options = initOptions(params.options) +options = initOptions(params.options) process GATK4_GENOTYPEGVCF { tag "$meta.id" @@ -47,6 +48,9 @@ process GATK4_GENOTYPEGVCF { -V ${gvcf} \ -O ${interval.baseName}_${meta.id}.vcf - echo \$(gatk GenotypeGVCFs --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/haplotypecaller/functions.nf b/modules/local/gatk4/haplotypecaller/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/haplotypecaller/functions.nf +++ b/modules/local/gatk4/haplotypecaller/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/haplotypecaller/main.nf b/modules/local/gatk4/haplotypecaller/main.nf index d88bd0954f..9bff975321 100644 --- a/modules/local/gatk4/haplotypecaller/main.nf +++ b/modules/local/gatk4/haplotypecaller/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -55,6 +55,10 @@ process GATK4_HAPLOTYPECALLER { -O ${prefix}.vcf \\ --tmp-dir . \ $options.args - gatk --version | grep Picard | sed "s/Picard Version: //g" > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/markduplicatesspark/functions.nf b/modules/local/gatk4/markduplicatesspark/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/markduplicatesspark/functions.nf +++ b/modules/local/gatk4/markduplicatesspark/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/markduplicatesspark/main.nf b/modules/local/gatk4/markduplicatesspark/main.nf index a7671f5b07..60c3699753 100644 --- a/modules/local/gatk4/markduplicatesspark/main.nf +++ b/modules/local/gatk4/markduplicatesspark/main.nf @@ -1,8 +1,8 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] -def options = initOptions(params.options) +options = initOptions(params.options) process GATK4_MARKDUPLICATES_SPARK { tag "$meta.id" @@ -49,6 +49,9 @@ process GATK4_MARKDUPLICATES_SPARK { --spark-master local[${task.cpus}] \\ $options.args - echo \$(gatk MarkDuplicatesSpark --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/mutect2/merge_stats/functions.nf b/modules/local/gatk4/mutect2/merge_stats/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/mutect2/merge_stats/functions.nf +++ b/modules/local/gatk4/mutect2/merge_stats/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/mutect2/merge_stats/main.nf b/modules/local/gatk4/mutect2/merge_stats/main.nf index ec6d3a6cc5..1c05e1afe3 100644 --- a/modules/local/gatk4/mutect2/merge_stats/main.nf +++ b/modules/local/gatk4/mutect2/merge_stats/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -55,6 +55,9 @@ process GATK4_MUTECT2_SOMATIC { ${PON} \ -O ${prefix}.vcf - echo \$(gatk Mutect2 --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/mutect2/somatic/functions.nf b/modules/local/gatk4/mutect2/somatic/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/mutect2/somatic/functions.nf +++ b/modules/local/gatk4/mutect2/somatic/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/mutect2/somatic/main.nf b/modules/local/gatk4/mutect2/somatic/main.nf index 5b56c9dd0f..0c7205c3f6 100644 --- a/modules/local/gatk4/mutect2/somatic/main.nf +++ b/modules/local/gatk4/mutect2/somatic/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -59,6 +59,9 @@ process GATK4_MUTECT2_SOMATIC { ${PON} \ -O ${prefix}.vcf - echo \$(gatk Mutect2 --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/gatk4/mutect2/tumor/functions.nf b/modules/local/gatk4/mutect2/tumor/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/gatk4/mutect2/tumor/functions.nf +++ b/modules/local/gatk4/mutect2/tumor/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/gatk4/mutect2/tumor/main.nf b/modules/local/gatk4/mutect2/tumor/main.nf index 791996357b..e5b1ed440b 100644 --- a/modules/local/gatk4/mutect2/tumor/main.nf +++ b/modules/local/gatk4/mutect2/tumor/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -58,6 +58,9 @@ process GATK4_MUTECT2_TUMOR { ${PON} \ -O ${prefix}.vcf - echo \$(gatk Mutect2 --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS """ } diff --git a/modules/local/get_software_versions.nf b/modules/local/get_software_versions.nf index caa467a6c5..b8be48ca88 100644 --- a/modules/local/get_software_versions.nf +++ b/modules/local/get_software_versions.nf @@ -1,7 +1,8 @@ // Import generic module functions -include { saveFiles } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] +options = initOptions(params.options) process GET_SOFTWARE_VERSIONS { publishDir "${params.outdir}", diff --git a/modules/local/index_target_bed/functions.nf b/modules/local/index_target_bed/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/local/index_target_bed/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/local/index_target_bed/main.nf b/modules/local/index_target_bed/main.nf index e0e6cb789c..7fc9c5b4e0 100644 --- a/modules/local/index_target_bed/main.nf +++ b/modules/local/index_target_bed/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) diff --git a/modules/local/manta/somatic/functions.nf b/modules/local/manta/somatic/functions.nf deleted file mode 100644 index da9da093d3..0000000000 --- a/modules/local/manta/somatic/functions.nf +++ /dev/null @@ -1,68 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/modules/local/manta/somatic/main.nf b/modules/local/manta/somatic/main.nf deleted file mode 100644 index c0df7f4938..0000000000 --- a/modules/local/manta/somatic/main.nf +++ /dev/null @@ -1,64 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process MANTA_SOMATIC { - tag "$meta.id" - label 'process_high' - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - - conda (params.enable_conda ? "bioconda::manta=1.6.0" : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/manta:1.6.0--py27_0" - } else { - container "quay.io/biocontainers/manta:1.6.0--py27_0" - } - - input: - tuple val(meta), path(cram_normal), path(crai_normal), path(cram_tumor), path(crai_tumor) - path fasta - path fai - tuple path(target_bed), path(target_bed_tbi) - - output: - tuple val(meta), path(cram_normal), path(crai_normal), path(cram_tumor), path(crai_tumor), path("*.candidateSmallIndels.vcf.gz"), path("*.candidateSmallIndels.vcf.gz.tbi"), emit: manta_csi_for_strelka_bp - tuple val(meta), path("*.candidateSmallIndels.vcf.gz"), path("*.candidateSmallIndels.vcf.gz.tbi"), emit: candidate_small_indels_vcf - tuple val(meta), path("*.candidateSV.vcf.gz"), path("*.candidateSV.vcf.gz.tbi"), emit: candidate_sv_vcf - tuple val(meta), path("*.diploidSV.vcf.gz"), path("*.diploidSV.vcf.gz.tbi"), emit: diploid_sv_vcf - tuple val(meta), path("*.somaticSV.vcf.gz"), path("*.somaticSV.vcf.gz.tbi"), emit: somatic_sv_vcf - path "*.version.txt", emit: version - - script: - def software = getSoftwareName(task.process) - def ioptions = initOptions(options) - def prefix = ioptions.suffix ? "manta_${meta.id}${ioptions.suffix}" : "manta_${meta.id}" - // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "$ioptions.args" variable - // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter - // using the Nextflow "task" variable e.g. "--threads $task.cpus" - options_manta = params.target_bed ? "--exome --callRegions $target_bed" : "" - """ - configManta.py \ - --tumorBam $cram_tumor \ - --normalBam $cram_normal \ - --reference $fasta \ - $options_manta \ - --runDir manta - - python manta/runWorkflow.py -m local -j $task.cpus - - mv manta/results/variants/candidateSmallIndels.vcf.gz ${prefix}.candidateSmallIndels.vcf.gz - mv manta/results/variants/candidateSmallIndels.vcf.gz.tbi ${prefix}.candidateSmallIndels.vcf.gz.tbi - mv manta/results/variants/candidateSV.vcf.gz ${prefix}.candidateSV.vcf.gz - mv manta/results/variants/candidateSV.vcf.gz.tbi ${prefix}.candidateSV.vcf.gz.tbi - mv manta/results/variants/diploidSV.vcf.gz ${prefix}.diploidSV.vcf.gz - mv manta/results/variants/diploidSV.vcf.gz.tbi ${prefix}.diploidSV.vcf.gz.tbi - mv manta/results/variants/somaticSV.vcf.gz ${prefix}.somaticSV.vcf.gz - mv manta/results/variants/somaticSV.vcf.gz.tbi ${prefix}.somaticSV.vcf.gz.tbi - - echo configManta.py --version &> ${software}.version.txt #2>&1 - """ -} diff --git a/modules/local/msisensorpro/msi/functions.nf b/modules/local/msisensorpro/msi/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/msisensorpro/msi/functions.nf +++ b/modules/local/msisensorpro/msi/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/msisensorpro/msi/main.nf b/modules/local/msisensorpro/msi/main.nf index 1bdd6f2a9e..9f58c790d5 100644 --- a/modules/local/msisensorpro/msi/main.nf +++ b/modules/local/msisensorpro/msi/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -19,11 +19,15 @@ process MSISENSORPRO_MSI { } input: - tuple val(meta), path(cram_normal), path(crai_normal), path(cram_tumor), path(crai_tumor) - path msisensorpro_scan + tuple val(meta), path(cram_normal), path(crai_normal), path(cram_tumor), path(crai_tumor) + path msisensorpro_scan output: - tuple val(meta), path("msisensorpro_*.list") + tuple val(meta), path("${prefix}.list") , emit: output + tuple val(meta), path("${prefix}_dis.list") , emit: output_dis + tuple val(meta), path("${prefix}_germline.list"), emit: output_germline + tuple val(meta), path("${prefix}_somatic.list") , emit: output_somatic + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) @@ -37,9 +41,14 @@ process MSISENSORPRO_MSI { -b $task.cpus \\ $options.args - mv ${prefix} msisensorpro_${prefix}.list - mv ${prefix}_dis msisensorpro_${prefix}_dis.list - mv ${prefix}_germline msisensorpro_${prefix}_germline.list - mv ${prefix}_somatic msisensorpro_${prefix}_somatic.list + mv ${prefix} ${prefix}.list + mv ${prefix}_dis ${prefix}_dis.list + mv ${prefix}_germline ${prefix}_germline.list + mv ${prefix}_somatic ${prefix}_somatic.list + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(msisensor 2>&1 | sed -nE 's/Version:\\sv([0-9]\\.[0-9])/\\1/ p') + END_VERSIONS """ } diff --git a/modules/local/msisensorpro/scan/functions.nf b/modules/local/msisensorpro/scan/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/msisensorpro/scan/functions.nf +++ b/modules/local/msisensorpro/scan/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/msisensorpro/scan/main.nf b/modules/local/msisensorpro/scan/main.nf index 9df314398e..f9993a0e4e 100644 --- a/modules/local/msisensorpro/scan/main.nf +++ b/modules/local/msisensorpro/scan/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -22,8 +22,8 @@ process MSISENSORPRO_SCAN { path fasta output: - path "*.list", emit: list - path "*.version.txt", emit: version + tuple val(meta), path("*.list"), emit: list + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) @@ -34,6 +34,9 @@ process MSISENSORPRO_SCAN { -o ${fasta.baseName}.list \\ $options.args - echo \$(msisensor-pro 2>&1) | sed -nE 's/Version:\\sv([0-9]\\.[0-9])/\\1/ p' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(msisensor 2>&1 | sed -nE 's/Version:\\sv([0-9]\\.[0-9])/\\1/ p') + END_VERSIONS """ } diff --git a/modules/local/qualimap/bamqccram/functions.nf b/modules/local/qualimap/bamqccram/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/qualimap/bamqccram/functions.nf +++ b/modules/local/qualimap/bamqccram/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/qualimap/bamqccram/main.nf b/modules/local/qualimap/bamqccram/main.nf index 29537a5094..1ddfa010eb 100644 --- a/modules/local/qualimap/bamqccram/main.nf +++ b/modules/local/qualimap/bamqccram/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) diff --git a/modules/local/samtools/mergecram/functions.nf b/modules/local/samtools/mergecram/functions.nf index da9da093d3..85628ee0eb 100644 --- a/modules/local/samtools/mergecram/functions.nf +++ b/modules/local/samtools/mergecram/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/modules/local/samtools/mergecram/main.nf b/modules/local/samtools/mergecram/main.nf index 3b09ad914c..e4089f259b 100644 --- a/modules/local/samtools/mergecram/main.nf +++ b/modules/local/samtools/mergecram/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,13 +24,17 @@ process SAMTOOLS_MERGE_CRAM { output: tuple val(meta), path("${prefix}.cram"), emit: cram - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" """ samtools merge -@${task.cpus} --reference ${fasta} ${prefix}.cram $crams - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS """ } diff --git a/modules/local/strelka/germline/functions.nf b/modules/local/strelka/germline/functions.nf deleted file mode 100644 index da9da093d3..0000000000 --- a/modules/local/strelka/germline/functions.nf +++ /dev/null @@ -1,68 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/modules/local/strelka/somatic/functions.nf b/modules/local/strelka/somatic/functions.nf deleted file mode 100644 index da9da093d3..0000000000 --- a/modules/local/strelka/somatic/functions.nf +++ /dev/null @@ -1,68 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/modules/local/strelka/somatic/main.nf b/modules/local/strelka/somatic/main.nf deleted file mode 100644 index 3b81d64041..0000000000 --- a/modules/local/strelka/somatic/main.nf +++ /dev/null @@ -1,55 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process STRELKA_SOMATIC { - tag "$meta.id" - label 'process_high' - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - - conda (params.enable_conda ? "bioconda::strelka=2.9.10" : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/strelka:2.9.10--0" - } else { - container "quay.io/biocontainers/strelka:2.9.10--0" - } - - input: - tuple val(meta), path(cram_normal), path(crai_normal), path(cram_tumor), path(crai_tumor) - path fasta - path fai - tuple path(target_bed), path(target_bed_tbi) - - output: - tuple val(meta), path("*_somatic_indels.vcf.gz"), path("*_somatic_indels.vcf.gz.tbi"), emit: indels_vcf - tuple val(meta), path("*_somatic_snvs.vcf.gz"), path("*_somatic_snvs.vcf.gz.tbi"), emit: snvs_vcf - path "*.version.txt", emit: version - - script: - def software = getSoftwareName(task.process) - def ioptions = initOptions(options) - def prefix = ioptions.suffix ? "strelka_${meta.id}${ioptions.suffix}" : "strelka_${meta.id}" - def options_strelka = params.target_bed ? "--exome --callRegions ${target_bed}" : "" - """ - configureStrelkaSomaticWorkflow.py \\ - --tumor $cram_tumor \\ - --normal $cram_normal \\ - --referenceFasta $fasta \\ - $options_strelka \\ - $options.args \\ - --runDir strelka - - python strelka/runWorkflow.py -m local -j $task.cpus - - mv strelka/results/variants/somatic.indels.vcf.gz ${prefix}_somatic_indels.vcf.gz - mv strelka/results/variants/somatic.indels.vcf.gz.tbi ${prefix}_somatic_indels.vcf.gz.tbi - mv strelka/results/variants/somatic.snvs.vcf.gz ${prefix}_somatic_snvs.vcf.gz - mv strelka/results/variants/somatic.snvs.vcf.gz.tbi ${prefix}_somatic_snvs.vcf.gz.tbi - - echo configureStrelkaSomaticWorkflow.py --version &> ${software}.version.txt #2>&1 - """ -} diff --git a/modules/local/strelka/somatic/meta.yml b/modules/local/strelka/somatic/meta.yml deleted file mode 100644 index 5bf5878e1c..0000000000 --- a/modules/local/strelka/somatic/meta.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: strelka_somatic -description: Strelka2 is a fast and accurate small variant caller optimized for analysis of somatic variation -keywords: - - variantcalling - - somatic - - wgs - - vcf - - variants -tools: - - strelka: - description: Strelka calls somatic and germline small variants from mapped sequencing reads - homepage: https://github.com/Illumina/strelka - documentation: https://github.com/Illumina/strelka/blob/v2.9.x/docs/userGuide/README.md - tool_dev_url: https://github.com/Illumina/strelka - doi: 10.1038/s41592-018-0051-x - licence: ['GPL v3'] - -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test'] - - bam: - type: file - description: BAM file - pattern: "*.{bam}" - - bai: - type: file - description: BAM index file - pattern: "*.{bai}" - - target_bed: - type: file - description: An optional bed file - pattern: "*.{bed}" -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test'] - - indels_vcf: - type: file - description: gzipped somatic indels file and associated index file - pattern: "(*_somatic_indels.vcf.gz, *_somatic_indels.vcf.gz.tbi)" - - snvs_vcf: - type: file - description: gzipped somatic snvs file and associated index file - pattern: "(*_somatic_snvs.vcf.gz, _somatic_snvs.vcf.gz.tbi)" - - version: - type: file - description: File containing software version - pattern: "*.{version.txt}" -authors: - - "@arontommi" diff --git a/modules/local/strelka/somaticbp/functions.nf b/modules/local/strelka/somaticbp/functions.nf deleted file mode 100644 index da9da093d3..0000000000 --- a/modules/local/strelka/somaticbp/functions.nf +++ /dev/null @@ -1,68 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/modules/local/strelka/somaticbp/main.nf b/modules/local/strelka/somaticbp/main.nf deleted file mode 100644 index bf67e90a88..0000000000 --- a/modules/local/strelka/somaticbp/main.nf +++ /dev/null @@ -1,56 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process STRELKA_SOMATIC_BEST_PRACTICES { - tag "$meta.id" - label 'process_high' - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - - conda (params.enable_conda ? "bioconda::strelka=2.9.10" : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/strelka:2.9.10--0" - } else { - container "quay.io/biocontainers/strelka:2.9.10--0" - } - - input: - tuple val(meta), path(bam_normal), path(bai_normal), path(bam_tumor), path(bai_tumor), path(manta_csi), path(manta_csi_tbi) - path fasta - path fai - path target_bed - - output: - tuple val(meta), path("*_somatic_indels.vcf.gz"), path("*_somatic_indels.vcf.gz.tbi"), emit: indels_vcf - tuple val(meta), path("*_somatic_snvs.vcf.gz"), path("*_somatic_snvs.vcf.gz.tbi"), emit: snvs_vcf - path "*.version.txt", emit: version - - script: - def software = getSoftwareName(task.process) - def ioptions = initOptions(options) - def prefix = ioptions.suffix ? "strelka_${meta.id}${ioptions.suffix}" : "strelka_${meta.id}" - def options_strelka = params.target_bed ? "--exome --callRegions ${target_bed}" : "" - """ - configureStrelkaSomaticWorkflow.py \\ - --tumor $bam_tumor \\ - --normal $bam_normal \\ - --referenceFasta $fasta \\ - --indelCandidates $manta_csi \ - $options_strelka \\ - $options.args \\ - --runDir strelka - - python strelka/runWorkflow.py -m local -j $task.cpus - - mv strelka/results/variants/somatic.indels.vcf.gz ${prefix}_somatic_indels.vcf.gz - mv strelka/results/variants/somatic.indels.vcf.gz.tbi ${prefix}_somatic_indels.vcf.gz.tbi - mv strelka/results/variants/somatic.snvs.vcf.gz ${prefix}_somatic_snvs.vcf.gz - mv strelka/results/variants/somatic.snvs.vcf.gz.tbi ${prefix}_somatic_snvs.vcf.gz.tbi - - echo configureStrelkaSomaticWorkflow.py --version &> ${software}.version.txt #2>&1 - """ -} diff --git a/modules/local/strelka/somaticbp/meta.yml b/modules/local/strelka/somaticbp/meta.yml deleted file mode 100644 index 5bf5878e1c..0000000000 --- a/modules/local/strelka/somaticbp/meta.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: strelka_somatic -description: Strelka2 is a fast and accurate small variant caller optimized for analysis of somatic variation -keywords: - - variantcalling - - somatic - - wgs - - vcf - - variants -tools: - - strelka: - description: Strelka calls somatic and germline small variants from mapped sequencing reads - homepage: https://github.com/Illumina/strelka - documentation: https://github.com/Illumina/strelka/blob/v2.9.x/docs/userGuide/README.md - tool_dev_url: https://github.com/Illumina/strelka - doi: 10.1038/s41592-018-0051-x - licence: ['GPL v3'] - -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test'] - - bam: - type: file - description: BAM file - pattern: "*.{bam}" - - bai: - type: file - description: BAM index file - pattern: "*.{bai}" - - target_bed: - type: file - description: An optional bed file - pattern: "*.{bed}" -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test'] - - indels_vcf: - type: file - description: gzipped somatic indels file and associated index file - pattern: "(*_somatic_indels.vcf.gz, *_somatic_indels.vcf.gz.tbi)" - - snvs_vcf: - type: file - description: gzipped somatic snvs file and associated index file - pattern: "(*_somatic_snvs.vcf.gz, _somatic_snvs.vcf.gz.tbi)" - - version: - type: file - description: File containing software version - pattern: "*.{version.txt}" -authors: - - "@arontommi" diff --git a/modules/nf-core/modules/freebayes/functions.nf b/modules/nf-core/modules/freebayes/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/freebayes/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/freebayes/main.nf b/modules/nf-core/modules/freebayes/main.nf new file mode 100644 index 0000000000..0b23dc40e4 --- /dev/null +++ b/modules/nf-core/modules/freebayes/main.nf @@ -0,0 +1,81 @@ +// Import generic module functions +include { initOptions; saveFiles; getProcessName; getSoftwareName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process FREEBAYES { + tag "$meta.id" + label 'process_low' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? "bioconda::freebayes=1.3.5" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/freebayes:1.3.5--py38ha193a2f_3" + } else { + container "quay.io/biocontainers/freebayes:1.3.5--py38ha193a2f_3" + } + + input: + tuple val(meta), path(input_1), path(input_1_index), path(input_2), path(input_2_index) + path fasta + path fai + path targets + path samples + path populations + path cnv + + output: + tuple val(meta), path("*.vcf.gz") , emit: vcf + path "versions.yml" , emit: versions + + script: + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def input = input_2 ? "${input_1} ${input_2}" : "${input_1}" + def targets_file = targets ? "--target ${targets}" : "" + def samples_file = samples ? "--samples ${samples}" : "" + def populations_file = populations ? "--populations ${populations}" : "" + def cnv_file = cnv ? "--cnv-map ${cnv}" : "" + + if (task.cpus > 1) { + """ + freebayes-parallel \\ + <(fasta_generate_regions.py ${fasta}.fai 10000) ${task.cpus} \\ + -f $fasta \\ + $targets_file \\ + $samples_file \\ + $populations_file \\ + $cnv_file \\ + $options.args \\ + $input > ${prefix}.vcf + + gzip --no-name ${prefix}.vcf + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(freebayes --version 2>&1) | sed 's/version:\s*v//g' ) + END_VERSIONS + """ + + } else { + """ + freebayes \\ + -f $fasta \\ + $targets_file \\ + $samples_file \\ + $populations_file \\ + $cnv_file \\ + $options.args \\ + $input > ${prefix}.vcf + + gzip --no-name ${prefix}.vcf + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(freebayes --version 2>&1) | sed 's/version:\s*v//g' ) + END_VERSIONS + """ + } +} diff --git a/modules/nf-core/modules/freebayes/meta.yml b/modules/nf-core/modules/freebayes/meta.yml new file mode 100644 index 0000000000..75d44826d8 --- /dev/null +++ b/modules/nf-core/modules/freebayes/meta.yml @@ -0,0 +1,83 @@ +name: freebayes +description: A haplotype-based variant detector +keywords: + - variant caller + - SNP + - genotyping + - somatic variant calling + - germline variant calling + - bacterial variant calling + - bayesian + +tools: + - freebayes: + description: Bayesian haplotype-based polymorphism discovery and genotyping + homepage: https://github.com/freebayes/freebayes + documentation: https://github.com/freebayes/freebayes + tool_dev_url: https://github.com/freebayes/freebayes + doi: "arXiv:1207.3907" + licence: ['MIT'] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - input: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - input_index: + type: file + description: BAM/CRAM/SAM index file + pattern: "*.bam.bai" + - fasta: + type: file + description: reference fasta file + pattern: ".{fa,fa.gz,fasta,fasta.gz}" + - fai: + type: file + description: reference fasta file index + pattern: "*.fai" + - targets: + type: file + description: Optional - Limit analysis to targets listed in this BED-format FILE. + pattern: "*.bed" + - samples: + type: file + description: Optional - Limit analysis to samples listed (one per line) in the FILE. + pattern: "*.txt" + - populations: + type: file + description: Optional - Each line of FILE should list a sample and a population which it is part of. + pattern: "*.txt" + - cnv: + type: file + description: | + A copy number map BED file, which has + either a sample-level ploidy: + sample_name copy_number + or a region-specific format: + seq_name start end sample_name copy_number + pattern: "*.bed" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" + - vcf: + type: file + description: Compressed VCF file + pattern: "*.vcf.gz" + +authors: + - "@maxibor" + - "@FriederikeHanssen" + - "@maxulysse" diff --git a/modules/nf-core/modules/manta/germline/functions.nf b/modules/nf-core/modules/manta/germline/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/manta/germline/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/manta/germline/main.nf b/modules/nf-core/modules/manta/germline/main.nf new file mode 100644 index 0000000000..f957a7ece5 --- /dev/null +++ b/modules/nf-core/modules/manta/germline/main.nf @@ -0,0 +1,68 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process MANTA_GERMLINE { + tag "$meta.id" + label 'process_high' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? "bioconda::manta=1.6.0" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/manta:1.6.0--h9ee0642_1" + } else { + container "quay.io/biocontainers/manta:1.6.0--h9ee0642_1" + } + + input: + tuple val(meta), path(input), path(input_index) + path fasta + path fai + path target_bed + path target_bed_tbi + + output: + tuple val(meta), path("*candidate_small_indels.vcf.gz") , emit: candidate_small_indels_vcf + tuple val(meta), path("*candidate_small_indels.vcf.gz.tbi"), emit: candidate_small_indels_vcf_tbi + tuple val(meta), path("*candidate_sv.vcf.gz") , emit: candidate_sv_vcf + tuple val(meta), path("*candidate_sv.vcf.gz.tbi") , emit: candidate_sv_vcf_tbi + tuple val(meta), path("*diploid_sv.vcf.gz") , emit: diploid_sv_vcf + tuple val(meta), path("*diploid_sv.vcf.gz.tbi") , emit: diploid_sv_vcf_tbi + path "versions.yml" , emit: versions + + script: + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def options_manta = target_bed ? "--exome --callRegions $target_bed" : "" + """ + configManta.py \ + --bam $input \ + --reference $fasta \ + $options_manta \ + --runDir manta + + python manta/runWorkflow.py -m local -j $task.cpus + + mv manta/results/variants/candidateSmallIndels.vcf.gz \ + ${prefix}.candidate_small_indels.vcf.gz + mv manta/results/variants/candidateSmallIndels.vcf.gz.tbi \ + ${prefix}.candidate_small_indels.vcf.gz.tbi + mv manta/results/variants/candidateSV.vcf.gz \ + ${prefix}.candidate_sv.vcf.gz + mv manta/results/variants/candidateSV.vcf.gz.tbi \ + ${prefix}.candidate_sv.vcf.gz.tbi + mv manta/results/variants/diploidSV.vcf.gz \ + ${prefix}.diploid_sv.vcf.gz + mv manta/results/variants/diploidSV.vcf.gz.tbi \ + ${prefix}.diploid_sv.vcf.gz.tbi + + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( configManta.py --version ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/modules/manta/germline/meta.yml b/modules/nf-core/modules/manta/germline/meta.yml new file mode 100644 index 0000000000..3bdb82641b --- /dev/null +++ b/modules/nf-core/modules/manta/germline/meta.yml @@ -0,0 +1,87 @@ +name: manta_germline +description: Manta calls structural variants (SVs) and indels from mapped paired-end sequencing reads. It is optimized for analysis of germline variation in small sets of individuals and somatic variation in tumor/normal sample pairs. +keywords: + - somatic + - wgs + - wxs + - panel + - vcf + - structural variants + - small indels +tools: + - manta: + description: Structural variant and indel caller for mapped sequencing data + homepage: https://github.com/Illumina/manta + documentation: https://github.com/Illumina/manta/blob/v1.6.0/docs/userGuide/README.md + tool_dev_url: https://github.com/Illumina/manta + doi: "10.1093/bioinformatics/btv710" + licence: ['GPL v3'] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - input: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - input_index: + type: file + description: BAM/CRAM/SAM index file + pattern: "*.{bai,crai,sai}" + - fasta: + type: file + description: Genome reference FASTA file + pattern: "*.{fa,fasta}" + - fai: + type: file + description: Genome reference FASTA index file + pattern: "*.{fa.fai,fasta.fai}" + - target_bed: + type: file + description: BED file containing target regions for variant calling + pattern: "*.{bed}" + - target_bed_tbi: + type: file + description: Index for BED file containing target regions for variant calling + pattern: "*.{bed.tbi}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - candidate_small_indels_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - candidate_small_indels_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - candidate_sv_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - candidate_sv_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - diploid_sv_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - diploid_sv_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@maxulysse" diff --git a/modules/nf-core/modules/manta/somatic/functions.nf b/modules/nf-core/modules/manta/somatic/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/manta/somatic/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/manta/somatic/main.nf b/modules/nf-core/modules/manta/somatic/main.nf new file mode 100644 index 0000000000..f912d47816 --- /dev/null +++ b/modules/nf-core/modules/manta/somatic/main.nf @@ -0,0 +1,67 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process MANTA_SOMATIC { + tag "$meta.id" + label 'process_high' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? "bioconda::manta=1.6.0" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/manta:1.6.0--h9ee0642_1" + } else { + container "quay.io/biocontainers/manta:1.6.0--h9ee0642_1" + } + + input: + tuple val(meta), path(input_normal), path(input_index_normal), path(input_tumor), path(input_index_tumor) + path fasta + path fai + path target_bed + path target_bed_tbi + + output: + tuple val(meta), path("*.candidate_small_indels.vcf.gz") , emit: candidate_small_indels_vcf + tuple val(meta), path("*.candidate_small_indels.vcf.gz.tbi") , emit: candidate_small_indels_vcf_tbi + tuple val(meta), path("*.candidate_sv.vcf.gz") , emit: candidate_sv_vcf + tuple val(meta), path("*.candidate_sv.vcf.gz.tbi") , emit: candidate_sv_vcf_tbi + tuple val(meta), path("*.diploid_sv.vcf.gz") , emit: diploid_sv_vcf + tuple val(meta), path("*.diploid_sv.vcf.gz.tbi") , emit: diploid_sv_vcf_tbi + tuple val(meta), path("*.somatic_sv.vcf.gz") , emit: somatic_sv_vcf + tuple val(meta), path("*.somatic_sv.vcf.gz.tbi") , emit: somatic_sv_vcf_tbi + path "versions.yml" , emit: versions + + script: + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def options_manta = target_bed ? "--exome --callRegions $target_bed" : "" + + """ + configManta.py \ + --tumorBam $input_tumor \ + --normalBam $input_normal \ + --reference $fasta \ + $options_manta \ + --runDir manta + + python manta/runWorkflow.py -m local -j $task.cpus + + mv manta/results/variants/candidateSmallIndels.vcf.gz ${prefix}.candidate_small_indels.vcf.gz + mv manta/results/variants/candidateSmallIndels.vcf.gz.tbi ${prefix}.candidate_small_indels.vcf.gz.tbi + mv manta/results/variants/candidateSV.vcf.gz ${prefix}.candidate_sv.vcf.gz + mv manta/results/variants/candidateSV.vcf.gz.tbi ${prefix}.candidate_sv.vcf.gz.tbi + mv manta/results/variants/diploidSV.vcf.gz ${prefix}.diploid_sv.vcf.gz + mv manta/results/variants/diploidSV.vcf.gz.tbi ${prefix}.diploid_sv.vcf.gz.tbi + mv manta/results/variants/somaticSV.vcf.gz ${prefix}.somatic_sv.vcf.gz + mv manta/results/variants/somaticSV.vcf.gz.tbi ${prefix}.somatic_sv.vcf.gz.tbi + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( configManta.py --version ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/modules/manta/somatic/meta.yml b/modules/nf-core/modules/manta/somatic/meta.yml new file mode 100644 index 0000000000..ddd0eafe74 --- /dev/null +++ b/modules/nf-core/modules/manta/somatic/meta.yml @@ -0,0 +1,103 @@ +name: manta_somatic +description: Manta calls structural variants (SVs) and indels from mapped paired-end sequencing reads. It is optimized for analysis of germline variation in small sets of individuals and somatic variation in tumor/normal sample pairs. +keywords: + - somatic + - wgs + - wxs + - panel + - vcf + - structural variants + - small indels +tools: + - manta: + description: Structural variant and indel caller for mapped sequencing data + homepage: https://github.com/Illumina/manta + documentation: https://github.com/Illumina/manta/blob/v1.6.0/docs/userGuide/README.md + tool_dev_url: https://github.com/Illumina/manta + doi: "10.1093/bioinformatics/btv710" + licence: ['GPL v3'] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - input_normal: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - input_index_normal: + type: file + description: BAM/CRAM/SAM index file + pattern: "*.{bai,crai,sai}" + - input_tumor: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - input_index_tumor: + type: file + description: BAM/CRAM/SAM index file + pattern: "*.{bai,crai,sai}" + - fasta: + type: file + description: Genome reference FASTA file + pattern: "*.{fa,fasta}" + - fai: + type: file + description: Genome reference FASTA index file + pattern: "*.{fa.fai,fasta.fai}" + - target_bed: + type: file + description: BED file containing target regions for variant calling + pattern: "*.{bed}" + - target_bed_tbi: + type: file + description: Index for BED file containing target regions for variant calling + pattern: "*.{bed.tbi}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - candidate_small_indels_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - candidate_small_indels_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - candidate_sv_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - candidate_sv_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - diploid_sv_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - diploid_sv_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - somatic_sv_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - somatic_sv_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@FriederikeHanssen" diff --git a/modules/nf-core/modules/manta/tumoronly/functions.nf b/modules/nf-core/modules/manta/tumoronly/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/manta/tumoronly/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/manta/tumoronly/main.nf b/modules/nf-core/modules/manta/tumoronly/main.nf new file mode 100644 index 0000000000..f20e81287c --- /dev/null +++ b/modules/nf-core/modules/manta/tumoronly/main.nf @@ -0,0 +1,68 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process MANTA_TUMORONLY { + tag "$meta.id" + label 'process_high' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? "bioconda::manta=1.6.0" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/manta:1.6.0--h9ee0642_1" + } else { + container "quay.io/biocontainers/manta:1.6.0--h9ee0642_1" + } + + input: + tuple val(meta), path(input), path(input_index) + path fasta + path fai + path target_bed + path target_bed_tbi + + output: + tuple val(meta), path("*candidate_small_indels.vcf.gz") , emit: candidate_small_indels_vcf + tuple val(meta), path("*candidate_small_indels.vcf.gz.tbi"), emit: candidate_small_indels_vcf_tbi + tuple val(meta), path("*candidate_sv.vcf.gz") , emit: candidate_sv_vcf + tuple val(meta), path("*candidate_sv.vcf.gz.tbi") , emit: candidate_sv_vcf_tbi + tuple val(meta), path("*tumor_sv.vcf.gz") , emit: tumor_sv_vcf + tuple val(meta), path("*tumor_sv.vcf.gz.tbi") , emit: tumor_sv_vcf_tbi + path "versions.yml" , emit: versions + + script: + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def options_manta = target_bed ? "--exome --callRegions $target_bed" : "" + """ + configManta.py \ + --tumorBam $input \ + --reference $fasta \ + $options_manta \ + --runDir manta + + python manta/runWorkflow.py -m local -j $task.cpus + + mv manta/results/variants/candidateSmallIndels.vcf.gz \ + ${prefix}.candidate_small_indels.vcf.gz + mv manta/results/variants/candidateSmallIndels.vcf.gz.tbi \ + ${prefix}.candidate_small_indels.vcf.gz.tbi + mv manta/results/variants/candidateSV.vcf.gz \ + ${prefix}.candidate_sv.vcf.gz + mv manta/results/variants/candidateSV.vcf.gz.tbi \ + ${prefix}.candidate_sv.vcf.gz.tbi + mv manta/results/variants/tumorSV.vcf.gz \ + ${prefix}.tumor_sv.vcf.gz + mv manta/results/variants/tumorSV.vcf.gz.tbi \ + ${prefix}.tumor_sv.vcf.gz.tbi + + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( configManta.py --version ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/modules/manta/tumoronly/meta.yml b/modules/nf-core/modules/manta/tumoronly/meta.yml new file mode 100644 index 0000000000..86d1c6c0c5 --- /dev/null +++ b/modules/nf-core/modules/manta/tumoronly/meta.yml @@ -0,0 +1,87 @@ +name: manta_tumoronly +description: Manta calls structural variants (SVs) and indels from mapped paired-end sequencing reads. It is optimized for analysis of germline variation in small sets of individuals and somatic variation in tumor/normal sample pairs. +keywords: + - somatic + - wgs + - wxs + - panel + - vcf + - structural variants + - small indels +tools: + - manta: + description: Structural variant and indel caller for mapped sequencing data + homepage: https://github.com/Illumina/manta + documentation: https://github.com/Illumina/manta/blob/v1.6.0/docs/userGuide/README.md + tool_dev_url: https://github.com/Illumina/manta + doi: "10.1093/bioinformatics/btv710" + licence: ['GPL v3'] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - input: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - input_index: + type: file + description: BAM/CRAM/SAM index file + pattern: "*.{bai,crai,sai}" + - fasta: + type: file + description: Genome reference FASTA file + pattern: "*.{fa,fasta}" + - fai: + type: file + description: Genome reference FASTA index file + pattern: "*.{fa.fai,fasta.fai}" + - target_bed: + type: file + description: BED file containing target regions for variant calling + pattern: "*.{bed}" + - target_bed_tbi: + type: file + description: Index for BED file containing target regions for variant calling + pattern: "*.{bed.tbi}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - candidate_small_indels_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - candidate_small_indels_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - candidate_sv_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - candidate_sv_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - tumor_sv_vcf: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - tumor_sv_vcf_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@maxulysse" diff --git a/modules/nf-core/modules/strelka/germline/functions.nf b/modules/nf-core/modules/strelka/germline/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/strelka/germline/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/local/strelka/germline/main.nf b/modules/nf-core/modules/strelka/germline/main.nf similarity index 74% rename from modules/local/strelka/germline/main.nf rename to modules/nf-core/modules/strelka/germline/main.nf index 88afa59b73..5e913c40f6 100644 --- a/modules/local/strelka/germline/main.nf +++ b/modules/nf-core/modules/strelka/germline/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -19,25 +19,26 @@ process STRELKA_GERMLINE { } input: - tuple val(meta), path(cram), path(crai) - path (fasta) - path (fai) - tuple path(target_bed), path(target_bed_tbi) + tuple val(meta), path(input), path(input_index) + path fasta + path fai + path target_bed + path target_bed_tbi + output: tuple val(meta), path("*variants.vcf.gz") , emit: vcf tuple val(meta), path("*variants.vcf.gz.tbi"), emit: vcf_tbi tuple val(meta), path("*genome.vcf.gz") , emit: genome_vcf tuple val(meta), path("*genome.vcf.gz.tbi") , emit: genome_vcf_tbi - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: - def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" - def regions = params.target_bed ? "--exome --callRegions ${target_bed}" : "" + def regions = target_bed ? "--exome --callRegions ${target_bed}" : "" """ configureStrelkaGermlineWorkflow.py \\ - --bam $cram \\ + --bam $input \\ --referenceFasta $fasta \\ $regions \\ $options.args \\ @@ -49,6 +50,9 @@ process STRELKA_GERMLINE { mv strelka/results/variants/variants.vcf.gz ${prefix}.variants.vcf.gz mv strelka/results/variants/variants.vcf.gz.tbi ${prefix}.variants.vcf.gz.tbi - echo configureStrelkaGermlineWorkflow.py --version &> ${software}.version.txt #2>&1 + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( configureStrelkaGermlineWorkflow.py --version ) + END_VERSIONS """ } diff --git a/modules/local/strelka/germline/meta.yml b/modules/nf-core/modules/strelka/germline/meta.yml similarity index 85% rename from modules/local/strelka/germline/meta.yml rename to modules/nf-core/modules/strelka/germline/meta.yml index ae0ecb472a..2eeb0f8fa8 100644 --- a/modules/local/strelka/germline/meta.yml +++ b/modules/nf-core/modules/strelka/germline/meta.yml @@ -21,14 +21,14 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test'] - - bam: + - input: type: file - description: BAM file - pattern: "*.{bam}" - - bai: + description: BAM/CRAM file + pattern: "*.{bam,cram}" + - input_index: type: file - description: BAM index file - pattern: "*.{bai}" + description: BAM/CRAI index file + pattern: "*.{bai,crai}" - target_bed: type: file description: An optional bed file @@ -55,9 +55,9 @@ output: type: file description: index file for the genome_vcf file pattern: "*_genome.vcf.gz.tbi" - - version: + - versions: type: file - description: File containing software version - pattern: "*.{version.txt}" + description: File containing software versions + pattern: "versions.yml" authors: - "@arontommi" diff --git a/modules/nf-core/modules/strelka/somatic/functions.nf b/modules/nf-core/modules/strelka/somatic/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/strelka/somatic/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/strelka/somatic/main.nf b/modules/nf-core/modules/strelka/somatic/main.nf new file mode 100644 index 0000000000..633b0a2c01 --- /dev/null +++ b/modules/nf-core/modules/strelka/somatic/main.nf @@ -0,0 +1,61 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process STRELKA_SOMATIC { + tag "$meta.id" + label 'process_high' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? "bioconda::strelka=2.9.10" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/strelka:2.9.10--h9ee0642_1" + } else { + container "quay.io/biocontainers/strelka:2.9.10--h9ee0642_1" + } + + input: + tuple val(meta), path(input_normal), path(input_index_normal), path(input_tumor), path(input_index_tumor), path(manta_candidate_small_indels), path(manta_candidate_small_indels_tbi) + path fasta + path fai + path target_bed + path target_bed_tbi + + output: + tuple val(meta), path("*.somatic_indels.vcf.gz") , emit: vcf_indels + tuple val(meta), path("*.somatic_indels.vcf.gz.tbi"), emit: vcf_indels_tbi + tuple val(meta), path("*.somatic_snvs.vcf.gz") , emit: vcf_snvs + tuple val(meta), path("*.somatic_snvs.vcf.gz.tbi") , emit: vcf_snvs_tbi + path "versions.yml" , emit: versions + + script: + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + def options_target_bed = target_bed ? "--exome --callRegions ${target_bed}" : "" + def options_manta = manta_candidate_small_indels ? "--indelCandidates ${manta_candidate_small_indels}" : "" + """ + configureStrelkaSomaticWorkflow.py \\ + --tumor $input_tumor \\ + --normal $input_normal \\ + --referenceFasta $fasta \\ + $options_target_bed \\ + $options_manta \\ + $options.args \\ + --runDir strelka + + python strelka/runWorkflow.py -m local -j $task.cpus + + mv strelka/results/variants/somatic.indels.vcf.gz ${prefix}.somatic_indels.vcf.gz + mv strelka/results/variants/somatic.indels.vcf.gz.tbi ${prefix}.somatic_indels.vcf.gz.tbi + mv strelka/results/variants/somatic.snvs.vcf.gz ${prefix}.somatic_snvs.vcf.gz + mv strelka/results/variants/somatic.snvs.vcf.gz.tbi ${prefix}.somatic_snvs.vcf.gz.tbi + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$( configureStrelkaSomaticWorkflow.py --version ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/modules/strelka/somatic/meta.yml b/modules/nf-core/modules/strelka/somatic/meta.yml new file mode 100644 index 0000000000..076c103611 --- /dev/null +++ b/modules/nf-core/modules/strelka/somatic/meta.yml @@ -0,0 +1,93 @@ +name: strelka_somatic +description: Strelka2 is a fast and accurate small variant caller optimized for analysis of germline variation in small cohorts and somatic variation in tumor/normal sample pairs +keywords: + - variant calling + - germline + - wgs + - vcf + - variants +tools: + - strelka: + description: Strelka calls somatic and germline small variants from mapped sequencing reads + homepage: https://github.com/Illumina/strelka + documentation: https://github.com/Illumina/strelka/blob/v2.9.x/docs/userGuide/README.md + tool_dev_url: https://github.com/Illumina/strelka + doi: 10.1038/s41592-018-0051-x + licence: ['GPL v3'] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - input_normal: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - input_index_normal: + type: file + description: BAM/CRAM/SAM index file + pattern: "*.{bai,crai,sai}" + - input_tumor: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + - input_index_tumor: + type: file + description: BAM/CRAM/SAM index file + pattern: "*.{bai,crai,sai}" + - manta_candidate_small_indels: + type: file + description: VCF.gz file + pattern: "*.{vcf.gz}" + - manta_candidate_small_indels_tbi: + type: file + description: VCF.gz index file + pattern: "*.tbi" + - fasta: + type: file + description: Genome reference FASTA file + pattern: "*.{fa,fasta}" + - fai: + type: file + description: Genome reference FASTA index file + pattern: "*.{fa.fai,fasta.fai}" + - target_bed: + type: file + description: BED file containing target regions for variant calling + pattern: "*.{bed}" + - target_bed_tbi: + type: file + description: Index for BED file containing target regions for variant calling + pattern: "*.{bed.tbi}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf_indels: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - vcf_indels_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - vcf_snvs: + type: file + description: Gzipped VCF file containing variants + pattern: "*.{vcf.gz}" + - vcf_snvs_tbi: + type: file + description: Index for gzipped VCF file containing variants + pattern: "*.{vcf.gz.tbi}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@drpatelh" From b70a44f780b5e22c00bbbc49fd19ef85f74402ef Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 16:25:40 +0100 Subject: [PATCH 24/28] fix: update output for versions --- modules/local/gatk4/applybqsr/main.nf | 2 +- modules/local/gatk4/applybqsrspark/main.nf | 2 +- modules/local/gatk4/baserecalibrator/main.nf | 2 +- modules/local/gatk4/baserecalibratorspark/main.nf | 2 +- .../local/gatk4/estimatelibrarycomplexity/main.nf | 2 +- modules/local/gatk4/gatherbqsrreports/main.nf | 4 ++-- modules/local/gatk4/genotypegvcf/main.nf | 2 +- modules/local/gatk4/haplotypecaller/main.nf | 2 +- modules/local/gatk4/markduplicatesspark/main.nf | 2 +- modules/local/gatk4/mutect2/merge_stats/main.nf | 4 ++-- modules/local/gatk4/mutect2/somatic/main.nf | 4 ++-- modules/local/gatk4/mutect2/tumor/main.nf | 4 ++-- subworkflows/local/germline_variant_calling.nf | 12 ++++++------ subworkflows/local/pair_variant_calling.nf | 10 +++++----- subworkflows/nf-core/mapping.nf | 8 ++++---- 15 files changed, 31 insertions(+), 31 deletions(-) diff --git a/modules/local/gatk4/applybqsr/main.nf b/modules/local/gatk4/applybqsr/main.nf index 9d275ad0a7..879a08d5db 100644 --- a/modules/local/gatk4/applybqsr/main.nf +++ b/modules/local/gatk4/applybqsr/main.nf @@ -26,7 +26,7 @@ process GATK4_APPLYBQSR { output: tuple val(meta), path("*.cram"), emit: cram - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/applybqsrspark/main.nf b/modules/local/gatk4/applybqsrspark/main.nf index f429e88b14..b6ac369f44 100644 --- a/modules/local/gatk4/applybqsrspark/main.nf +++ b/modules/local/gatk4/applybqsrspark/main.nf @@ -26,7 +26,7 @@ process GATK4_APPLYBQSR_SPARK { output: tuple val(meta), path("*.cram"), emit: cram - path "*.version.txt", emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/baserecalibrator/main.nf b/modules/local/gatk4/baserecalibrator/main.nf index e84913ba55..9b7dd37e01 100644 --- a/modules/local/gatk4/baserecalibrator/main.nf +++ b/modules/local/gatk4/baserecalibrator/main.nf @@ -28,7 +28,7 @@ process GATK4_BASERECALIBRATOR { output: tuple val(meta), path("*.table"), emit: table - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/baserecalibratorspark/main.nf b/modules/local/gatk4/baserecalibratorspark/main.nf index 5d02d04980..7c1fa5cd66 100644 --- a/modules/local/gatk4/baserecalibratorspark/main.nf +++ b/modules/local/gatk4/baserecalibratorspark/main.nf @@ -28,7 +28,7 @@ process GATK4_BASERECALIBRATOR_SPARK { output: tuple val(meta), path("*.table"), emit: table - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/estimatelibrarycomplexity/main.nf b/modules/local/gatk4/estimatelibrarycomplexity/main.nf index b09bae29f8..1e6fe047ed 100644 --- a/modules/local/gatk4/estimatelibrarycomplexity/main.nf +++ b/modules/local/gatk4/estimatelibrarycomplexity/main.nf @@ -26,7 +26,7 @@ process GATK4_ESTIMATELIBRARYCOMPLEXITY { output: path('*.md.metrics'), emit: metrics - path "*.version.txt", emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/gatherbqsrreports/main.nf b/modules/local/gatk4/gatherbqsrreports/main.nf index 1e69c75a20..ad54043ce6 100644 --- a/modules/local/gatk4/gatherbqsrreports/main.nf +++ b/modules/local/gatk4/gatherbqsrreports/main.nf @@ -23,8 +23,8 @@ process GATK4_GATHERBQSRREPORTS { output: tuple val(meta), path("${meta.sample}.recal.table"), emit: table - path "${meta.sample}.recal.table", emit: report - path "*.version.txt", emit: version + path "${meta.sample}.recal.table" , emit: report + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/genotypegvcf/main.nf b/modules/local/gatk4/genotypegvcf/main.nf index 261a9ca47a..37025fa97c 100644 --- a/modules/local/gatk4/genotypegvcf/main.nf +++ b/modules/local/gatk4/genotypegvcf/main.nf @@ -28,7 +28,7 @@ process GATK4_GENOTYPEGVCF { output: tuple val(meta), path("${interval.baseName}_${meta.id}.vcf"), emit: vcf - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: // Using -L is important for speed and we have to index the interval files also diff --git a/modules/local/gatk4/haplotypecaller/main.nf b/modules/local/gatk4/haplotypecaller/main.nf index 9bff975321..3e3e53aedb 100644 --- a/modules/local/gatk4/haplotypecaller/main.nf +++ b/modules/local/gatk4/haplotypecaller/main.nf @@ -30,7 +30,7 @@ process GATK4_HAPLOTYPECALLER { output: tuple val(meta), path("*.vcf") , emit: vcf tuple val(meta), path(interval), path("*.vcf"), emit: interval_vcf - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/markduplicatesspark/main.nf b/modules/local/gatk4/markduplicatesspark/main.nf index 60c3699753..3d8d0d6264 100644 --- a/modules/local/gatk4/markduplicatesspark/main.nf +++ b/modules/local/gatk4/markduplicatesspark/main.nf @@ -28,7 +28,7 @@ process GATK4_MARKDUPLICATES_SPARK { output: tuple val(meta), path("*.${format}"), emit: output - path("*.version.txt") , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/mutect2/merge_stats/main.nf b/modules/local/gatk4/mutect2/merge_stats/main.nf index 1c05e1afe3..ceed23a697 100644 --- a/modules/local/gatk4/mutect2/merge_stats/main.nf +++ b/modules/local/gatk4/mutect2/merge_stats/main.nf @@ -27,9 +27,9 @@ process GATK4_MUTECT2_SOMATIC { path(germline_resource_tbi) output: - tuple val(meta), path("*.vcf"), emit: vcf + tuple val(meta), path("*.vcf") , emit: vcf tuple val(meta), path("*.vcf.stats"), emit: vcf_stats - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/mutect2/somatic/main.nf b/modules/local/gatk4/mutect2/somatic/main.nf index 0c7205c3f6..be74ef101b 100644 --- a/modules/local/gatk4/mutect2/somatic/main.nf +++ b/modules/local/gatk4/mutect2/somatic/main.nf @@ -31,9 +31,9 @@ process GATK4_MUTECT2_SOMATIC { //target_bed? output: - tuple val(meta), path("*.vcf"), emit: vcf + tuple val(meta), path("*.vcf") , emit: vcf tuple val(meta), path("*.vcf.stats"), emit: vcf_stats - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/modules/local/gatk4/mutect2/tumor/main.nf b/modules/local/gatk4/mutect2/tumor/main.nf index e5b1ed440b..3ed4ea333e 100644 --- a/modules/local/gatk4/mutect2/tumor/main.nf +++ b/modules/local/gatk4/mutect2/tumor/main.nf @@ -30,9 +30,9 @@ process GATK4_MUTECT2_TUMOR { path(germline_resource_tbi) output: - tuple val(meta), path("*.vcf"), emit: vcf + tuple val(meta), path("*.vcf") , emit: vcf tuple val(meta), path("*.vcf.stats"), emit: vcf_stats - path "*.version.txt" , emit: version + path "versions.yml" , emit: versions script: def software = getSoftwareName(task.process) diff --git a/subworkflows/local/germline_variant_calling.nf b/subworkflows/local/germline_variant_calling.nf index 806e45ab87..26e46e769b 100644 --- a/subworkflows/local/germline_variant_calling.nf +++ b/subworkflows/local/germline_variant_calling.nf @@ -9,12 +9,12 @@ params.concat_gvcf_options = [:] params.concat_haplotypecaller_options = [:] params.strelka_options = [:] -include { GATK4_HAPLOTYPECALLER as HAPLOTYPECALLER } from '../../modules/local/gatk4/haplotypecaller/main' addParams(options: params.haplotypecaller_options) -include { DEEPVARIANT } from '../../modules/local/deepvariant/main' addParams(options: params.deepvariant_options) -include { GATK4_GENOTYPEGVCF as GENOTYPEGVCF } from '../../modules/local/gatk4/genotypegvcf/main' addParams(options: params.genotypegvcf_options) -include { CONCAT_VCF as CONCAT_GVCF } from '../../modules/local/concat_vcf/main' addParams(options: params.concat_gvcf_options) -include { CONCAT_VCF as CONCAT_HAPLOTYPECALLER } from '../../modules/local/concat_vcf/main' addParams(options: params.concat_haplotypecaller_options) -include { STRELKA_GERMLINE as STRELKA } from '../../modules/local/strelka/germline/main' addParams(options: params.strelka_options) +include { GATK4_HAPLOTYPECALLER as HAPLOTYPECALLER } from '../../modules/local/gatk4/haplotypecaller/main' addParams(options: params.haplotypecaller_options) +include { DEEPVARIANT } from '../../modules/local/deepvariant/main' addParams(options: params.deepvariant_options) +include { GATK4_GENOTYPEGVCF as GENOTYPEGVCF } from '../../modules/local/gatk4/genotypegvcf/main' addParams(options: params.genotypegvcf_options) +include { CONCAT_VCF as CONCAT_GVCF } from '../../modules/local/concat_vcf/main' addParams(options: params.concat_gvcf_options) +include { CONCAT_VCF as CONCAT_HAPLOTYPECALLER } from '../../modules/local/concat_vcf/main' addParams(options: params.concat_haplotypecaller_options) +include { STRELKA_GERMLINE as STRELKA } from '../../modules/nf-core/modules/strelka/germline/main' addParams(options: params.strelka_options) workflow GERMLINE_VARIANT_CALLING { take: diff --git a/subworkflows/local/pair_variant_calling.nf b/subworkflows/local/pair_variant_calling.nf index dc14143d15..9ba58b6db6 100644 --- a/subworkflows/local/pair_variant_calling.nf +++ b/subworkflows/local/pair_variant_calling.nf @@ -8,11 +8,11 @@ params.strelka_options = [:] params.strelka_bp_options = [:] params.mutect2_somatic_options = [:] -include { MANTA_SOMATIC as MANTA } from '../../modules/nf-core/software/manta/somatic/main' addParams(options: params.manta_options) -include { MSISENSORPRO_MSI } from '../../modules/nf-core/software/msisensorpro/msi/main' addParams(options: params.msisensorpro_msi_options) -include { STRELKA_SOMATIC as STRELKA } from '../../modules/nf-core/software/strelka/somatic/main' addParams(options: params.strelka_options) -include { STRELKA_SOMATIC_BEST_PRACTICES as STRELKA_BP } from '../../modules/nf-core/software/strelka/somaticbp/main' addParams(options: params.strelka_bp_options) -include { GATK4_MUTECT2_SOMATIC as MUTECT2 } from '../../modules/nf-core/software/gatk4/mutect2/somatic/main' addParams(options: params.mutect2_somatic_options) +include { MANTA_SOMATIC as MANTA } from '../../modules/nf-core/software/manta/somatic/main' addParams(options: params.manta_options) +include { MSISENSORPRO_MSI } from '../../modules/nf-core/software/msisensorpro/msi/main' addParams(options: params.msisensorpro_msi_options) +include { STRELKA_SOMATIC as STRELKA } from '../../modules/nf-core/software/strelka/somatic/main' addParams(options: params.strelka_options) +include { STRELKA_SOMATIC as STRELKA_BP } from '../../modules/nf-core/software/strelka/somatic/main' addParams(options: params.strelka_bp_options) +include { GATK4_MUTECT2_SOMATIC as MUTECT2 } from '../../modules/nf-core/software/gatk4/mutect2/somatic/main' addParams(options: params.mutect2_somatic_options) workflow PAIR_VARIANT_CALLING { take: diff --git a/subworkflows/nf-core/mapping.nf b/subworkflows/nf-core/mapping.nf index 6993fea9c4..10c6d13a9b 100644 --- a/subworkflows/nf-core/mapping.nf +++ b/subworkflows/nf-core/mapping.nf @@ -63,8 +63,8 @@ workflow MAPPING { bam_bwamem1_t = BWAMEM1_MEM_T.out.bam bam_bwamem1 = bam_bwamem1_n.mix(bam_bwamem1_t) - bwamem1_n_version = BWAMEM1_MEM.out.version - bwamem1_t_version = BWAMEM1_MEM_T.out.version + bwamem1_n_version = BWAMEM1_MEM.out.versions + bwamem1_t_version = BWAMEM1_MEM_T.out.versions bwamem1_version = bwamem1_n_version.mix(bwamem1_t_version).first() @@ -77,8 +77,8 @@ workflow MAPPING { bam_bwamem2_t = BWAMEM2_MEM_T.out.bam bam_bwamem2 = bam_bwamem2_n.mix(bam_bwamem2_t) - bwamem2_n_version = BWAMEM2_MEM.out.version - bwamem2_t_version = BWAMEM2_MEM_T.out.version + bwamem2_n_version = BWAMEM2_MEM.out.versions + bwamem2_t_version = BWAMEM2_MEM_T.out.versions bwamem2_version = bwamem2_n_version.mix(bwamem2_t_version).first() tool_versions = tool_versions.mix(bwamem2_version) From 16ccc5ccebc3c525584f6c54263c11abaff0a7c5 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 17:21:25 +0100 Subject: [PATCH 25/28] chores: updates more subworkflows --- modules/local/get_software_versions.nf | 34 -------- .../local/{ => tabix/bgziptabix}/functions.nf | 0 .../tabix/bgziptabix/main.nf | 2 +- .../tabix/bgziptabix/meta.yml | 0 .../modules/tabix/bgziptabix/functions.nf | 78 ------------------- .../nf-core/modules/tabix/tabix/functions.nf | 78 ------------------- modules/nf-core/modules/tabix/tabix/main.nf | 37 --------- modules/nf-core/modules/tabix/tabix/meta.yml | 41 ---------- subworkflows/local/annotate.nf | 44 ++++++----- subworkflows/nf-core/ensemblvep_annotate.nf | 9 +-- subworkflows/nf-core/fastqc_trimgalore.nf | 33 ++++---- subworkflows/nf-core/markduplicates.nf | 29 ++++++- subworkflows/nf-core/prepare_recalibration.nf | 2 +- subworkflows/nf-core/snpeff_annotate.nf | 9 +-- workflows/sarek.nf | 34 ++++---- 15 files changed, 88 insertions(+), 342 deletions(-) delete mode 100644 modules/local/get_software_versions.nf rename modules/local/{ => tabix/bgziptabix}/functions.nf (100%) rename modules/{nf-core/modules => local}/tabix/bgziptabix/main.nf (95%) rename modules/{nf-core/modules => local}/tabix/bgziptabix/meta.yml (100%) delete mode 100644 modules/nf-core/modules/tabix/bgziptabix/functions.nf delete mode 100644 modules/nf-core/modules/tabix/tabix/functions.nf delete mode 100644 modules/nf-core/modules/tabix/tabix/main.nf delete mode 100644 modules/nf-core/modules/tabix/tabix/meta.yml diff --git a/modules/local/get_software_versions.nf b/modules/local/get_software_versions.nf deleted file mode 100644 index b8be48ca88..0000000000 --- a/modules/local/get_software_versions.nf +++ /dev/null @@ -1,34 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process GET_SOFTWARE_VERSIONS { - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) } - - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/python:3.8.3" - } else { - container "quay.io/biocontainers/python:3.8.3" - } - - cache false - - input: - path versions - - output: - path "software_versions.tsv" , emit: tsv - path 'software_versions_mqc.yaml', emit: yaml - - script: // This script is bundled with the pipeline, in nf-core/sarek/bin/ - """ - echo $workflow.manifest.version > pipeline.version.txt - echo $workflow.nextflow.version > nextflow.version.txt - scrape_software_versions.py &> software_versions_mqc.yaml - """ -} diff --git a/modules/local/functions.nf b/modules/local/tabix/bgziptabix/functions.nf similarity index 100% rename from modules/local/functions.nf rename to modules/local/tabix/bgziptabix/functions.nf diff --git a/modules/nf-core/modules/tabix/bgziptabix/main.nf b/modules/local/tabix/bgziptabix/main.nf similarity index 95% rename from modules/nf-core/modules/tabix/bgziptabix/main.nf rename to modules/local/tabix/bgziptabix/main.nf index e44a722671..f94fdb0372 100644 --- a/modules/nf-core/modules/tabix/bgziptabix/main.nf +++ b/modules/local/tabix/bgziptabix/main.nf @@ -22,7 +22,7 @@ process TABIX_BGZIPTABIX { tuple val(meta), path(input) output: - tuple val(meta), path("*.gz"), path("*.tbi"), emit: tbi + tuple val(meta), path("*.gz"), path("*.tbi"), emit: gz_tbi path "versions.yml" , emit: versions script: diff --git a/modules/nf-core/modules/tabix/bgziptabix/meta.yml b/modules/local/tabix/bgziptabix/meta.yml similarity index 100% rename from modules/nf-core/modules/tabix/bgziptabix/meta.yml rename to modules/local/tabix/bgziptabix/meta.yml diff --git a/modules/nf-core/modules/tabix/bgziptabix/functions.nf b/modules/nf-core/modules/tabix/bgziptabix/functions.nf deleted file mode 100644 index 85628ee0eb..0000000000 --- a/modules/nf-core/modules/tabix/bgziptabix/functions.nf +++ /dev/null @@ -1,78 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Extract name of module from process name using $task.process -// -def getProcessName(task_process) { - return task_process.tokenize(':')[-1] -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - - // Do not publish versions.yml unless running from pytest workflow - if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { - return null - } - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } -} diff --git a/modules/nf-core/modules/tabix/tabix/functions.nf b/modules/nf-core/modules/tabix/tabix/functions.nf deleted file mode 100644 index 85628ee0eb..0000000000 --- a/modules/nf-core/modules/tabix/tabix/functions.nf +++ /dev/null @@ -1,78 +0,0 @@ -// -// Utility functions used in nf-core DSL2 module files -// - -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -// -// Extract name of module from process name using $task.process -// -def getProcessName(task_process) { - return task_process.tokenize(':')[-1] -} - -// -// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules -// -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.args3 = args.args3 ?: '' - options.publish_by_meta = args.publish_by_meta ?: [] - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -// -// Tidy up and join elements of a list to return a path string -// -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -// -// Function to save/publish module results -// -def saveFiles(Map args) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - - // Do not publish versions.yml unless running from pytest workflow - if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { - return null - } - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) - } - } - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } -} diff --git a/modules/nf-core/modules/tabix/tabix/main.nf b/modules/nf-core/modules/tabix/tabix/main.nf deleted file mode 100644 index 1574c0b558..0000000000 --- a/modules/nf-core/modules/tabix/tabix/main.nf +++ /dev/null @@ -1,37 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process TABIX_TABIX { - tag "$meta.id" - label 'process_medium' - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } - - conda (params.enable_conda ? 'bioconda::tabix=1.11' : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0" - } else { - container "quay.io/biocontainers/tabix:1.11--hdfd78af_0" - } - - input: - tuple val(meta), path(tab) - - output: - tuple val(meta), path("*.tbi"), emit: tbi - path "versions.yml" , emit: versions - - script: - """ - tabix $options.args $tab - - cat <<-END_VERSIONS > versions.yml - ${getProcessName(task.process)}: - ${getSoftwareName(task.process)}: \$(echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/ .*\$//') - END_VERSIONS - """ -} diff --git a/modules/nf-core/modules/tabix/tabix/meta.yml b/modules/nf-core/modules/tabix/tabix/meta.yml deleted file mode 100644 index 2e37c4ff90..0000000000 --- a/modules/nf-core/modules/tabix/tabix/meta.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: tabix_tabix -description: create tabix index from a sorted bgzip tab-delimited genome file -keywords: - - index - - tabix - - vcf -tools: - - tabix: - description: Generic indexer for TAB-delimited genome position files. - homepage: https://www.htslib.org/doc/tabix.html - documentation: https://www.htslib.org/doc/tabix.1.html - doi: 10.1093/bioinformatics/btq671 - licence: ['MIT'] -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - tab: - type: file - description: TAB-delimited genome position file compressed with bgzip - pattern: "*.{bed.gz,gff.gz,sam.gz,vcf.gz}" -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - tbi: - type: file - description: tabix index file - pattern: "*.{tbi}" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@joseespinosa" - - "@drpatelh" - - "@maxulysse" diff --git a/subworkflows/local/annotate.nf b/subworkflows/local/annotate.nf index 2841a277cd..9b1e7be36b 100644 --- a/subworkflows/local/annotate.nf +++ b/subworkflows/local/annotate.nf @@ -45,35 +45,37 @@ workflow ANNOTATE { vep_cache main: - merge_vcf_ann = Channel.empty() - merge_vep_report = Channel.empty() - merge_vep_version = Channel.empty() - snpeff_report = Channel.empty() - snpeff_vcf_ann = Channel.empty() - snpeff_version = Channel.empty() - vep_report = Channel.empty() - vep_vcf_ann = Channel.empty() - vep_version = Channel.empty() + ch_reports = Channel.empty() + ch_vcf_ann = Channel.empty() + ch_versions = Channel.empty() if ('snpeff' in tools || 'merge' in tools) { - (snpeff_vcf_ann, snpeff_report, snpeff_version) = SNPEFF_ANNOTATE(vcf, snpeff_db, snpeff_cache) + SNPEFF_ANNOTATE(vcf, snpeff_db, snpeff_cache) + + ch_reports = ch_reports.mix(SNPEFF_ANNOTATE.out.reports) + ch_vcf_ann = ch_vcf_ann.mix(SNPEFF_ANNOTATE.out.vcf_tbi) + ch_versions = ch_versions.mix(SNPEFF_ANNOTATE.out.versions.first()) } if ('merge' in tools) { - vcf_ann_for_merge = snpeff_vcf_ann.map{ meta, vcf, tbi -> [meta, vcf] } - (merge_vcf_ann, merge_vep_report, merge_vep_version) = MERGE_ANNOTATE(vcf_ann_for_merge, vep_genome, vep_species, vep_cache_version, vep_cache) + vcf_ann_for_merge = SNPEFF_ANNOTATE.out.vcf_tbi.map{ meta, vcf, tbi -> [meta, vcf] } + MERGE_ANNOTATE(vcf_ann_for_merge, vep_genome, vep_species, vep_cache_version, vep_cache) + + ch_reports = ch_reports.mix(MERGE_ANNOTATE.out.reports) + ch_vcf_ann = ch_vcf_ann.mix(MERGE_ANNOTATE.out.vcf_tbi) + ch_versions = ch_versions.mix(MERGE_ANNOTATE.out.versions.first()) } if ('vep' in tools) { - (vep_vcf_ann, vep_report, vep_version) = ENSEMBLVEP_ANNOTATE(vcf, vep_genome, vep_species, vep_cache_version, vep_cache) - } + ENSEMBLVEP_ANNOTATE(vcf, vep_genome, vep_species, vep_cache_version, vep_cache) - vcf_ann = snpeff_vcf_ann.mix(merge_vcf_ann, vep_vcf_ann) - reports = snpeff_report.mix(merge_vep_report, vep_report) - version = snpeff_version.first().mix(merge_vep_version.first(), vep_version.first()) + ch_reports = ch_reports.mix(ENSEMBLVEP_ANNOTATE.out.reports) + ch_vcf_ann = ch_vcf_ann.mix(ENSEMBLVEP_ANNOTATE.out.vcf_tbi) + ch_versions = ch_versions.mix(ENSEMBLVEP_ANNOTATE.out.versions.first()) + } emit: - reports - vcf_ann - version -} + vcf_ann = ch_vcf_ann // channel: [ val(meta), vcf.gz, vcf.gz.tbi ] + reports = ch_reports // path: *.html + versions = ch_versions // path: versions.yml +} \ No newline at end of file diff --git a/subworkflows/nf-core/ensemblvep_annotate.nf b/subworkflows/nf-core/ensemblvep_annotate.nf index a1e45771f9..9b1cbbc951 100644 --- a/subworkflows/nf-core/ensemblvep_annotate.nf +++ b/subworkflows/nf-core/ensemblvep_annotate.nf @@ -17,19 +17,18 @@ include { TABIX_BGZIPTABIX } from '../../modules/nf-core/modules/tabix/bgziptabi workflow ENSEMBLVEP_ANNOTATE { take: - vcf // channel: [ val(meta), vcf, tbi ] + vcf // channel: [ val(meta), vcf ] vep_genome // value: which genome vep_species // value: which species vep_cache_version // value: which cache version vep_cache // path: path_to_vep_cache (optionnal) - // skip // boolean: true/false main: ENSEMBLVEP(vcf, vep_genome, vep_species, vep_cache_version, vep_cache) TABIX_BGZIPTABIX(ENSEMBLVEP.out.vcf) emit: - vcf = TABIX_BGZIPTABIX.out.tbi // channel: [ val(meta), vcf, tbi ] - vep_report = ENSEMBLVEP.out.report // path: *.html - vep_version = ENSEMBLVEP.out.versions // path: *.version.txt + vcf_tbi = TABIX_BGZIPTABIX.out.gz_tbi // channel: [ val(meta), vcf.gz, vcf.gz.tbi ] + reports = ENSEMBLVEP.out.report // path: *.html + versions = ENSEMBLVEP.out.versions // path: versions.yml } diff --git a/subworkflows/nf-core/fastqc_trimgalore.nf b/subworkflows/nf-core/fastqc_trimgalore.nf index 576f5f4157..6044703400 100644 --- a/subworkflows/nf-core/fastqc_trimgalore.nf +++ b/subworkflows/nf-core/fastqc_trimgalore.nf @@ -5,7 +5,7 @@ params.fastqc_options = [:] params.trimgalore_options = [:] -include { FASTQC } from '../../modules/nf-core/modules/fastqc/main' addParams( options: params.fastqc_options ) +include { FASTQC } from '../../modules/nf-core/modules/fastqc/main' addParams( options: params.fastqc_options ) include { TRIMGALORE } from '../../modules/nf-core/modules/trimgalore/main' addParams( options: params.trimgalore_options ) workflow FASTQC_TRIMGALORE { @@ -15,28 +15,27 @@ workflow FASTQC_TRIMGALORE { skip_trimming // boolean: true/false main: - fastqc_html = Channel.empty() - fastqc_zip = Channel.empty() - fastqc_version = Channel.empty() + ch_versions = Channel.empty() + fastqc_html = Channel.empty() + fastqc_zip = Channel.empty() + if (!skip_fastqc) { - FASTQC(reads) - fastqc_html = FASTQC.out.html - fastqc_zip = FASTQC.out.zip - fastqc_version = FASTQC.out.versions.first() + FASTQC ( reads ).html.set { fastqc_html } + fastqc_zip = FASTQC.out.zip + ch_versions = ch_versions.mix(FASTQC.out.versions.first()) } trim_reads = reads trim_html = Channel.empty() trim_zip = Channel.empty() trim_log = Channel.empty() - trimgalore_version = Channel.empty() + if (!skip_trimming) { - TRIMGALORE(reads) - trim_reads = TRIMGALORE.out.reads - trim_html = TRIMGALORE.out.html - trim_zip = TRIMGALORE.out.zip - trim_log = TRIMGALORE.out.log - trimgalore_version = TRIMGALORE.out.versions.first() + TRIMGALORE ( reads ).reads.set { trim_reads } + trim_html = TRIMGALORE.out.html + trim_zip = TRIMGALORE.out.zip + trim_log = TRIMGALORE.out.log + ch_versions = ch_versions.mix(TRIMGALORE.out.versions.first()) } emit: @@ -44,10 +43,10 @@ workflow FASTQC_TRIMGALORE { fastqc_html // channel: [ val(meta), [ html ] ] fastqc_zip // channel: [ val(meta), [ zip ] ] - fastqc_version // path: *.versions.txt trim_html // channel: [ val(meta), [ html ] ] trim_zip // channel: [ val(meta), [ zip ] ] trim_log // channel: [ val(meta), [ txt ] ] - trimgalore_version // path: *.versions.txt + + versions = ch_versions.ifEmpty(null) // channel: [ versions.yml ] } diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index 370809ef89..1b85ba9ba1 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -20,6 +20,7 @@ include { SAMTOOLS_STATS } from '../../modules/nf-c include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM } from '../../modules/local/samtools/view/main' addParams(options: params.samtools_view_options) include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM_SPARK } from '../../modules/local/samtools/view/main' addParams(options: params.samtools_view_options) + workflow MARKDUPLICATES { take: bam_mapped // channel: [mandatory] meta, bam @@ -36,12 +37,15 @@ workflow MARKDUPLICATES { main: + ch_versions = Channel.empty() report_markduplicates = Channel.empty() if (skip_markduplicates) { bam_markduplicates = bam_indexed SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fasta_fai) cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram_crai + + ch_versions = ch_versions.mix(SAMTOOLS_BAM_TO_CRAM.out.versions.first()) } else { if (use_gatk_spark) { //If BAMQC should be run on MD output, then don't use MDSpark to convert to cram, but use bam output instead @@ -52,15 +56,24 @@ workflow MARKDUPLICATES { SAMTOOLS_BAM_TO_CRAM_SPARK(bam_markduplicates, fasta, fasta_fai) cram_markduplicates = SAMTOOLS_BAM_TO_CRAM_SPARK.out.cram_crai + + ch_versions = ch_versions.mix(GATK4_MARKDUPLICATES_SPARK.out.versions.first()) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) + ch_versions = ch_versions.mix(SAMTOOLS_BAM_TO_CRAM_SPARK.out.versions.first()) } else { GATK4_MARKDUPLICATES_SPARK(bam_mapped, fasta, fasta_fai, dict, "cram") SAMTOOLS_INDEX(GATK4_MARKDUPLICATES_SPARK.out.output) cram_markduplicates = GATK4_MARKDUPLICATES_SPARK.out.output.join(SAMTOOLS_INDEX.out.crai) + + ch_versions = ch_versions.mix(GATK4_MARKDUPLICATES_SPARK.out.versions.first()) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) } if (save_metrics) { GATK4_ESTIMATELIBRARYCOMPLEXITY(bam_mapped, fasta, fasta_fai, dict) report_markduplicates = GATK4_ESTIMATELIBRARYCOMPLEXITY.out.metrics + + ch_versions = ch_versions.mix(GATK4_ESTIMATELIBRARYCOMPLEXITY.out.versions.first()) } } else { @@ -70,6 +83,9 @@ workflow MARKDUPLICATES { SAMTOOLS_BAM_TO_CRAM(bam_markduplicates, fasta, fasta_fai) cram_markduplicates = SAMTOOLS_BAM_TO_CRAM.out.cram_crai + + ch_versions = ch_versions.mix(GATK4_MARKDUPLICATES.out.versions.first()) + ch_versions = ch_versions.mix(SAMTOOLS_BAM_TO_CRAM.out.versions.first()) } } @@ -78,15 +94,18 @@ workflow MARKDUPLICATES { //After bamqc finishes, convert to cram for further analysis samtools_stats = Channel.empty() if (!skip_samtools) { - SAMTOOLS_STATS(cram_markduplicates, fasta) + SAMTOOLS_STATS(cram_markduplicates, fasta, fasta_fai) samtools_stats = SAMTOOLS_STATS.out.stats + + ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first()) } qualimap_bamqc = Channel.empty() if (!skip_bamqc) { - bam_markduplicates_for_bamqc = bam_markduplicates.map { meta, bam, bai -> [meta, bam] } - QUALIMAP_BAMQC(bam_markduplicates_for_bamqc, target_bed, params.target_bed) + QUALIMAP_BAMQC(bam_markduplicates, target_bed, params.target_bed) qualimap_bamqc = QUALIMAP_BAMQC.out.results + + ch_versions = ch_versions.mix(QUALIMAP_BAMQC.out.versions.first()) } qc_reports = samtools_stats.mix(qualimap_bamqc) @@ -95,4 +114,6 @@ workflow MARKDUPLICATES { emit: cram = cram_markduplicates qc = qc_reports -} + + versions = ch_versions.ifEmpty(null) // channel: [ versions.yml ] +} \ No newline at end of file diff --git a/subworkflows/nf-core/prepare_recalibration.nf b/subworkflows/nf-core/prepare_recalibration.nf index 343d988699..d0d41165b1 100644 --- a/subworkflows/nf-core/prepare_recalibration.nf +++ b/subworkflows/nf-core/prepare_recalibration.nf @@ -61,4 +61,4 @@ workflow PREPARE_RECALIBRATION { emit: table_bqsr = table_bqsr versions = ch_versions // channel: [versions.yml] -} +} \ No newline at end of file diff --git a/subworkflows/nf-core/snpeff_annotate.nf b/subworkflows/nf-core/snpeff_annotate.nf index dd42b5e1e5..8589400b9a 100644 --- a/subworkflows/nf-core/snpeff_annotate.nf +++ b/subworkflows/nf-core/snpeff_annotate.nf @@ -17,17 +17,16 @@ include { TABIX_BGZIPTABIX } from '../../modules/nf-core/modules/tabix/bgziptabi workflow SNPEFF_ANNOTATE { take: - vcf // channel: [ val(meta), vcf, tbi ] + vcf // channel: [ val(meta), vcf ] snpeff_db // value: version of db to use snpeff_cache // path: path_to_snpeff_cache (optionnal) - // skip // boolean: true/false main: SNPEFF(vcf, snpeff_db, snpeff_cache) TABIX_BGZIPTABIX(SNPEFF.out.vcf) emit: - vcf = TABIX_BGZIPTABIX.out.tbi // channel: [ val(meta), vcf, tbi ] - snpeff_report = SNPEFF.out.report // path: *.html - snpeff_version = SNPEFF.out.versions // path: *.version.txt + vcf_tbi = TABIX_BGZIPTABIX.out.gz_tbi // channel: [ val(meta), vcf.gz, vcf.gz.tbi ] + reports = SNPEFF.out.reports // path: *.html + versions = SNPEFF.out.versions // path: versions.yml } diff --git a/workflows/sarek.nf b/workflows/sarek.nf index 4160bc7563..77e77a5d09 100644 --- a/workflows/sarek.nf +++ b/workflows/sarek.nf @@ -249,23 +249,23 @@ modules['multiqc'].args += params.multiqc_title ? Utils.joinModuleArgs(["--title // include { FASTQC_TRIMGALORE } from '../subworkflows/nf-core/fastqc_trimgalore' addParams( - fastqc_options: modules['fastqc'], - trimgalore_options: modules['trimgalore'] + fastqc_options: modules['fastqc'], + trimgalore_options: modules['trimgalore'] ) // // MODULES: Installed directly from nf-core/modules // -include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' addParams(options: modules['multiqc']) -include { GET_SOFTWARE_VERSIONS } from '../modules/local/get_software_versions' addParams(options: [publish_files : ['tsv':'']]) +include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' addParams( options: [publish_files : ['_versions.yml':'']] ) +include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' addParams( options: modules['multiqc'] ) def multiqc_report = [] workflow SAREK { - ch_software_versions = Channel.empty() - qc_reports = Channel.empty() + ch_versions = Channel.empty() + qc_reports = Channel.empty() // Build indices if needed PREPARE_GENOME( @@ -302,7 +302,7 @@ workflow SAREK { intervals.count().map{ num_intervals = it } // Get versions from all software used - ch_software_versions = ch_software_versions.mix(PREPARE_GENOME.out.versions.ifEmpty(null)) + ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) // PREPROCESSING @@ -331,8 +331,7 @@ workflow SAREK { qc_reports = qc_reports.mix(FASTQC_TRIMGALORE.out.trim_zip.collect{it[1]}.ifEmpty([])) // Get versions from all software used - ch_software_versions = ch_software_versions.mix(FASTQC_TRIMGALORE.out.fastqc_version.ifEmpty(null)) - ch_software_versions = ch_software_versions.mix(FASTQC_TRIMGALORE.out.trimgalore_version.ifEmpty(null)) + ch_versions = ch_versions.mix(FASTQC_TRIMGALORE.out.versions) // STEP 1: MAPPING READS TO REFERENCE GENOME MAPPING( @@ -350,6 +349,9 @@ workflow SAREK { // Create CSV to restart from this step MAPPING_CSV(bam_indexed, save_bam_mapped, params.skip_markduplicates) + + // Get versions from all software used + ch_versions = ch_versions.mix(MAPPING.out.versions) } if (step == 'preparerecalibration') { @@ -502,18 +504,10 @@ workflow SAREK { } } - ch_software_versions - .map { it -> if (it) [ it.baseName, it ] } - .groupTuple() - .map { it[1][0] } - .flatten() - .collect() - .set { ch_software_versions } - ch_version_yaml = Channel.empty() if (!('versions' in skip_qc)) { - GET_SOFTWARE_VERSIONS(ch_software_versions.map{it}.collect()) - ch_version_yaml = GET_SOFTWARE_VERSIONS.out.yaml.collect() + CUSTOM_DUMPSOFTWAREVERSIONS(ch_versions.unique().collectFile()) + ch_version_yaml = CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect() } workflow_summary = WorkflowSarek.paramsSummaryMultiqc(workflow, summary_params) @@ -529,7 +523,7 @@ workflow SAREK { multiqc_report = Channel.empty() if (!('multiqc' in skip_qc)) { MULTIQC(ch_multiqc_files.collect()) - multiqc_report = MULTIQC.out.report.toList() + multiqc_report = MULTIQC.out.report.toList() } } From a109e03e2eaa2a85fdf72bd96362b74f90a87a29 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 17:32:52 +0100 Subject: [PATCH 26/28] chores: update more modules --- .../qualimap/bamqc/functions.nf | 0 .../modules => local}/qualimap/bamqc/main.nf | 2 +- .../modules => local}/qualimap/bamqc/meta.yml | 0 .../modules/tabix/bgziptabix/functions.nf | 78 +++++++++++++++++++ .../nf-core/modules/tabix/bgziptabix/main.nf | 39 ++++++++++ .../nf-core/modules/tabix/bgziptabix/meta.yml | 45 +++++++++++ .../nf-core/modules/tabix/tabix/functions.nf | 78 +++++++++++++++++++ modules/nf-core/modules/tabix/tabix/main.nf | 37 +++++++++ modules/nf-core/modules/tabix/tabix/meta.yml | 41 ++++++++++ subworkflows/nf-core/ensemblvep_annotate.nf | 2 +- subworkflows/nf-core/markduplicates.nf | 4 +- subworkflows/nf-core/snpeff_annotate.nf | 2 +- 12 files changed, 323 insertions(+), 5 deletions(-) rename modules/{nf-core/modules => local}/qualimap/bamqc/functions.nf (100%) rename modules/{nf-core/modules => local}/qualimap/bamqc/main.nf (97%) rename modules/{nf-core/modules => local}/qualimap/bamqc/meta.yml (100%) create mode 100644 modules/nf-core/modules/tabix/bgziptabix/functions.nf create mode 100644 modules/nf-core/modules/tabix/bgziptabix/main.nf create mode 100644 modules/nf-core/modules/tabix/bgziptabix/meta.yml create mode 100644 modules/nf-core/modules/tabix/tabix/functions.nf create mode 100644 modules/nf-core/modules/tabix/tabix/main.nf create mode 100644 modules/nf-core/modules/tabix/tabix/meta.yml diff --git a/modules/nf-core/modules/qualimap/bamqc/functions.nf b/modules/local/qualimap/bamqc/functions.nf similarity index 100% rename from modules/nf-core/modules/qualimap/bamqc/functions.nf rename to modules/local/qualimap/bamqc/functions.nf diff --git a/modules/nf-core/modules/qualimap/bamqc/main.nf b/modules/local/qualimap/bamqc/main.nf similarity index 97% rename from modules/nf-core/modules/qualimap/bamqc/main.nf rename to modules/local/qualimap/bamqc/main.nf index d33f1e67bf..577a3c9e35 100644 --- a/modules/nf-core/modules/qualimap/bamqc/main.nf +++ b/modules/local/qualimap/bamqc/main.nf @@ -19,7 +19,7 @@ process QUALIMAP_BAMQC { } input: - tuple val(meta), path(bam) + tuple val(meta), path(bam), path(index) path gff val use_gff diff --git a/modules/nf-core/modules/qualimap/bamqc/meta.yml b/modules/local/qualimap/bamqc/meta.yml similarity index 100% rename from modules/nf-core/modules/qualimap/bamqc/meta.yml rename to modules/local/qualimap/bamqc/meta.yml diff --git a/modules/nf-core/modules/tabix/bgziptabix/functions.nf b/modules/nf-core/modules/tabix/bgziptabix/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/tabix/bgziptabix/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/tabix/bgziptabix/main.nf b/modules/nf-core/modules/tabix/bgziptabix/main.nf new file mode 100644 index 0000000000..e44a722671 --- /dev/null +++ b/modules/nf-core/modules/tabix/bgziptabix/main.nf @@ -0,0 +1,39 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process TABIX_BGZIPTABIX { + tag "$meta.id" + label 'process_medium' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? 'bioconda::tabix=1.11' : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0" + } else { + container "quay.io/biocontainers/tabix:1.11--hdfd78af_0" + } + + input: + tuple val(meta), path(input) + + output: + tuple val(meta), path("*.gz"), path("*.tbi"), emit: tbi + path "versions.yml" , emit: versions + + script: + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + """ + bgzip -c $options.args $input > ${prefix}.gz + tabix $options.args2 ${prefix}.gz + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + END_VERSIONS + """ +} diff --git a/modules/nf-core/modules/tabix/bgziptabix/meta.yml b/modules/nf-core/modules/tabix/bgziptabix/meta.yml new file mode 100644 index 0000000000..f2aed84d13 --- /dev/null +++ b/modules/nf-core/modules/tabix/bgziptabix/meta.yml @@ -0,0 +1,45 @@ +name: tabix_bgziptabix +description: bgzip a sorted tab-delimited genome file and then create tabix index +keywords: + - bgzip + - compress + - index + - tabix + - vcf +tools: + - tabix: + description: Generic indexer for TAB-delimited genome position files. + homepage: https://www.htslib.org/doc/tabix.html + documentation: https://www.htslib.org/doc/tabix.1.html + doi: 10.1093/bioinformatics/btq671 + licence: ['MIT'] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - tab: + type: file + description: TAB-delimited genome position file + pattern: "*.{bed,gff,sam,vcf}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - gz: + type: file + description: Output compressed file + pattern: "*.{gz}" + - tbi: + type: file + description: tabix index file + pattern: "*.{gz.tbi}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@maxulysse" diff --git a/modules/nf-core/modules/tabix/tabix/functions.nf b/modules/nf-core/modules/tabix/tabix/functions.nf new file mode 100644 index 0000000000..85628ee0eb --- /dev/null +++ b/modules/nf-core/modules/tabix/tabix/functions.nf @@ -0,0 +1,78 @@ +// +// Utility functions used in nf-core DSL2 module files +// + +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + +// +// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules +// +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_meta = args.publish_by_meta ?: [] + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +// +// Tidy up and join elements of a list to return a path string +// +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +// +// Function to save/publish module results +// +def saveFiles(Map args) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] + } + path = path instanceof String ? path : '' + path_list.add(path) + } + } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } +} diff --git a/modules/nf-core/modules/tabix/tabix/main.nf b/modules/nf-core/modules/tabix/tabix/main.nf new file mode 100644 index 0000000000..1574c0b558 --- /dev/null +++ b/modules/nf-core/modules/tabix/tabix/main.nf @@ -0,0 +1,37 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process TABIX_TABIX { + tag "$meta.id" + label 'process_medium' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + + conda (params.enable_conda ? 'bioconda::tabix=1.11' : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/tabix:1.11--hdfd78af_0" + } else { + container "quay.io/biocontainers/tabix:1.11--hdfd78af_0" + } + + input: + tuple val(meta), path(tab) + + output: + tuple val(meta), path("*.tbi"), emit: tbi + path "versions.yml" , emit: versions + + script: + """ + tabix $options.args $tab + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + ${getSoftwareName(task.process)}: \$(echo \$(tabix -h 2>&1) | sed 's/^.*Version: //; s/ .*\$//') + END_VERSIONS + """ +} diff --git a/modules/nf-core/modules/tabix/tabix/meta.yml b/modules/nf-core/modules/tabix/tabix/meta.yml new file mode 100644 index 0000000000..2e37c4ff90 --- /dev/null +++ b/modules/nf-core/modules/tabix/tabix/meta.yml @@ -0,0 +1,41 @@ +name: tabix_tabix +description: create tabix index from a sorted bgzip tab-delimited genome file +keywords: + - index + - tabix + - vcf +tools: + - tabix: + description: Generic indexer for TAB-delimited genome position files. + homepage: https://www.htslib.org/doc/tabix.html + documentation: https://www.htslib.org/doc/tabix.1.html + doi: 10.1093/bioinformatics/btq671 + licence: ['MIT'] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - tab: + type: file + description: TAB-delimited genome position file compressed with bgzip + pattern: "*.{bed.gz,gff.gz,sam.gz,vcf.gz}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - tbi: + type: file + description: tabix index file + pattern: "*.{tbi}" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" +authors: + - "@joseespinosa" + - "@drpatelh" + - "@maxulysse" diff --git a/subworkflows/nf-core/ensemblvep_annotate.nf b/subworkflows/nf-core/ensemblvep_annotate.nf index 9b1cbbc951..67cd0e116f 100644 --- a/subworkflows/nf-core/ensemblvep_annotate.nf +++ b/subworkflows/nf-core/ensemblvep_annotate.nf @@ -13,7 +13,7 @@ include { ENSEMBLVEP } from '../../modules/nf-core/modules/ensemblvep/main' addP vep_tag: params.vep_tag ) -include { TABIX_BGZIPTABIX } from '../../modules/nf-core/modules/tabix/bgziptabix/main' addParams(options: params.bgziptabix_vep_options) +include { TABIX_BGZIPTABIX } from '../../modules/local/tabix/bgziptabix/main' addParams(options: params.bgziptabix_vep_options) workflow ENSEMBLVEP_ANNOTATE { take: diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index 1b85ba9ba1..670886b63b 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -14,7 +14,7 @@ params.samtools_view_options = [:] include { GATK4_ESTIMATELIBRARYCOMPLEXITY } from '../../modules/local/gatk4/estimatelibrarycomplexity/main' addParams(options: params.estimatelibrarycomplexity_options) include { GATK4_MARKDUPLICATES } from '../../modules/local/gatk4/markduplicates/main' addParams(options: params.markduplicates_options) include { GATK4_MARKDUPLICATES_SPARK } from '../../modules/local/gatk4/markduplicatesspark/main' addParams(options: params.markduplicatesspark_options) -include { QUALIMAP_BAMQC } from '../../modules/nf-core/modules/qualimap/bamqc/main' addParams(options: params.qualimap_bamqc_options) +include { QUALIMAP_BAMQC } from '../../modules/local/qualimap/bamqc/main' addParams(options: params.qualimap_bamqc_options) include { SAMTOOLS_INDEX } from '../../modules/local/samtools/index/main' addParams(options: params.samtools_index_options) include { SAMTOOLS_STATS } from '../../modules/nf-core/modules/samtools/stats/main' addParams(options: params.samtools_stats_options) include { SAMTOOLS_VIEW as SAMTOOLS_BAM_TO_CRAM } from '../../modules/local/samtools/view/main' addParams(options: params.samtools_view_options) @@ -94,7 +94,7 @@ workflow MARKDUPLICATES { //After bamqc finishes, convert to cram for further analysis samtools_stats = Channel.empty() if (!skip_samtools) { - SAMTOOLS_STATS(cram_markduplicates, fasta, fasta_fai) + SAMTOOLS_STATS(cram_markduplicates, fasta) samtools_stats = SAMTOOLS_STATS.out.stats ch_versions = ch_versions.mix(SAMTOOLS_STATS.out.versions.first()) diff --git a/subworkflows/nf-core/snpeff_annotate.nf b/subworkflows/nf-core/snpeff_annotate.nf index 8589400b9a..2fbe4b21f6 100644 --- a/subworkflows/nf-core/snpeff_annotate.nf +++ b/subworkflows/nf-core/snpeff_annotate.nf @@ -13,7 +13,7 @@ include { SNPEFF } from '../../modules/nf-core/modules/snpeff/main' addParams( use_cache: params.use_cache ) -include { TABIX_BGZIPTABIX } from '../../modules/nf-core/modules/tabix/bgziptabix/main' addParams(options: params.bgziptabix_snpeff_options) +include { TABIX_BGZIPTABIX } from '../../modules/local/tabix/bgziptabix/main' addParams(options: params.bgziptabix_snpeff_options) workflow SNPEFF_ANNOTATE { take: From b36aae75995ebc8c0622cd24d3f19dd16c8451a6 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 17:35:41 +0100 Subject: [PATCH 27/28] fix: nf-core lint + EClint --- modules.json | 3 --- subworkflows/local/annotate.nf | 2 +- subworkflows/nf-core/markduplicates.nf | 2 +- subworkflows/nf-core/prepare_recalibration.nf | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/modules.json b/modules.json index faa90c9170..a832e38269 100644 --- a/modules.json +++ b/modules.json @@ -36,9 +36,6 @@ "multiqc": { "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" }, - "qualimap/bamqc": { - "git_sha": "3aacd46da2b221ed47aaa05c413a828538d2c2ae" - }, "samtools/faidx": { "git_sha": "19035c99d1cd8412b5046c6bb4af787658b84e5b" }, diff --git a/subworkflows/local/annotate.nf b/subworkflows/local/annotate.nf index 9b1e7be36b..78cd4c5588 100644 --- a/subworkflows/local/annotate.nf +++ b/subworkflows/local/annotate.nf @@ -78,4 +78,4 @@ workflow ANNOTATE { vcf_ann = ch_vcf_ann // channel: [ val(meta), vcf.gz, vcf.gz.tbi ] reports = ch_reports // path: *.html versions = ch_versions // path: versions.yml -} \ No newline at end of file +} diff --git a/subworkflows/nf-core/markduplicates.nf b/subworkflows/nf-core/markduplicates.nf index 670886b63b..d1ae4dd262 100644 --- a/subworkflows/nf-core/markduplicates.nf +++ b/subworkflows/nf-core/markduplicates.nf @@ -116,4 +116,4 @@ workflow MARKDUPLICATES { qc = qc_reports versions = ch_versions.ifEmpty(null) // channel: [ versions.yml ] -} \ No newline at end of file +} diff --git a/subworkflows/nf-core/prepare_recalibration.nf b/subworkflows/nf-core/prepare_recalibration.nf index d0d41165b1..343d988699 100644 --- a/subworkflows/nf-core/prepare_recalibration.nf +++ b/subworkflows/nf-core/prepare_recalibration.nf @@ -61,4 +61,4 @@ workflow PREPARE_RECALIBRATION { emit: table_bqsr = table_bqsr versions = ch_versions // channel: [versions.yml] -} \ No newline at end of file +} From 76df2dce3f0dc3e343d4c48347fcbad36cce8d31 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 24 Nov 2021 17:43:28 +0100 Subject: [PATCH 28/28] fix: output --- subworkflows/nf-core/snpeff_annotate.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/snpeff_annotate.nf b/subworkflows/nf-core/snpeff_annotate.nf index 2fbe4b21f6..033f36440f 100644 --- a/subworkflows/nf-core/snpeff_annotate.nf +++ b/subworkflows/nf-core/snpeff_annotate.nf @@ -27,6 +27,6 @@ workflow SNPEFF_ANNOTATE { emit: vcf_tbi = TABIX_BGZIPTABIX.out.gz_tbi // channel: [ val(meta), vcf.gz, vcf.gz.tbi ] - reports = SNPEFF.out.reports // path: *.html + reports = SNPEFF.out.report // path: *.html versions = SNPEFF.out.versions // path: versions.yml }