From 5707f6cf7885daa39c196b87519c01ceb924ac8e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 23 Apr 2024 09:49:15 +0200 Subject: [PATCH] migrate to nf-schema --- CHANGELOG.md | 1 + assets/schema_input.json | 11 +- main.nf | 6 +- nextflow.config | 13 +- nextflow_schema.json | 16 +- .../utils_nfcore_wisecondorx_pipeline/main.nf | 6 +- .../nf-core/utils_nfvalidation_plugin/main.nf | 6 +- workflows/cmgg-wisecondorx.nf | 279 ------------------ workflows/wisecondorx.nf | 2 +- 9 files changed, 34 insertions(+), 306 deletions(-) delete mode 100644 workflows/cmgg-wisecondorx.nf diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d48d5f..9fdb0dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changes 1. Updated the pipeline template to nf-core v2.13.1 +2. Migrate from nf-validation to nf-schema ## v1.1.0 - Naive Junior - [15 Sep 2023] diff --git a/assets/schema_input.json b/assets/schema_input.json index 321ff35..7787288 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-cmgg/wisecondorx/master/assets/schema_input.json", "title": "nf-cmgg/wisecondorx pipeline - params.input schema", "description": "Schema for the file provided with params.input", @@ -10,16 +10,19 @@ "cram": { "type": "string", "pattern": "^\\S+\\.(b|cr)am$", - "format": "file-path" + "format": "file-path", + "exists": true }, "crai": { "type": "string", "pattern": "^\\S+\\.(b|cr)ai$", - "format": "file-path" + "format": "file-path", + "exists": true }, "gender": { "type": "string", - "pattern": "^(male|female)$", + "errorMessage": "The gender needs to be either 'male' or 'female'", + "enum": ["male", "female"], "meta": ["gender"] } }, diff --git a/main.nf b/main.nf index 7bd71dc..5165c1a 100644 --- a/main.nf +++ b/main.nf @@ -25,9 +25,9 @@ params.fai = getGenomeAttribute('fai') IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { paramsSummaryLog } from 'plugin/nf-validation' -include { paramsHelp } from 'plugin/nf-validation' -include { validateParameters } from 'plugin/nf-validation' +include { paramsSummaryLog } from 'plugin/nf-schema' +include { paramsHelp } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' include { WISECONDORX } from './workflows/wisecondorx' include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_wisecondorx_pipeline' include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_wisecondorx_pipeline' diff --git a/nextflow.config b/nextflow.config index 7fe94e6..8a9c04d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -57,10 +57,6 @@ params { max_time = '240.h' // Schema validation default options - validationFailUnrecognisedParams = false - validationLenientMode = false - validationSchemaIgnoreParams = 'genomes,igenomes_base,test_data' - validationShowHiddenParams = false validate_params = true } @@ -192,7 +188,14 @@ singularity.registry = 'quay.io' // Nextflow plugins plugins { - id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.0.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} + +validation { + failUnrecognisedParams = false + lenientMode = false + defaultIgnoreParams = ['genomes','igenomes_base','test_data'] + showHiddenParams = false } // Load igenomes.config if required diff --git a/nextflow_schema.json b/nextflow_schema.json index 176557d..5d0f7f8 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-cmgg/wisecondorx/master/nextflow_schema.json", "title": "nf-cmgg/wisecondorx pipeline parameters", "description": "A nextflow pipeline for creating references for WisecondorX", "type": "object", - "definitions": { + "defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -325,22 +325,22 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/defs/input_output_options" }, { - "$ref": "#/definitions/other_options" + "$ref": "#/defs/other_options" }, { - "$ref": "#/definitions/reference_genome_options" + "$ref": "#/defs/reference_genome_options" }, { - "$ref": "#/definitions/institutional_config_options" + "$ref": "#/defs/institutional_config_options" }, { - "$ref": "#/definitions/max_job_request_options" + "$ref": "#/defs/max_job_request_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/defs/generic_options" } ] } diff --git a/subworkflows/local/utils_nfcore_wisecondorx_pipeline/main.nf b/subworkflows/local/utils_nfcore_wisecondorx_pipeline/main.nf index 9148e18..3758924 100644 --- a/subworkflows/local/utils_nfcore_wisecondorx_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_wisecondorx_pipeline/main.nf @@ -9,8 +9,8 @@ */ include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' -include { paramsSummaryMap } from 'plugin/nf-validation' -include { fromSamplesheet } from 'plugin/nf-validation' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' @@ -81,7 +81,7 @@ workflow PIPELINE_INITIALISATION { // Create channel from input file provided through params.input // - Channel.fromSamplesheet("input") + Channel.fromList(samplesheetToList(params.input, "assets/schema_input.json")) .set { ch_samplesheet } emit: diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf index 2585b65..50155c0 100644 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf @@ -8,9 +8,9 @@ ======================================================================================== */ -include { paramsHelp } from 'plugin/nf-validation' -include { paramsSummaryLog } from 'plugin/nf-validation' -include { validateParameters } from 'plugin/nf-validation' +include { paramsHelp } from 'plugin/nf-schema' +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' /* ======================================================================================== diff --git a/workflows/cmgg-wisecondorx.nf b/workflows/cmgg-wisecondorx.nf deleted file mode 100644 index 38d506a..0000000 --- a/workflows/cmgg-wisecondorx.nf +++ /dev/null @@ -1,279 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - IMPORT PLUGINS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -include { fromSamplesheet; paramsSummaryMap } from 'plugin/nf-validation' - -def summary_params = paramsSummaryMap(workflow) - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - VALIDATE INPUTS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -def checkPathParamList = [ - params.input, - params.multiqc_config, - params.fasta, - params.fai -] -for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true) } } - -// Check mandatory parameters -if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - CONFIG FILES -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() -ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() -ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - IMPORT NF-CORE MODULES/SUBWORKFLOWS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -// -// MODULE: Installed directly from nf-core/modules -// - -include { SAMTOOLS_FAIDX } from '../modules/nf-core/samtools/faidx/main' -include { SAMTOOLS_INDEX } from '../modules/nf-core/samtools/index/main' -include { NGSBITS_SAMPLEGENDER } from '../modules/nf-core/ngsbits/samplegender/main' -include { WISECONDORX_CONVERT } from '../modules/nf-core/wisecondorx/convert/main' -include { WISECONDORX_NEWREF } from '../modules/nf-core/wisecondorx/newref/main' -include { MULTIQC } from '../modules/nf-core/multiqc/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN MAIN WORKFLOW -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -// Info required for completion email and summary -def multiqc_report = [] - -workflow CMGGWISECONDORX { - - ch_versions = Channel.empty() - - // - // Create optional input files - // - - ch_fasta = Channel.fromPath(params.fasta, checkIfExists:true) - .map { [[id:params.genome ?: "fasta"], it] } - .collect() - - if(!params.fai) { - SAMTOOLS_FAIDX( - ch_fasta, - [[],[]] - ) - ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) - - SAMTOOLS_FAIDX.out.fai - .set { ch_fai } - } else { - ch_fai = Channel.fromPath(params.fai, checkIfExists:true) - .map { [[id:params.genome ?: "fasta"], it] } - .collect() - } - - // - // Validate and convert the samplesheet - // - - Channel.fromSamplesheet("input", immutable_meta: false) - .branch { meta, cram, crai -> - new_meta = meta + [id:cram.baseName] - indexed: crai - return [ new_meta, cram, crai ] - not_indexed: !crai - return [ new_meta, cram ] - } - .set { ch_input } - - // - // Index the non-indexed input files - // - - SAMTOOLS_INDEX(ch_input.not_indexed) - ch_versions = ch_versions.mix(SAMTOOLS_INDEX.out.versions.first()) - - ch_input.not_indexed - .join(SAMTOOLS_INDEX.out.bai, failOnDuplicate:true, failOnMismatch:true) - .mix(ch_input.indexed) - .set { ch_indexed } - - if(!params.no_metrics){ - - // - // Define the gender if it's not given - // - - ch_indexed - .branch { meta, cram, crai -> - gender: meta.gender - [ meta, meta.gender ] - no_gender: !meta.gender - } - .set { ch_ngsbits_input } - - NGSBITS_SAMPLEGENDER( - ch_ngsbits_input.no_gender, - ch_fasta, - ch_fai, - 'xy' - ) - ch_versions = ch_versions.mix(NGSBITS_SAMPLEGENDER.out.versions.first()) - - NGSBITS_SAMPLEGENDER.out.tsv - .map { meta, tsv -> - gender = get_gender(tsv) - new_meta = meta + [gender: gender] - [ new_meta, gender ] - } - .mix(ch_ngsbits_input.gender) - .set { ch_genders } - - // - // Create a small metrics file - // - - ch_genders - .reduce([:]) { counts, entry -> - meta = entry[0] - gender = entry[1] - counts[gender] = (counts[gender] ?: []) + meta.id - counts - } - .map { genders -> create_metrics(genders)} - - } - - // - // Convert the input files to NPZ files - // - - WISECONDORX_CONVERT( - ch_indexed, - ch_fasta, - ch_fai - ) - ch_versions = ch_versions.mix(WISECONDORX_CONVERT.out.versions.first()) - - // - // Create the WisecondorX reference - // - - // Define reference name (with timestamp) => only used when --prefix is null - Date date = new Date() - String dateFormat = "WisecondorX_${date.format("ddMMyyyy")}" - - WISECONDORX_CONVERT.out.npz - .map { meta, npz -> - new_meta = [id:params.prefix ?: dateFormat] - [ new_meta, npz ] - } - .groupTuple() // All files should be present here, so no size is needed - .combine(params.bin_sizes.split(",")) - .map { meta, npz, bin_size -> - new_meta = meta + [bin_size:bin_size] - [ new_meta, npz ] - } - .set { ch_newref_input } - - WISECONDORX_NEWREF(ch_newref_input) - ch_versions = ch_versions.mix(WISECONDORX_NEWREF.out.versions.first()) - - // - // Dump software versions - // - - CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') - ) - - // - // MODULE: MultiQC - // - workflow_summary = WorkflowCmggWisecondorx.paramsSummaryMultiqc(workflow, summary_params) - ch_workflow_summary = Channel.value(workflow_summary) - - methods_description = WorkflowCmggWisecondorx.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description) - ch_methods_description = Channel.value(methods_description) - - ch_multiqc_files = Channel.empty() - ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) - - MULTIQC ( - ch_multiqc_files.collect(), - ch_multiqc_config.toList(), - ch_multiqc_custom_config.toList(), - ch_multiqc_logo.toList() - ) - multiqc_report = MULTIQC.out.report.toList() -} - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - COMPLETION EMAIL AND SUMMARY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -workflow.onComplete { - if (params.email || params.email_on_fail) { - NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report) - } - NfcoreTemplate.summary(workflow, params, log) - if (params.hook_url) { - NfcoreTemplate.IM_notification(workflow, params, summary_params, projectDir, log) - } -} - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - FUNCTIONS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -def get_gender(tsv) { - split_tsv = tsv.splitCsv(sep:"\t", header:true, strip:true) - return split_tsv[0].gender -} - -def create_metrics(genders) { - male = genders["male"] - female = genders["female"] - male_count = male.size() - female_count = female.size() - total_count = male_count + female_count - ratio_male_to_female = male_count / female_count - - output = file("${params.outdir}/metrics.txt") - output.write("Ratio male to female: ${ratio_male_to_female}\n") - output.append("Male count: ${male_count}\n") - output.append("Female count: ${female_count}\n") - output.append("Total count: ${total_count}\n") - output.append("Males: ${male.join(',')}\n") - output.append("Females: ${female.join(',')}\n") -} - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - THE END -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ diff --git a/workflows/wisecondorx.nf b/workflows/wisecondorx.nf index 854413a..02b06f0 100644 --- a/workflows/wisecondorx.nf +++ b/workflows/wisecondorx.nf @@ -10,7 +10,7 @@ include { NGSBITS_SAMPLEGENDER } from '../modules/nf-core/ngsbits/sampleg include { WISECONDORX_CONVERT } from '../modules/nf-core/wisecondorx/convert/main' include { WISECONDORX_NEWREF } from '../modules/nf-core/wisecondorx/newref/main' include { MULTIQC } from '../modules/nf-core/multiqc/main' -include { paramsSummaryMap } from 'plugin/nf-validation' +include { paramsSummaryMap } from 'plugin/nf-schema' include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_wisecondorx_pipeline'