From 09d8b85f20335e05a180722f8d2ad8a24948497c Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 9 Dec 2024 14:40:20 +0100 Subject: [PATCH 01/14] fix subworkflows --- modules.json | 8 +- nextflow.config | 2 +- .../main.nf | 25 +- .../nf-core/utils_nextflow_pipeline/main.nf | 72 ++-- .../tests/main.workflow.nf.test | 10 +- .../tests/nextflow.config | 2 +- .../nf-core/utils_nfcore_pipeline/main.nf | 361 +++++++++--------- .../tests/main.function.nf.test | 46 +-- .../tests/main.function.nf.test.snap | 30 -- .../nf-core/utils_nfschema_plugin/main.nf | 46 +++ .../nf-core/utils_nfschema_plugin/meta.yml | 35 ++ .../utils_nfschema_plugin/tests/main.nf.test | 117 ++++++ .../tests/nextflow.config | 8 + .../tests/nextflow_schema.json | 8 +- .../nf-core/utils_nfvalidation_plugin/main.nf | 62 --- .../utils_nfvalidation_plugin/meta.yml | 44 --- .../tests/main.nf.test | 200 ---------- .../utils_nfvalidation_plugin/tests/tags.yml | 2 - workflows/sampletracking.nf | 2 +- 19 files changed, 455 insertions(+), 625 deletions(-) create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/main.nf create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/meta.yml create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test create mode 100644 subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config rename subworkflows/nf-core/{utils_nfvalidation_plugin => utils_nfschema_plugin}/tests/nextflow_schema.json (95%) delete mode 100644 subworkflows/nf-core/utils_nfvalidation_plugin/main.nf delete mode 100644 subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml delete mode 100644 subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test delete mode 100644 subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml diff --git a/modules.json b/modules.json index eb631bf..dd9c8ab 100644 --- a/modules.json +++ b/modules.json @@ -34,17 +34,17 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "c2b22d85f30a706a3073387f30380704fcae013b", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "51ae5406a030d4da1e49e4dab49756844fdd6c7a", "installed_by": ["subworkflows"] }, - "utils_nfvalidation_plugin": { + "utils_nfschema_plugin": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "2fd2cd6d0e7b273747f32e465fdc6bcc3ae0814e", "installed_by": ["subworkflows"] } } diff --git a/nextflow.config b/nextflow.config index 77a00b9..7bac49d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -182,7 +182,7 @@ singularity.registry = 'quay.io' // Nextflow plugins plugins { - id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.2.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } // Export these variables to prevent local Python/R libraries from conflicting with those in the container diff --git a/subworkflows/local/utils_nfcore_sampletracking_pipeline/main.nf b/subworkflows/local/utils_nfcore_sampletracking_pipeline/main.nf index 8514b67..9f5bb3a 100644 --- a/subworkflows/local/utils_nfcore_sampletracking_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_sampletracking_pipeline/main.nf @@ -8,17 +8,14 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' -include { paramsSummaryMap } from 'plugin/nf-validation' -include { fromSamplesheet } from 'plugin/nf-validation' +include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' -include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' -include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' include { imNotification } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' -include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' /* ======================================================================================== @@ -30,9 +27,7 @@ workflow PIPELINE_INITIALISATION { take: version // boolean: Display version and exit - help // boolean: Display help text validate_params // boolean: Boolean whether to validate parameters against the schema at runtime - monochrome_logs // boolean: Do not use coloured log outputs nextflow_cli_args // array: List of positional nextflow CLI args outdir // string: The output directory where the results will be saved input // string: Path to input samplesheet @@ -54,16 +49,10 @@ workflow PIPELINE_INITIALISATION { // // Validate parameters and generate parameter summary to stdout // - pre_help_text = nfCoreLogo(monochrome_logs) - post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) - def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " - UTILS_NFVALIDATION_PLUGIN ( - help, - workflow_command, - pre_help_text, - post_help_text, + UTILS_NFSCHEMA_PLUGIN ( + workflow, validate_params, - "nextflow_schema.json" + [] ) // @@ -81,7 +70,7 @@ workflow PIPELINE_INITIALISATION { // Create channel from input file provided through params.input // Channel - .fromSamplesheet("input") + .fromList(samplesheetToList(input, "assets/schema_input.json")) .map { meta, sample_bam, sample_bam_index, snp_fastq_1, snp_fastq_2, snp_bam, snp_bam_index -> return [ meta + ['single_end': false], sample_bam, sample_bam_index, [snp_fastq_1, snp_fastq_2].flatten(), snp_bam, snp_bam_index ] diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index ac31f28..d6e593e 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -2,18 +2,13 @@ // Subworkflow with functionality that may be useful for any Nextflow pipeline // -import org.yaml.snakeyaml.Yaml -import groovy.json.JsonOutput -import nextflow.extension.FilesEx - /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SUBWORKFLOW DEFINITION -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ workflow UTILS_NEXTFLOW_PIPELINE { - take: print_version // boolean: print version dump_parameters // boolean: dump parameters @@ -26,7 +21,7 @@ workflow UTILS_NEXTFLOW_PIPELINE { // Print workflow version and exit on --version // if (print_version) { - log.info "${workflow.manifest.name} ${getWorkflowVersion()}" + log.info("${workflow.manifest.name} ${getWorkflowVersion()}") System.exit(0) } @@ -49,16 +44,16 @@ workflow UTILS_NEXTFLOW_PIPELINE { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FUNCTIONS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // // Generate version string // def getWorkflowVersion() { - String version_string = "" + def version_string = "" as String if (workflow.manifest.version) { def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' version_string += "${prefix_v}${workflow.manifest.version}" @@ -76,13 +71,13 @@ def getWorkflowVersion() { // Dump pipeline parameters to a JSON file // def dumpParametersToJSON(outdir) { - def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def filename = "params_${timestamp}.json" - def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = JsonOutput.toJson(params) - temp_pf.text = JsonOutput.prettyPrint(jsonStr) + def timestamp = new java.util.Date().format('yyyy-MM-dd_HH-mm-ss') + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") + def jsonStr = groovy.json.JsonOutput.toJson(params) + temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) - FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") temp_pf.delete() } @@ -90,37 +85,42 @@ def dumpParametersToJSON(outdir) { // When running with -profile conda, warn if channels have not been set-up appropriately // def checkCondaChannels() { - Yaml parser = new Yaml() + def parser = new org.yaml.snakeyaml.Yaml() def channels = [] try { def config = parser.load("conda config --show channels".execute().text) channels = config.channels - } catch(NullPointerException | IOException e) { - log.warn "Could not verify conda channel configuration." - return + } + catch (NullPointerException e) { + log.debug(e) + log.warn("Could not verify conda channel configuration.") + return null + } + catch (IOException e) { + log.debug(e) + log.warn("Could not verify conda channel configuration.") + return null } // Check that all channels are present // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def required_channels_in_order = ['conda-forge', 'bioconda'] def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order - def channel_priority_violation = false - def n = required_channels_in_order.size() - for (int i = 0; i < n - 1; i++) { - channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) - } + def channel_priority_violation = required_channels_in_order != channels.findAll { ch -> ch in required_channels_in_order } if (channels_missing | channel_priority_violation) { - log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " There is a problem with your Conda configuration!\n\n" + - " You will need to set-up the conda-forge and bioconda channels correctly.\n" + - " Please refer to https://bioconda.github.io/\n" + - " The observed channel order is \n" + - " ${channels}\n" + - " but the following channel order is required:\n" + - " ${required_channels_in_order}\n" + - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + log.warn """\ + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + There is a problem with your Conda configuration! + You will need to set-up the conda-forge and bioconda channels correctly. + Please refer to https://bioconda.github.io/ + The observed channel order is + ${channels} + but the following channel order is required: + ${required_channels_in_order} + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + """.stripIndent(true) } } diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test index ca964ce..02dbf09 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -52,10 +52,12 @@ nextflow_workflow { } then { - assertAll( - { assert workflow.success }, - { assert workflow.stdout.contains("nextflow_workflow v9.9.9") } - ) + expect { + with(workflow) { + assert success + assert "nextflow_workflow v9.9.9" in stdout + } + } } } diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config index d0a926b..a09572e 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.04.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a8b55d6..bfd2587 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -2,17 +2,13 @@ // Subworkflow with utility functions specific to the nf-core pipeline template // -import org.yaml.snakeyaml.Yaml -import nextflow.extension.FilesEx - /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SUBWORKFLOW DEFINITION -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ workflow UTILS_NFCORE_PIPELINE { - take: nextflow_cli_args @@ -25,23 +21,20 @@ workflow UTILS_NFCORE_PIPELINE { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FUNCTIONS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // // Warn if a -profile or Nextflow config has not been provided to run the pipeline // def checkConfigProvided() { - valid_config = true + def valid_config = true as Boolean if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { - log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + - "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + - " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + - " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + - " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + - "Please refer to the quick start section and usage docs for the pipeline.\n " + log.warn( + "[${workflow.manifest.name}] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + "Please refer to the quick start section and usage docs for the pipeline.\n " + ) valid_config = false } return valid_config @@ -52,33 +45,22 @@ def checkConfigProvided() { // def checkProfileProvided(nextflow_cli_args) { if (workflow.profile.endsWith(',')) { - error "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + - "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + error( + "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + ) } if (nextflow_cli_args[0]) { - log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + - "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + log.warn( + "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + ) } } -// -// Citation string for pipeline -// -def workflowCitation() { - return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + - "* The pipeline\n" + - " ${workflow.manifest.doi}\n\n" + - "* The nf-core framework\n" + - " https://doi.org/10.1038/s41587-020-0439-x\n\n" + - "* Software dependencies\n" + - " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" -} - // // Generate workflow version string // def getWorkflowVersion() { - String version_string = "" + def version_string = "" as String if (workflow.manifest.version) { def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' version_string += "${prefix_v}${workflow.manifest.version}" @@ -96,8 +78,8 @@ def getWorkflowVersion() { // Get software versions for pipeline // def processVersionsFromYAML(yaml_file) { - Yaml yaml = new Yaml() - versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } + def yaml = new org.yaml.snakeyaml.Yaml() + def versions = yaml.load(yaml_file).collectEntries { k, v -> [k.tokenize(':')[-1], v] } return yaml.dumpAsMap(versions).trim() } @@ -107,8 +89,8 @@ def processVersionsFromYAML(yaml_file) { def workflowVersionToYAML() { return """ Workflow: - $workflow.manifest.name: ${getWorkflowVersion()} - Nextflow: $workflow.nextflow.version + ${workflow.manifest.name}: ${getWorkflowVersion()} + Nextflow: ${workflow.nextflow.version} """.stripIndent().trim() } @@ -116,11 +98,7 @@ def workflowVersionToYAML() { // Get channel of software versions used in pipeline in YAML format // def softwareVersionsToYAML(ch_versions) { - return ch_versions - .unique() - .map { processVersionsFromYAML(it) } - .unique() - .mix(Channel.of(workflowVersionToYAML())) + return ch_versions.unique().map { version -> processVersionsFromYAML(version) }.unique().mix(Channel.of(workflowVersionToYAML())) } // @@ -128,61 +106,40 @@ def softwareVersionsToYAML(ch_versions) { // def paramsSummaryMultiqc(summary_params) { def summary_section = '' - for (group in summary_params.keySet()) { - def group_params = summary_params.get(group) // This gets the parameters of that particular group - if (group_params) { - summary_section += "

$group

\n" - summary_section += "
\n" - for (param in group_params.keySet()) { - summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" + summary_params + .keySet() + .each { group -> + def group_params = summary_params.get(group) + // This gets the parameters of that particular group + if (group_params) { + summary_section += "

${group}

\n" + summary_section += "
\n" + group_params + .keySet() + .sort() + .each { param -> + summary_section += "
${param}
${group_params.get(param) ?: 'N/A'}
\n" + } + summary_section += "
\n" } - summary_section += "
\n" } - } - String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" - yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" - yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" - yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" - yaml_file_text += "plot_type: 'html'\n" - yaml_file_text += "data: |\n" - yaml_file_text += "${summary_section}" + def yaml_file_text = "id: '${workflow.manifest.name.replace('/', '-')}-summary'\n" as String + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" return yaml_file_text } -// -// nf-core logo -// -def nfCoreLogo(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) - String.format( - """\n - ${dashedLine(monochrome_logs)} - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} - ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} - ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} - ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} - ${dashedLine(monochrome_logs)} - """.stripIndent() - ) -} - -// -// Return dashed line -// -def dashedLine(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) - return "-${colors.dim}----------------------------------------------------${colors.reset}-" -} - // // ANSII colours used for terminal logging // def logColours(monochrome_logs=true) { - Map colorcodes = [:] + def colorcodes = [:] as Map // Reset / Meta colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" @@ -194,79 +151,76 @@ def logColours(monochrome_logs=true) { colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" // Regular Colors - colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" - colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" - colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" - colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" - colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" - colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" - colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" - colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" // Bold - colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" - colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" - colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" - colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" - colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" - colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" - colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" - colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" // Underline - colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" - colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" - colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" - colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" - colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" - colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" - colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" - colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" // High Intensity - colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" - colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" - colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" - colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" - colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" - colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" - colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" - colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" // Bold High Intensity - colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" - colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" - colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" - colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" - colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" - colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" - colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" - colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" return colorcodes } -// -// Attach the multiqc report to email -// -def attachMultiqcReport(multiqc_report) { - def mqc_report = null - try { - if (workflow.success) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" - } - mqc_report = mqc_report[0] - } - } - } catch (all) { - if (multiqc_report) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" +// Return a single report from an object that may be a Path or List +// +def getSingleReport(multiqc_reports) { + if (multiqc_reports instanceof Path) { + return multiqc_reports + } else if (multiqc_reports instanceof List) { + if (multiqc_reports.size() == 0) { + log.warn("[${workflow.manifest.name}] No reports found from process 'MULTIQC'") + return null + } else if (multiqc_reports.size() == 1) { + return multiqc_reports.first() + } else { + log.warn("[${workflow.manifest.name}] Found multiple reports from process 'MULTIQC', will use only one") + return multiqc_reports.first() } + } else { + return null } - return mqc_report } // @@ -275,26 +229,35 @@ def attachMultiqcReport(multiqc_report) { def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { // Set up the e-mail variables - def subject = "[$workflow.manifest.name] Successful: $workflow.runName" + def subject = "[${workflow.manifest.name}] Successful: ${workflow.runName}" if (!workflow.success) { - subject = "[$workflow.manifest.name] FAILED: $workflow.runName" + subject = "[${workflow.manifest.name}] FAILED: ${workflow.runName}" } def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } + summary_params + .keySet() + .sort() + .each { group -> + summary << summary_params[group] + } def misc_fields = [:] misc_fields['Date Started'] = workflow.start misc_fields['Date Completed'] = workflow.complete misc_fields['Pipeline script file path'] = workflow.scriptFile misc_fields['Pipeline script hash ID'] = workflow.scriptId - if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository - if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId - if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision - misc_fields['Nextflow Version'] = workflow.nextflow.version - misc_fields['Nextflow Build'] = workflow.nextflow.build + if (workflow.repository) { + misc_fields['Pipeline repository Git URL'] = workflow.repository + } + if (workflow.commitId) { + misc_fields['Pipeline repository Git Commit'] = workflow.commitId + } + if (workflow.revision) { + misc_fields['Pipeline Git branch/tag'] = workflow.revision + } + misc_fields['Nextflow Version'] = workflow.nextflow.version + misc_fields['Nextflow Build'] = workflow.nextflow.build misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp def email_fields = [:] @@ -311,7 +274,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi email_fields['summary'] = summary << misc_fields // On success try attach the multiqc report - def mqc_report = attachMultiqcReport(multiqc_report) + def mqc_report = getSingleReport(multiqc_report) // Check if we are only sending emails on failure def email_address = email @@ -331,40 +294,45 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as MemoryUnit + def smail_fields = [email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes()] def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) def sendmail_html = sendmail_template.toString() // Send the HTML e-mail - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map if (email_address) { try { - if (plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + if (plaintext_email) { + new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') + } // Try to send HTML e-mail using sendmail def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") sendmail_tf.withWriter { w -> w << sendmail_html } - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" - } catch (all) { + ['sendmail', '-t'].execute() << sendmail_html + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (sendmail)-") + } + catch (Exception msg) { + log.debug(msg.toString()) + log.debug("Trying with mail instead of sendmail") // Catch failures and try with plaintext - def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] + def mail_cmd = ['mail', '-s', subject, '--content-type=text/html', email_address] mail_cmd.execute() << email_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Sent summary e-mail to ${email_address} (mail)-") } } // Write summary e-mail HTML to a file def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); + nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html") output_hf.delete() // Write summary e-mail TXT to a file def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } - FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); + nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt") output_tf.delete() } @@ -372,15 +340,17 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi // Print pipeline summary on completion // def completionSummary(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map if (workflow.success) { if (workflow.stats.ignoredCount == 0) { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.green} Pipeline completed successfully${colors.reset}-") } - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" + else { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-") + } + } + else { + log.info("-${colors.purple}[${workflow.manifest.name}]${colors.red} Pipeline completed with errors${colors.reset}-") } } @@ -389,21 +359,30 @@ def completionSummary(monochrome_logs=true) { // def imNotification(summary_params, hook_url) { def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } + summary_params + .keySet() + .sort() + .each { group -> + summary << summary_params[group] + } def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) misc_fields['repository'] = workflow.repository - if (workflow.commitId) misc_fields['commitid'] = workflow.commitId - if (workflow.revision) misc_fields['revision'] = workflow.revision - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) { + misc_fields['repository'] = workflow.repository + } + if (workflow.commitId) { + misc_fields['commitid'] = workflow.commitId + } + if (workflow.revision) { + misc_fields['revision'] = workflow.revision + } + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp def msg_fields = [:] msg_fields['version'] = getWorkflowVersion() @@ -428,13 +407,13 @@ def imNotification(summary_params, hook_url) { def json_message = json_template.toString() // POST - def post = new URL(hook_url).openConnection(); + def post = new URL(hook_url).openConnection() post.setRequestMethod("POST") post.setDoOutput(true) post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")); - def postRC = post.getResponseCode(); - if (! postRC.equals(200)) { - log.warn(post.getErrorStream().getText()); + post.getOutputStream().write(json_message.getBytes("UTF-8")) + def postRC = post.getResponseCode() + if (!postRC.equals(200)) { + log.warn(post.getErrorStream().getText()) } } diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test index 1dc317f..f117040 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -41,26 +41,14 @@ nextflow_function { } } - test("Test Function workflowCitation") { - - function "workflowCitation" - - then { - assertAll( - { assert function.success }, - { assert snapshot(function.result).match() } - ) - } - } - - test("Test Function nfCoreLogo") { + test("Test Function without logColours") { - function "nfCoreLogo" + function "logColours" when { function { """ - input[0] = false + input[0] = true """ } } @@ -73,9 +61,8 @@ nextflow_function { } } - test("Test Function dashedLine") { - - function "dashedLine" + test("Test Function with logColours") { + function "logColours" when { function { @@ -93,14 +80,13 @@ nextflow_function { } } - test("Test Function without logColours") { - - function "logColours" + test("Test Function getSingleReport with a single file") { + function "getSingleReport" when { function { """ - input[0] = true + input[0] = file(params.modules_testdata_base_path + '/generic/tsv/test.tsv', checkIfExists: true) """ } } @@ -108,18 +94,22 @@ nextflow_function { then { assertAll( { assert function.success }, - { assert snapshot(function.result).match() } + { assert function.result.contains("test.tsv") } ) } } - test("Test Function with logColours") { - function "logColours" + test("Test Function getSingleReport with multiple files") { + function "getSingleReport" when { function { """ - input[0] = false + input[0] = [ + file(params.modules_testdata_base_path + '/generic/tsv/test.tsv', checkIfExists: true), + file(params.modules_testdata_base_path + '/generic/tsv/network.tsv', checkIfExists: true), + file(params.modules_testdata_base_path + '/generic/tsv/expression.tsv', checkIfExists: true) + ] """ } } @@ -127,7 +117,9 @@ nextflow_function { then { assertAll( { assert function.success }, - { assert snapshot(function.result).match() } + { assert function.result.contains("test.tsv") }, + { assert !function.result.contains("network.tsv") }, + { assert !function.result.contains("expression.tsv") } ) } } diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap index 1037232..02c6701 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -17,26 +17,6 @@ }, "timestamp": "2024-02-28T12:02:59.729647" }, - "Test Function nfCoreLogo": { - "content": [ - "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow v9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T12:03:10.562934" - }, - "Test Function workflowCitation": { - "content": [ - "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T12:03:07.019761" - }, "Test Function without logColours": { "content": [ { @@ -95,16 +75,6 @@ }, "timestamp": "2024-02-28T12:03:17.969323" }, - "Test Function dashedLine": { - "content": [ - "-\u001b[2m----------------------------------------------------\u001b[0m-" - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-02-28T12:03:14.366181" - }, "Test Function with logColours": { "content": [ { diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf new file mode 100644 index 0000000..4994303 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -0,0 +1,46 @@ +// +// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary +// + +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' + +workflow UTILS_NFSCHEMA_PLUGIN { + + take: + input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow + validate_params // boolean: validate the parameters + parameters_schema // string: path to the parameters JSON schema. + // this has to be the same as the schema given to `validation.parametersSchema` + // when this input is empty it will automatically use the configured schema or + // "${projectDir}/nextflow_schema.json" as default. This input should not be empty + // for meta pipelines + + main: + + // + // Print parameter summary to stdout. This will display the parameters + // that differ from the default given in the JSON schema + // + if(parameters_schema) { + log.info paramsSummaryLog(input_workflow, parameters_schema:parameters_schema) + } else { + log.info paramsSummaryLog(input_workflow) + } + + // + // Validate the parameters using nextflow_schema.json or the schema + // given via the validation.parametersSchema configuration option + // + if(validate_params) { + if(parameters_schema) { + validateParameters(parameters_schema:parameters_schema) + } else { + validateParameters() + } + } + + emit: + dummy_emit = true +} + diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml new file mode 100644 index 0000000..f7d9f02 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "utils_nfschema_plugin" +description: Run nf-schema to validate parameters and create a summary of changed parameters +keywords: + - validation + - JSON schema + - plugin + - parameters + - summary +components: [] +input: + - input_workflow: + type: object + description: | + The workflow object of the used pipeline. + This object contains meta data used to create the params summary log + - validate_params: + type: boolean + description: Validate the parameters and error if invalid. + - parameters_schema: + type: string + description: | + Path to the parameters JSON schema. + This has to be the same as the schema given to the `validation.parametersSchema` config + option. When this input is empty it will automatically use the configured schema or + "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way + for meta pipelines. +output: + - dummy_emit: + type: boolean + description: Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nvnieuwk" +maintainers: + - "@nvnieuwk" diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test new file mode 100644 index 0000000..8fb3016 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -0,0 +1,117 @@ +nextflow_workflow { + + name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" + script "../main.nf" + workflow "UTILS_NFSCHEMA_PLUGIN" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/utils_nfschema_plugin" + tag "plugin/nf-schema" + + config "./nextflow.config" + + test("Should run nothing") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params") { + + when { + + params { + test_data = '' + outdir = null + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } + + test("Should run nothing - custom schema") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params - custom schema") { + + when { + + params { + test_data = '' + outdir = null + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config new file mode 100644 index 0000000..0907ac5 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -0,0 +1,8 @@ +plugins { + id "nf-schema@2.1.0" +} + +validation { + parametersSchema = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + monochromeLogs = true +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json similarity index 95% rename from subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json rename to subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json index 7626c1c..331e0d2 100644 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", "title": ". pipeline parameters", "description": "", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -87,10 +87,10 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf deleted file mode 100644 index 2585b65..0000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf +++ /dev/null @@ -1,62 +0,0 @@ -// -// Subworkflow that uses the nf-validation plugin to render help text and parameter summary -// - -/* -======================================================================================== - IMPORT NF-VALIDATION PLUGIN -======================================================================================== -*/ - -include { paramsHelp } from 'plugin/nf-validation' -include { paramsSummaryLog } from 'plugin/nf-validation' -include { validateParameters } from 'plugin/nf-validation' - -/* -======================================================================================== - SUBWORKFLOW DEFINITION -======================================================================================== -*/ - -workflow UTILS_NFVALIDATION_PLUGIN { - - take: - print_help // boolean: print help - workflow_command // string: default commmand used to run pipeline - pre_help_text // string: string to be printed before help text and summary log - post_help_text // string: string to be printed after help text and summary log - validate_params // boolean: validate parameters - schema_filename // path: JSON schema file, null to use default value - - main: - - log.debug "Using schema file: ${schema_filename}" - - // Default values for strings - pre_help_text = pre_help_text ?: '' - post_help_text = post_help_text ?: '' - workflow_command = workflow_command ?: '' - - // - // Print help message if needed - // - if (print_help) { - log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text - System.exit(0) - } - - // - // Print parameter summary to stdout - // - log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text - - // - // Validate parameters relative to the parameter JSON schema - // - if (validate_params){ - validateParameters(parameters_schema: schema_filename) - } - - emit: - dummy_emit = true -} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml deleted file mode 100644 index 3d4a6b0..0000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml +++ /dev/null @@ -1,44 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "UTILS_NFVALIDATION_PLUGIN" -description: Use nf-validation to initiate and validate a pipeline -keywords: - - utility - - pipeline - - initialise - - validation -components: [] -input: - - print_help: - type: boolean - description: | - Print help message and exit - - workflow_command: - type: string - description: | - The command to run the workflow e.g. "nextflow run main.nf" - - pre_help_text: - type: string - description: | - Text to print before the help message - - post_help_text: - type: string - description: | - Text to print after the help message - - validate_params: - type: boolean - description: | - Validate the parameters and error if invalid. - - schema_filename: - type: string - description: | - The filename of the schema to validate against. -output: - - dummy_emit: - type: boolean - description: | - Dummy emit to make nf-core subworkflows lint happy -authors: - - "@adamrtalbot" -maintainers: - - "@adamrtalbot" - - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test deleted file mode 100644 index 5784a33..0000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test +++ /dev/null @@ -1,200 +0,0 @@ -nextflow_workflow { - - name "Test Workflow UTILS_NFVALIDATION_PLUGIN" - script "../main.nf" - workflow "UTILS_NFVALIDATION_PLUGIN" - tag "subworkflows" - tag "subworkflows_nfcore" - tag "plugin/nf-validation" - tag "'plugin/nf-validation'" - tag "utils_nfvalidation_plugin" - tag "subworkflows/utils_nfvalidation_plugin" - - test("Should run nothing") { - - when { - - params { - monochrome_logs = true - test_data = '' - } - - workflow { - """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success } - ) - } - } - - test("Should run help") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with command") { - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with extra text") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = "pre-help-text" - post_help_text = "post-help-text" - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('pre-help-text') } }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } }, - { assert workflow.stdout.any { it.contains('post-help-text') } } - ) - } - } - - test("Should validate params") { - - when { - - params { - monochrome_logs = true - test_data = '' - outdir = 1 - } - workflow { - """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = true - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.failed }, - { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } - ) - } - } -} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml deleted file mode 100644 index 60b1cff..0000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -subworkflows/utils_nfvalidation_plugin: - - subworkflows/nf-core/utils_nfvalidation_plugin/** diff --git a/workflows/sampletracking.nf b/workflows/sampletracking.nf index d0e4325..b1e6a23 100644 --- a/workflows/sampletracking.nf +++ b/workflows/sampletracking.nf @@ -7,7 +7,7 @@ include { BWA_MEM } from '../modules/nf-core/bwa/mem/main' include { PICARD_CROSSCHECKFINGERPRINTS } from '../modules/nf-core/picard/crosscheckfingerprints/main' include { MULTIQC } from '../modules/nf-core/multiqc/main' -include { paramsSummaryMap } from 'plugin/nf-validation' +include { paramsSummaryMap } from 'plugin/nf-schema' include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_sampletracking_pipeline' From eb6a26ebd254090e81acfd5f5138883d4d864ddf Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Mon, 9 Dec 2024 14:45:51 +0100 Subject: [PATCH 02/14] update config and modules --- conf/empty_genomes.config | 3 + modules.json | 9 +- modules/nf-core/bwa/index/environment.yml | 4 +- modules/nf-core/bwa/index/main.nf | 8 +- modules/nf-core/bwa/index/meta.yml | 47 +-- .../nf-core/bwa/index/tests/main.nf.test.snap | 10 +- modules/nf-core/bwa/mem/environment.yml | 10 +- modules/nf-core/bwa/mem/main.nf | 6 +- modules/nf-core/bwa/mem/meta.yml | 113 +++--- modules/nf-core/bwa/mem/tests/main.nf.test | 174 ++++----- .../nf-core/bwa/mem/tests/main.nf.test.snap | 332 ++++++++---------- modules/nf-core/multiqc/environment.yml | 4 +- modules/nf-core/multiqc/main.nf | 16 +- modules/nf-core/multiqc/meta.yml | 78 ++-- modules/nf-core/multiqc/multiqc.diff | 9 +- modules/nf-core/multiqc/tests/main.nf.test | 8 + .../nf-core/multiqc/tests/main.nf.test.snap | 24 +- modules/nf-core/multiqc/tests/nextflow.config | 5 + nextflow.config | 77 ++-- 19 files changed, 463 insertions(+), 474 deletions(-) create mode 100644 conf/empty_genomes.config create mode 100644 modules/nf-core/multiqc/tests/nextflow.config diff --git a/conf/empty_genomes.config b/conf/empty_genomes.config new file mode 100644 index 0000000..a90c75d --- /dev/null +++ b/conf/empty_genomes.config @@ -0,0 +1,3 @@ +// DON'T REMOVE THIS FILE! + +params.genomes = [:] diff --git a/modules.json b/modules.json index dd9c8ab..19ce99e 100644 --- a/modules.json +++ b/modules.json @@ -7,18 +7,17 @@ "nf-core": { "bwa/index": { "branch": "master", - "git_sha": "6278bf9afd4a4b2d00fa6052250e73da3d91546f", + "git_sha": "2d20463181b1c38981a02e90d3084b5f9fa8d540", "installed_by": ["modules"] }, "bwa/mem": { "branch": "master", - "git_sha": "b97197968ac12dde2463fa54541f6350c46f2035", - "installed_by": ["modules"], - "patch": "modules/nf-core/bwa/mem/bwa-mem.diff" + "git_sha": "2d20463181b1c38981a02e90d3084b5f9fa8d540", + "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", + "git_sha": "cf17ca47590cc578dfb47db1c2a44ef86f89976d", "installed_by": ["modules"], "patch": "modules/nf-core/multiqc/multiqc.diff" }, diff --git a/modules/nf-core/bwa/index/environment.yml b/modules/nf-core/bwa/index/environment.yml index 5d3cb32..d8789a2 100644 --- a/modules/nf-core/bwa/index/environment.yml +++ b/modules/nf-core/bwa/index/environment.yml @@ -1,7 +1,5 @@ -name: bwa_index channels: - conda-forge - bioconda - - defaults dependencies: - - bioconda::bwa=0.7.17 + - bioconda::bwa=0.7.18 diff --git a/modules/nf-core/bwa/index/main.nf b/modules/nf-core/bwa/index/main.nf index 24b5a2e..29d9957 100644 --- a/modules/nf-core/bwa/index/main.nf +++ b/modules/nf-core/bwa/index/main.nf @@ -4,15 +4,15 @@ process BWA_INDEX { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bwa:0.7.17--hed695b0_7' : - 'biocontainers/bwa:0.7.17--hed695b0_7' }" + 'https://depot.galaxyproject.org/singularity/bwa:0.7.18--he4a0461_0' : + 'biocontainers/bwa:0.7.18--he4a0461_0' }" input: tuple val(meta), path(fasta) output: - tuple val(meta), path(bwa) , emit: index - path "versions.yml" , emit: versions + tuple val(meta), path("bwa") , emit: index + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/bwa/index/meta.yml b/modules/nf-core/bwa/index/meta.yml index 730628d..ce5cb8f 100644 --- a/modules/nf-core/bwa/index/meta.yml +++ b/modules/nf-core/bwa/index/meta.yml @@ -11,35 +11,42 @@ tools: BWA is a software package for mapping DNA sequences against a large reference genome, such as the human genome. homepage: http://bio-bwa.sourceforge.net/ - documentation: http://www.htslib.org/doc/samtools.html + documentation: https://bio-bwa.sourceforge.net/bwa.shtml arxiv: arXiv:1303.3997 licence: ["GPL-3.0-or-later"] + identifier: "" input: - - meta: - type: map - description: | - Groovy Map containing reference information. - e.g. [ id:'test', single_end:false ] - - fasta: - type: file - description: Input genome fasta file + - - meta: + type: map + description: | + Groovy Map containing reference information. + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file output: - - meta: - type: map - description: | - Groovy Map containing reference information. - e.g. [ id:'test', single_end:false ] - index: - type: file - description: BWA genome index files - pattern: "*.{amb,ann,bwt,pac,sa}" + - meta: + type: map + description: | + Groovy Map containing reference information. + e.g. [ id:'test', single_end:false ] + pattern: "*.{amb,ann,bwt,pac,sa}" + - bwa: + type: map + description: | + Groovy Map containing reference information. + e.g. [ id:'test', single_end:false ] + pattern: "*.{amb,ann,bwt,pac,sa}" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@maxulysse" maintainers: - "@drpatelh" - "@maxulysse" + - "@gallvp" diff --git a/modules/nf-core/bwa/index/tests/main.nf.test.snap b/modules/nf-core/bwa/index/tests/main.nf.test.snap index e51ad5b..7c8f046 100644 --- a/modules/nf-core/bwa/index/tests/main.nf.test.snap +++ b/modules/nf-core/bwa/index/tests/main.nf.test.snap @@ -17,7 +17,7 @@ ] ], "1": [ - "versions.yml:md5,0f20525da90e7489a7ebb02adca3265f" + "versions.yml:md5,a64462ac7dfb21f4ade9b02e7f65c5bb" ], "index": [ [ @@ -34,10 +34,14 @@ ] ], "versions": [ - "versions.yml:md5,0f20525da90e7489a7ebb02adca3265f" + "versions.yml:md5,a64462ac7dfb21f4ade9b02e7f65c5bb" ] } ], - "timestamp": "2023-10-17T17:20:20.180927714" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-05-16T11:40:09.925307" } } \ No newline at end of file diff --git a/modules/nf-core/bwa/mem/environment.yml b/modules/nf-core/bwa/mem/environment.yml index 3f136d0..ef7b966 100644 --- a/modules/nf-core/bwa/mem/environment.yml +++ b/modules/nf-core/bwa/mem/environment.yml @@ -1,10 +1,8 @@ -name: bwa_mem channels: - conda-forge - bioconda - - defaults + dependencies: - - bwa=0.7.17 - # renovate: datasource=conda depName=bioconda/samtools - - samtools=1.19.2 - - htslib=1.19.1 + - bwa=0.7.18 + - htslib=1.20.0 + - samtools=1.20 diff --git a/modules/nf-core/bwa/mem/main.nf b/modules/nf-core/bwa/mem/main.nf index d8bd281..d18cd93 100644 --- a/modules/nf-core/bwa/mem/main.nf +++ b/modules/nf-core/bwa/mem/main.nf @@ -4,8 +4,8 @@ process BWA_MEM { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:a34558545ae1413d94bde4578787ebef08027945-0' : - 'biocontainers/mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:a34558545ae1413d94bde4578787ebef08027945-0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:1bd8542a8a0b42e0981337910954371d0230828e-0' : + 'biocontainers/mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:1bd8542a8a0b42e0981337910954371d0230828e-0' }" input: tuple val(meta) , path(reads) @@ -53,10 +53,8 @@ process BWA_MEM { """ stub: - def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def samtools_command = sort_bam ? 'sort' : 'view' def extension = args2.contains("--output-fmt sam") ? "sam" : args2.contains("--output-fmt cram") ? "cram": sort_bam && args2.contains("-O cram")? "cram": diff --git a/modules/nf-core/bwa/mem/meta.yml b/modules/nf-core/bwa/mem/meta.yml index 1532c26..37467d2 100644 --- a/modules/nf-core/bwa/mem/meta.yml +++ b/modules/nf-core/bwa/mem/meta.yml @@ -14,58 +14,85 @@ tools: BWA is a software package for mapping DNA sequences against a large reference genome, such as the human genome. homepage: http://bio-bwa.sourceforge.net/ - documentation: http://www.htslib.org/doc/samtools.html + documentation: https://bio-bwa.sourceforge.net/bwa.shtml arxiv: arXiv:1303.3997 licence: ["GPL-3.0-or-later"] + identifier: "" input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - reads: - type: file - description: | - List of input FastQ files of size 1 and 2 for single-end and paired-end data, - respectively. - - meta2: - type: map - description: | - Groovy Map containing reference information. - e.g. [ id:'test', single_end:false ] - - index: - type: file - description: BWA genome index files - pattern: "Directory containing BWA index *.{amb,ann,bwt,pac,sa}" - - fasta: - type: file - description: Reference genome in FASTA format - pattern: "*.{fasta,fa}" - - sort_bam: - type: boolean - description: use samtools sort (true) or samtools view (false) - pattern: "true or false" + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. + - - meta2: + type: map + description: | + Groovy Map containing reference information. + e.g. [ id:'test', single_end:false ] + - index: + type: file + description: BWA genome index files + pattern: "Directory containing BWA index *.{amb,ann,bwt,pac,sa}" + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Reference genome in FASTA format + pattern: "*.{fasta,fa}" + - - sort_bam: + type: boolean + description: use samtools sort (true) or samtools view (false) + pattern: "true or false" output: - bam: - type: file - description: Output BAM file containing read alignments - pattern: "*.{bam}" + - meta: + type: file + description: Output BAM file containing read alignments + pattern: "*.{bam}" + - "*.bam": + type: file + description: Output BAM file containing read alignments + pattern: "*.{bam}" - cram: - type: file - description: Output CRAM file containing read alignments - pattern: "*.{cram}" + - meta: + type: file + description: Output CRAM file containing read alignments + pattern: "*.{cram}" + - "*.cram": + type: file + description: Output CRAM file containing read alignments + pattern: "*.{cram}" - csi: - type: file - description: Optional index file for BAM file - pattern: "*.{csi}" + - meta: + type: file + description: Optional index file for BAM file + pattern: "*.{csi}" + - "*.csi": + type: file + description: Optional index file for BAM file + pattern: "*.{csi}" - crai: - type: file - description: Optional index file for CRAM file - pattern: "*.{crai}" + - meta: + type: file + description: Optional index file for CRAM file + pattern: "*.{crai}" + - "*.crai": + type: file + description: Optional index file for CRAM file + pattern: "*.{crai}" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@drpatelh" - "@jeremy1805" diff --git a/modules/nf-core/bwa/mem/tests/main.nf.test b/modules/nf-core/bwa/mem/tests/main.nf.test index 1fa9b56..5de2c2f 100644 --- a/modules/nf-core/bwa/mem/tests/main.nf.test +++ b/modules/nf-core/bwa/mem/tests/main.nf.test @@ -9,21 +9,21 @@ nextflow_process { script "../main.nf" process "BWA_MEM" - test("Single-End") { - - setup { - run("BWA_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = [ - [id: 'test'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ] - """ - } + setup { + run("BWA_INDEX") { + script "../../index/main.nf" + process { + """ + input[0] = [ + [id: 'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ } } + } + + test("Single-End") { when { process { @@ -44,7 +44,15 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out.cram, + process.out.csi, + process.out.crai, + process.out.versions, + bam(process.out.bam[0][1]).getHeaderMD5(), + bam(process.out.bam[0][1]).getReadsMD5() + ).match() + } ) } @@ -52,20 +60,6 @@ nextflow_process { test("Single-End Sort") { - setup { - run("BWA_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = [ - [id: 'test'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ] - """ - } - } - } - when { process { """ @@ -85,7 +79,15 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out.cram, + process.out.csi, + process.out.crai, + process.out.versions, + bam(process.out.bam[0][1]).getHeaderMD5(), + bam(process.out.bam[0][1]).getReadsMD5() + ).match() + } ) } @@ -93,20 +95,6 @@ nextflow_process { test("Paired-End") { - setup { - run("BWA_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = [ - [id: 'test'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ] - """ - } - } - } - when { process { """ @@ -127,7 +115,15 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out.cram, + process.out.csi, + process.out.crai, + process.out.versions, + bam(process.out.bam[0][1]).getHeaderMD5(), + bam(process.out.bam[0][1]).getReadsMD5() + ).match() + } ) } @@ -135,20 +131,6 @@ nextflow_process { test("Paired-End Sort") { - setup { - run("BWA_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = [ - [id: 'test'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ] - """ - } - } - } - when { process { """ @@ -169,7 +151,15 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out.cram, + process.out.csi, + process.out.crai, + process.out.versions, + bam(process.out.bam[0][1]).getHeaderMD5(), + bam(process.out.bam[0][1]).getReadsMD5() + ).match() + } ) } @@ -177,20 +167,6 @@ nextflow_process { test("Paired-End - no fasta") { - setup { - run("BWA_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = [ - [id: 'test'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ] - """ - } - } - } - when { process { """ @@ -211,27 +187,24 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out.cram, + process.out.csi, + process.out.crai, + process.out.versions, + bam(process.out.bam[0][1]).getHeaderMD5(), + bam(process.out.bam[0][1]).getReadsMD5() + ).match() + } ) } } test("Single-end - stub") { + options "-stub" - setup { - run("BWA_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = [ - [id: 'test'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ] - """ - } - } - } + when { process { """ @@ -251,30 +224,15 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.bam[0][1]).name, - file(process.out.csi[0][1]).name, - process.out.versions - ).match() } + { assert snapshot(process.out).match() } ) } } test("Paired-end - stub") { + options "-stub" - setup { - run("BWA_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = [ - [id: 'test'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ] - """ - } - } - } + when { process { """ @@ -295,11 +253,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( - file(process.out.bam[0][1]).name, - file(process.out.csi[0][1]).name, - process.out.versions - ).match() } + { assert snapshot(process.out).match() } ) } } diff --git a/modules/nf-core/bwa/mem/tests/main.nf.test.snap b/modules/nf-core/bwa/mem/tests/main.nf.test.snap index 0d1bdb4..2079ea2 100644 --- a/modules/nf-core/bwa/mem/tests/main.nf.test.snap +++ b/modules/nf-core/bwa/mem/tests/main.nf.test.snap @@ -1,5 +1,97 @@ { "Single-End": { + "content": [ + [ + + ], + [ + + ], + [ + + ], + [ + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" + ], + "b6d9cb250261a4c125413c5d867d87a7", + "798439cbd7fd81cbcc5078022dc5479d" + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-08-02T12:22:28.051598" + }, + "Single-End Sort": { + "content": [ + [ + + ], + [ + + ], + [ + + ], + [ + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" + ], + "848434ae4b79cfdcb2281c60b33663ce", + "94fcf617f5b994584c4e8d4044e16b4f" + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-08-02T12:22:39.671154" + }, + "Paired-End": { + "content": [ + [ + + ], + [ + + ], + [ + + ], + [ + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" + ], + "5b34d31be84478761f789e3e2e805e31", + "57aeef88ed701a8ebc8e2f0a381b2a6" + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-08-02T12:22:51.919479" + }, + "Paired-End Sort": { + "content": [ + [ + + ], + [ + + ], + [ + + ], + [ + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" + ], + "69003376d9a8952622d8587b39c3eaae", + "af8628d9df18b2d3d4f6fd47ef2bb872" + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-08-02T12:23:00.833562" + }, + "Single-end - stub": { "content": [ { "0": [ @@ -8,73 +100,32 @@ "id": "test", "single_end": true }, - "test.bam:md5,a74710a0345b4717bb4431bf9c257120" + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "1": [ ], "2": [ - - ], - "3": [ - - ], - "4": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" - ], - "bam": [ [ { "id": "test", "single_end": true }, - "test.bam:md5,a74710a0345b4717bb4431bf9c257120" + "test.csi:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "crai": [ - - ], - "cram": [ - - ], - "csi": [ - - ], - "versions": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" - ] - } - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-03-14T14:14:56.820798254" - }, - "Single-End Sort": { - "content": [ - { - "0": [ + "3": [ [ { "id": "test", "single_end": true }, - "test.bam:md5,cb1e038bc4d990683fa485d632550b54" + "test.crai:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "1": [ - - ], - "2": [ - - ], - "3": [ - ], "4": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" ], "bam": [ [ @@ -82,30 +133,65 @@ "id": "test", "single_end": true }, - "test.bam:md5,cb1e038bc4d990683fa485d632550b54" + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "crai": [ - + [ + { + "id": "test", + "single_end": true + }, + "test.crai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "cram": [ ], "csi": [ - + [ + { + "id": "test", + "single_end": true + }, + "test.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "versions": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-03-14T14:15:20.271428534" + "timestamp": "2024-08-02T12:31:29.46282" }, - "Paired-End": { + "Paired-End - no fasta": { + "content": [ + [ + + ], + [ + + ], + [ + + ], + [ + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" + ], + "5b34d31be84478761f789e3e2e805e31", + "57aeef88ed701a8ebc8e2f0a381b2a6" + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-08-02T12:23:09.942545" + }, + "Paired-end - stub": { "content": [ { "0": [ @@ -114,73 +200,32 @@ "id": "test", "single_end": false }, - "test.bam:md5,aea123a3828a99da1906126355f15a12" + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "1": [ ], "2": [ - - ], - "3": [ - - ], - "4": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" - ], - "bam": [ [ { "id": "test", "single_end": false }, - "test.bam:md5,aea123a3828a99da1906126355f15a12" + "test.csi:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "crai": [ - - ], - "cram": [ - - ], - "csi": [ - - ], - "versions": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" - ] - } - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-03-14T14:15:38.264256823" - }, - "Paired-End Sort": { - "content": [ - { - "0": [ + "3": [ [ { "id": "test", "single_end": false }, - "test.bam:md5,4682087bcdc3617384b375093fecd8dd" + "test.crai:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "1": [ - - ], - "2": [ - - ], - "3": [ - ], "4": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" ], "bam": [ [ @@ -188,108 +233,39 @@ "id": "test", "single_end": false }, - "test.bam:md5,4682087bcdc3617384b375093fecd8dd" + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "crai": [ - - ], - "cram": [ - - ], - "csi": [ - - ], - "versions": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" - ] - } - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-03-14T14:16:00.528642686" - }, - "Single-end - stub": { - "content": [ - "test.bam", - "test.csi", - [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-03-15T09:50:33.238543011" - }, - "Paired-End - no fasta": { - "content": [ - { - "0": [ [ { "id": "test", "single_end": false }, - "test.bam:md5,aea123a3828a99da1906126355f15a12" + "test.crai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "1": [ - - ], - "2": [ - - ], - "3": [ + "cram": [ ], - "4": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" - ], - "bam": [ + "csi": [ [ { "id": "test", "single_end": false }, - "test.bam:md5,aea123a3828a99da1906126355f15a12" + "test.csi:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "crai": [ - - ], - "cram": [ - - ], - "csi": [ - ], "versions": [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" + "versions.yml:md5,478b816fbd37871f5e8c617833d51d80" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-03-14T14:40:16.345342005" - }, - "Paired-end - stub": { - "content": [ - "test.bam", - "test.csi", - [ - "versions.yml:md5,c32f719a68bb2966c8511d808154d42d" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-03-15T09:50:42.268673302" + "timestamp": "2024-08-02T12:31:37.757037" } } \ No newline at end of file diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml index ca39fb6..6f5b867 100644 --- a/modules/nf-core/multiqc/environment.yml +++ b/modules/nf-core/multiqc/environment.yml @@ -1,7 +1,5 @@ -name: multiqc channels: - conda-forge - bioconda - - defaults dependencies: - - bioconda::multiqc=1.21 + - bioconda::multiqc=1.25.1 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 782b730..9acdcf2 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -4,14 +4,16 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.21--pyhdfd78af_0' : - 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.25.1--pyhdfd78af_0' : + 'biocontainers/multiqc:1.25.1--pyhdfd78af_0' }" input: tuple val(meta), path(multiqc_files) path(multiqc_config) path(extra_multiqc_config) path(multiqc_logo) + path(replace_names) + path(sample_names) output: path "*multiqc_report.html", emit: report @@ -24,16 +26,22 @@ process MULTIQC { script: def args = task.ext.args ?: '' + def prefix = task.ext.prefix ? "--filename ${task.ext.prefix}.html" : '' def config = multiqc_config ? "--config $multiqc_config" : '' def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' - def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' + def logo = multiqc_logo ? "--cl-config 'custom_logo: \"${multiqc_logo}\"'" : '' + def replace = replace_names ? "--replace-names ${replace_names}" : '' + def samples = sample_names ? "--sample-names ${sample_names}" : '' """ multiqc \\ --force \\ $args \\ $config \\ + $prefix \\ $extra_config \\ $logo \\ + $replace \\ + $samples \\ . cat <<-END_VERSIONS > versions.yml @@ -45,7 +53,7 @@ process MULTIQC { stub: """ mkdir multiqc_data - touch multiqc_plots + mkdir multiqc_plots touch multiqc_report.html cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index 45a9bc3..b16c187 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -1,5 +1,6 @@ name: multiqc -description: Aggregate results from bioinformatics analyses across many samples into a single report +description: Aggregate results from bioinformatics analyses across many samples into + a single report keywords: - QC - bioinformatics tools @@ -12,40 +13,59 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] + identifier: biotools:multiqc input: - - multiqc_files: - type: file - description: | - List of reports / files recognised by MultiQC, for example the html and zip output of FastQC - - multiqc_config: - type: file - description: Optional config yml for MultiQC - pattern: "*.{yml,yaml}" - - extra_multiqc_config: - type: file - description: Second optional config yml for MultiQC. Will override common sections in multiqc_config. - pattern: "*.{yml,yaml}" - - multiqc_logo: - type: file - description: Optional logo file for MultiQC - pattern: "*.{png}" + - - multiqc_files: + type: file + description: | + List of reports / files recognised by MultiQC, for example the html and zip output of FastQC + - - multiqc_config: + type: file + description: Optional config yml for MultiQC + pattern: "*.{yml,yaml}" + - - extra_multiqc_config: + type: file + description: Second optional config yml for MultiQC. Will override common sections + in multiqc_config. + pattern: "*.{yml,yaml}" + - - multiqc_logo: + type: file + description: Optional logo file for MultiQC + pattern: "*.{png}" + - - replace_names: + type: file + description: | + Optional two-column sample renaming file. First column a set of + patterns, second column a set of corresponding replacements. Passed via + MultiQC's `--replace-names` option. + pattern: "*.{tsv}" + - - sample_names: + type: file + description: | + Optional TSV file with headers, passed to the MultiQC --sample_names + argument. + pattern: "*.{tsv}" output: - report: - type: file - description: MultiQC report file - pattern: "multiqc_report.html" + - "*multiqc_report.html": + type: file + description: MultiQC report file + pattern: "multiqc_report.html" - data: - type: directory - description: MultiQC data dir - pattern: "multiqc_data" + - "*_data": + type: directory + description: MultiQC data dir + pattern: "multiqc_data" - plots: - type: file - description: Plots created by MultiQC - pattern: "*_data" + - "*_plots": + type: file + description: Plots created by MultiQC + pattern: "*_data" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@abhi18av" - "@bunop" diff --git a/modules/nf-core/multiqc/multiqc.diff b/modules/nf-core/multiqc/multiqc.diff index a5b10ef..eb5ca5c 100644 --- a/modules/nf-core/multiqc/multiqc.diff +++ b/modules/nf-core/multiqc/multiqc.diff @@ -1,4 +1,5 @@ Changes in module 'nf-core/multiqc' +Changes in 'multiqc/main.nf': --- modules/nf-core/multiqc/main.nf +++ modules/nf-core/multiqc/main.nf @@ -1,4 +1,5 @@ @@ -8,7 +9,7 @@ Changes in module 'nf-core/multiqc' conda "${moduleDir}/environment.yml" @@ -7,7 +8,7 @@ - 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" + 'biocontainers/multiqc:1.25.1--pyhdfd78af_0' }" input: - path multiqc_files, stageAs: "?/*" @@ -17,4 +18,10 @@ Changes in module 'nf-core/multiqc' path(extra_multiqc_config) path(multiqc_logo) +'modules/nf-core/multiqc/environment.yml' is unchanged +'modules/nf-core/multiqc/meta.yml' is unchanged +'modules/nf-core/multiqc/tests/tags.yml' is unchanged +'modules/nf-core/multiqc/tests/main.nf.test' is unchanged +'modules/nf-core/multiqc/tests/main.nf.test.snap' is unchanged +'modules/nf-core/multiqc/tests/nextflow.config' is unchanged ************************************************************ diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test index f1c4242..33316a7 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -8,6 +8,8 @@ nextflow_process { tag "modules_nfcore" tag "multiqc" + config "./nextflow.config" + test("sarscov2 single-end [fastqc]") { when { @@ -17,6 +19,8 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } @@ -41,6 +45,8 @@ nextflow_process { input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } @@ -66,6 +72,8 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap index bfebd80..2fcbb5f 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-02-29T08:48:55.657331" + "timestamp": "2024-10-02T17:51:46.317523" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-02-29T08:49:49.071937" + "timestamp": "2024-10-02T17:52:20.680978" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-02-29T08:49:25.457567" + "timestamp": "2024-10-02T17:52:09.185842" } } \ No newline at end of file diff --git a/modules/nf-core/multiqc/tests/nextflow.config b/modules/nf-core/multiqc/tests/nextflow.config new file mode 100644 index 0000000..c537a6a --- /dev/null +++ b/modules/nf-core/multiqc/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: 'MULTIQC' { + ext.prefix = null + } +} diff --git a/nextflow.config b/nextflow.config index 7bac49d..e0f19bc 100644 --- a/nextflow.config +++ b/nextflow.config @@ -51,10 +51,6 @@ params { max_time = '240.h' // Schema validation default options - validationFailUnrecognisedParams = false - validationLenientMode = false - validationSchemaIgnoreParams = '' - validationShowHiddenParams = false validate_params = true } @@ -62,13 +58,6 @@ params { // Load base.config by default for all pipelines includeConfig 'conf/base.config' -// Load nf-core custom profiles from different Institutions -try { - includeConfig "${params.custom_config_base}/nfcore_custom.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") -} - // Load nf-cmgg/sampletracking custom profiles from different institutions. // Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs! // try { @@ -202,22 +191,21 @@ process.shell = ['/bin/bash', '-euo', 'pipefail'] // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false -def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true - file = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html" + file = "${params.outdir}/${params.unique_out}/execution_timeline_${new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')}.html" } report { enabled = true - file = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html" + file = timeline.file.replace("timeline", "report") } trace { enabled = true - file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" + file = timeline.file.replace("timeline", "trace") } dag { enabled = true - file = "${params.outdir}/pipeline_info/pipeline_dag_${trace_timestamp}.html" + file = timeline.file.replace("execution_timeline", "pipeline_dag") } manifest { @@ -231,38 +219,29 @@ manifest { doi = '' } -// Load modules.config for DSL2 module specific options -includeConfig 'conf/modules.config' - -// Function to ensure that resource requirements don't go beyond -// a maximum limit -def check_max(obj, type) { - if (type == 'memory') { - try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit - else - return obj - } catch (all) { - println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'time') { - try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration - else - return obj - } catch (all) { - println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'cpus') { - try { - return Math.min( obj, params.max_cpus as int ) - } catch (all) { - println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" - return obj - } +validation { + defaultIgnoreParams = ["genomes", "test_data", "igenomes_base"] + help { + enabled = true + command = "nextflow run $manifest.name -profile --input samplesheet.csv --outdir " + fullParameter = "help_full" + showHiddenParameter = "show_hidden" + beforeText = """ +-\033[2m----------------------------------------------------\033[0m- + \033[0;34m ///\033[0;32m/// \033[0m +\033[0;34m ___ __ _ _ __ __ \033[0;34m ///\033[0;32m///// \033[0m +\033[0;34m |\\ | |__ __ / ` | \\/ | / _` / _` \033[0;34m////\033[0;32m////// \033[0m +\033[0;34m | \\| | \\__, | | \\__| \\__| \033[0;34m///\033[0;32m///// \033[0m + \033[0;34m///\033[0;32m/// \033[0m +\033[0;35m ${manifest.name} ${manifest.version}\033[0m +-\033[2m----------------------------------------------------\033[0m- +""" + } + summary { + beforeText = validation.help.beforeText + hideParams = ["genomes"] } } + +// Load modules.config for DSL2 module specific options +includeConfig 'conf/modules.config' From a1d75d0304a143a4f9da91ada934fb4be6bbb100 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 11:16:08 +0100 Subject: [PATCH 03/14] fix language server warnings/errors --- assets/schema_input.json | 6 +-- conf/base.config | 46 +++++++----------- conf/test.config | 10 ++++ main.nf | 47 ++++--------------- nextflow.config | 8 ++-- nextflow_schema.json | 14 +++--- .../main.nf | 2 +- workflows/sampletracking.nf | 37 +++++++++------ 8 files changed, 75 insertions(+), 95 deletions(-) diff --git a/assets/schema_input.json b/assets/schema_input.json index 1968ade..6119d78 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-cmgg/sampletracking/master/assets/schema_input.json", "title": "nf-cmgg/sampletracking pipeline - params.input schema", "description": "Schema for the file provided with params.input", @@ -11,8 +11,7 @@ "type": "string", "pattern": "^\\S+$", "errorMessage": "Sample name must be provided and cannot contain spaces", - "meta": ["id", "samplename"], - "unique": true + "meta": ["id", "samplename"] }, "pool": { "type": "string", @@ -60,6 +59,7 @@ "errorMessage": "snp BAM/CRAM index file must be provided, cannot contain spaces and must have extension '.bai' or '.csi'" } }, + "uniqueEntries": ["sample"], "anyOf": [ { "required": ["sample", "sample_bam", "snp_fastq_1", "snp_fastq_2"] diff --git a/conf/base.config b/conf/base.config index 690e4de..6ed8a91 100644 --- a/conf/base.config +++ b/conf/base.config @@ -1,6 +1,6 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - nf-cmgg/sampletracking Nextflow base config file + nf-cmgg/germline Nextflow base config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A 'blank slate' config file, appropriate for general use on most high performance compute environments. Assumes that all software is installed and available on @@ -10,47 +10,40 @@ process { - // TODO nf-core: Check the defaults for all processes - cpus = { check_max( 1 * task.attempt, 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 * task.attempt } + memory = { 8.GB * task.attempt } + time = { 4.h * task.attempt } errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' // Process-specific resource requirements - // NOTE - Please try and re-use the labels below as much as possible. - // These labels are used and recognised by default in DSL2 files hosted on nf-core/modules. - // If possible, it would be nice to keep the same label naming convention when - // adding in your local modules too. - // TODO nf-core: Customise requirements for specific processes. - // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors withLabel:process_single { - cpus = { check_max( 1 , 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 } + memory = { 8.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 2 * task.attempt } + memory = { 16.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_medium { - cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 36.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + cpus = { 4 * task.attempt } + memory = { 32.GB * task.attempt } + time = { 8.h * task.attempt } } withLabel:process_high { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 72.GB * task.attempt, 'memory' ) } - time = { check_max( 16.h * task.attempt, 'time' ) } + cpus = { 8 * task.attempt } + memory = { 64.GB * task.attempt } + time = { 16.h * task.attempt } } withLabel:process_long { - time = { check_max( 20.h * task.attempt, 'time' ) } + time = { 20.h * task.attempt } } withLabel:process_high_memory { - memory = { check_max( 200.GB * task.attempt, 'memory' ) } + memory = { 200.GB * task.attempt } } withLabel:error_ignore { errorStrategy = 'ignore' @@ -59,7 +52,4 @@ process { errorStrategy = 'retry' maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } } diff --git a/conf/test.config b/conf/test.config index 500bdb3..2483935 100644 --- a/conf/test.config +++ b/conf/test.config @@ -25,6 +25,7 @@ params { fai = "https://github.com/nf-cmgg/test-datasets/raw/sampletracking/data/genomics/homo_sapiens/genome/seq/GCA_000001405.15_GRCh38_full_plus_hs38d1_analysis_set_chr21.fna.fai" haplotype_map = "https://github.com/nf-cmgg/test-datasets/raw/sampletracking/data/genomics/homo_sapiens/genome/picard/haplotype_map.txt" } + aws { client { endpoint = "https://s3.ugent.be" @@ -34,3 +35,12 @@ aws { } } +process { + // Limit resources so that this can run on GitHub Actions + resourceLimits = [ + cpus : 2, + memory: 6.GB, + time : 6.h + ] +} + diff --git a/main.nf b/main.nf index 8393345..8dfa2ac 100644 --- a/main.nf +++ b/main.nf @@ -7,17 +7,15 @@ ---------------------------------------------------------------------------------------- */ -nextflow.enable.dsl = 2 - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { SAMPLETRACKING } from './workflows/sampletracking' -include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_sampletracking_pipeline' -include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_sampletracking_pipeline' +include { SAMPLETRACKING } from './workflows/sampletracking' +include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_sampletracking_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_sampletracking_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -25,33 +23,6 @@ include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_samp ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// -// WORKFLOW: Run main analysis pipeline depending on type of input -// -workflow NFCMGG_SAMPLETRACKING { - - take: - samplesheet // channel: samplesheet read in from --input - bwa_index - fasta_fai - haplotype_map - - main: - - // - // WORKFLOW: Run pipeline - // - SAMPLETRACKING ( - samplesheet, - bwa_index, - fasta_fai, - haplotype_map - ) - - emit: - multiqc_report = SAMPLETRACKING.out.multiqc_report // channel: /path/to/multiqc_report.html - -} /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RUN MAIN WORKFLOW @@ -67,9 +38,7 @@ workflow { // PIPELINE_INITIALISATION ( params.version, - params.help, params.validate_params, - params.monochrome_logs, args, params.outdir, params.input @@ -78,7 +47,7 @@ workflow { // // WORKFLOW: Run main workflow // - NFCMGG_SAMPLETRACKING ( + SAMPLETRACKING ( PIPELINE_INITIALISATION.out.samplesheet, Channel.value([ [id: "bwa"], @@ -92,7 +61,11 @@ workflow { Channel.value( [[id:"haplotype_map"], file(params.haplotype_map, checkIfExists: true) - ]) + ]), + params.outdir, + params.multiqc_config, + params.multiqc_logo, + params.multiqc_methods_description ) // @@ -105,7 +78,7 @@ workflow { params.outdir, params.monochrome_logs, params.hook_url, - NFCMGG_SAMPLETRACKING.out.multiqc_report + SAMPLETRACKING.out.multiqc_report ) } diff --git a/nextflow.config b/nextflow.config index e0f19bc..7f099f1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -193,19 +193,19 @@ nextflow.enable.configProcessNamesValidation = false timeline { enabled = true - file = "${params.outdir}/${params.unique_out}/execution_timeline_${new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')}.html" + file = "${params.outdir}/pipeline_info/execution_timeline_${new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')}.html" } report { enabled = true - file = timeline.file.replace("timeline", "report") + file = "${params.outdir}/pipeline_info/execution_report_${new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')}.html" } trace { enabled = true - file = timeline.file.replace("timeline", "trace") + file = "${params.outdir}/pipeline_info/execution_trace_${new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')}.txt" } dag { enabled = true - file = timeline.file.replace("execution_timeline", "pipeline_dag") + file = "${params.outdir}/pipeline_info/pipeline_dag_${new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')}.html" } manifest { diff --git a/nextflow_schema.json b/nextflow_schema.json index 4ca141d..9664b54 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-cmgg/sampletracking/master/nextflow_schema.json", "title": "nf-cmgg/sampletracking pipeline parameters", "description": "CMGG Sampletracking workflow", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -285,19 +285,19 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/reference_genome_options" + "$ref": "#/$defs/reference_genome_options" }, { - "$ref": "#/definitions/institutional_config_options" + "$ref": "#/$defs/institutional_config_options" }, { - "$ref": "#/definitions/max_job_request_options" + "$ref": "#/$defs/max_job_request_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/subworkflows/local/utils_nfcore_sampletracking_pipeline/main.nf b/subworkflows/local/utils_nfcore_sampletracking_pipeline/main.nf index 9f5bb3a..7a18e03 100644 --- a/subworkflows/local/utils_nfcore_sampletracking_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_sampletracking_pipeline/main.nf @@ -137,7 +137,7 @@ def validateInputSamplesheet(input) { def (metas, fastqs) = input[1..2] // Check that multiple runs of the same sample are of the same datatype i.e. single-end / paired-end - def endedness_ok = metas.collect{ it.single_end }.unique().size == 1 + def endedness_ok = metas.collect{ it -> it.single_end }.unique().size == 1 if (!endedness_ok) { error("Please check input samplesheet -> Multiple runs of a sample must be of the same datatype i.e. single-end or paired-end: ${metas[0].id}") } diff --git a/workflows/sampletracking.nf b/workflows/sampletracking.nf index b1e6a23..da052f6 100644 --- a/workflows/sampletracking.nf +++ b/workflows/sampletracking.nf @@ -21,10 +21,15 @@ include { methodsDescriptionText } from '../subworkflows/local/utils_nfco workflow SAMPLETRACKING { take: - ch_samplesheet // channel: samplesheet read in from --input - ch_bwa_index // channel: [meta, /path/to/bwa_index] - ch_fasta_fai // channel: [meta,/path/to/fasta, /path/to/fasta.fai] - ch_haplotype_map // channel: [meta, /path/to/haplotype_map] + ch_samplesheet // channel: samplesheet read in from --input + ch_bwa_index // channel: [meta, /path/to/bwa_index] + ch_fasta_fai // channel: [meta,/path/to/fasta, /path/to/fasta.fai] + ch_haplotype_map // channel: [meta, /path/to/haplotype_map] + + outdir // string: path/to/outdir + multiqc_config // string: path/to/multiqc_config + multiqc_logo // string: path/to/multiqc_logo + multiqc_methods_description // string: path/to/multiqc_methods_description main: @@ -49,7 +54,7 @@ workflow SAMPLETRACKING { BWA_MEM( ch_to_align.fastq, ch_bwa_index, - ch_fasta_fai.map{meta, fasta, fai -> [meta, fasta]}, + ch_fasta_fai.map{meta, fasta, _fai -> [meta, fasta]}, true ) ch_versions = ch_versions.mix(BWA_MEM.out.versions) @@ -62,7 +67,7 @@ workflow SAMPLETRACKING { return [[id: meta.pool], sample_bam, sample_bam_index, snp_bam, snp_bam_index] } .groupTuple() - .merge(ch_haplotype_map.map{it[1]}) + .merge(ch_haplotype_map.map{ it -> it[1]}) .map{ meta, sample_bam, sample_bam_index, snp_bam, snp_bam_index, haplotype_map -> return [meta, sample_bam.flatten(), sample_bam_index.flatten(), snp_bam.flatten(), snp_bam_index.flatten(), haplotype_map] } @@ -74,24 +79,24 @@ workflow SAMPLETRACKING { ch_fasta_fai ) ch_versions = ch_versions.mix(PICARD_CROSSCHECKFINGERPRINTS.out.versions) - ch_multiqc_files = ch_multiqc_files.mix(PICARD_CROSSCHECKFINGERPRINTS.out.crosscheck_metrics.map{it[1]}) + ch_multiqc_files = ch_multiqc_files.mix(PICARD_CROSSCHECKFINGERPRINTS.out.crosscheck_metrics.map{it -> it[1]}) // // Collate and save software versions // softwareVersionsToYAML(ch_versions) - .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) + .collectFile(storeDir: "${outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) .set { ch_collated_versions } // // MODULE: MultiQC // ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) - ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() - ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + ch_multiqc_custom_config = multiqc_config ? Channel.fromPath(multiqc_config, checkIfExists: true) : Channel.empty() + ch_multiqc_logo = multiqc_logo ? Channel.fromPath(multiqc_logo, checkIfExists: true) : Channel.empty() summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) - ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_multiqc_custom_methods_description = multiqc_methods_description ? file(multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) @@ -101,13 +106,15 @@ workflow SAMPLETRACKING { PICARD_CROSSCHECKFINGERPRINTS.out.crosscheck_metrics, ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), - ch_multiqc_logo.toList() + ch_multiqc_logo.toList(), + [], + [] ) emit: - multiqc_report = MULTIQC.out.report.toList() // channel: /path/to/multiqc_report.html - crosscheck_metrics = PICARD_CROSSCHECKFINGERPRINTS.out.crosscheck_metrics.map{it[1]} // channel: [ path(crosscheck_metrics.txt) ] - versions = ch_versions // channel: [ path(versions.yml) ] + multiqc_report = MULTIQC.out.report.toList() // channel: /path/to/multiqc_report.html + crosscheck_metrics = PICARD_CROSSCHECKFINGERPRINTS.out.crosscheck_metrics.map{it->it[1]} // channel: [ path(crosscheck_metrics.txt) ] + versions = ch_versions // channel: [ path(versions.yml) ] } /* From 6927e509103901924cfa7bc453b47c5f1b28c4e1 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 11:23:48 +0100 Subject: [PATCH 04/14] fix some linting issues --- nextflow.config | 9 +++------ nextflow_schema.json | 21 --------------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/nextflow.config b/nextflow.config index 7f099f1..bde11cd 100644 --- a/nextflow.config +++ b/nextflow.config @@ -44,12 +44,6 @@ params { config_profile_contact = null config_profile_url = null - // Max resource options - // Defaults only, expecting to be overwritten - max_memory = '128.GB' - max_cpus = 16 - max_time = '240.h' - // Schema validation default options validate_params = true @@ -58,6 +52,9 @@ params { // Load base.config by default for all pipelines includeConfig 'conf/base.config' +// Load nf-core custom profiles from different Institutions", +includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" + // Load nf-cmgg/sampletracking custom profiles from different institutions. // Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs! // try { diff --git a/nextflow_schema.json b/nextflow_schema.json index 9664b54..5e8bb38 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -258,27 +258,6 @@ "default": true, "fa_icon": "fas fa-check-square", "hidden": true - }, - "validationShowHiddenParams": { - "type": "boolean", - "fa_icon": "far fa-eye-slash", - "description": "Show all params when using `--help`", - "hidden": true, - "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "validationFailUnrecognisedParams": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters fails when an unrecognised parameter is found.", - "hidden": true, - "help_text": "By default, when an unrecognised parameter is found, it returns a warinig." - }, - "validationLenientMode": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters in lenient more.", - "hidden": true, - "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." } } } From 08d2161b23c9a9a6478020b0a3b69a5d07f52882 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 11:26:16 +0100 Subject: [PATCH 05/14] fix linting --- .github/CONTRIBUTING.md | 12 ++++----- .github/ISSUE_TEMPLATE/bug_report.yml | 32 +++++++--------------- .github/PULL_REQUEST_TEMPLATE.md | 4 +-- .github/workflows/linting_comment.yml | 2 +- .gitignore | 1 + .nf-core.yml | 1 + nextflow_schema.json | 38 --------------------------- 7 files changed, 21 insertions(+), 69 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index e8e86c6..ce9bd79 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -16,7 +16,7 @@ If you'd like to write some code for nf-cmgg/sampletracking, the standard workfl 1. Check that there isn't already an issue about your idea in the [nf-cmgg/sampletracking issues](https://github.com/nf-cmgg/sampletracking/issues) to avoid duplicating work. If there isn't one already, please create one so that others know you're working on this 2. [Fork](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) the [nf-cmgg/sampletracking repository](https://github.com/nf-cmgg/sampletracking) to your GitHub account 3. Make the necessary changes / additions within your forked repository following [Pipeline conventions](#pipeline-contribution-conventions) -4. Use `nf-core schema build` and add any new parameters to the pipeline JSON schema (requires [nf-core tools](https://github.com/nf-core/tools) >= 1.10). +4. Use `nf-core pipelines schema build` and add any new parameters to the pipeline JSON schema (requires [nf-core tools](https://github.com/nf-core/tools) >= 1.10). 5. Submit a Pull Request against the `dev` branch and wait for the code to be reviewed and merged If you're not used to this workflow with git, you can start with some [docs from GitHub](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests) or even their [excellent `git` resources](https://try.github.io/). @@ -37,7 +37,7 @@ There are typically two types of tests that run: ### Lint tests `nf-core` has a [set of guidelines](https://nf-co.re/developers/guidelines) which all pipelines must adhere to. -To enforce these and ensure that all pipelines stay in sync, we have developed a helper tool which runs checks on the pipeline code. This is in the [nf-core/tools repository](https://github.com/nf-core/tools) and once installed can be run locally with the `nf-core lint ` command. +To enforce these and ensure that all pipelines stay in sync, we have developed a helper tool which runs checks on the pipeline code. This is in the [nf-core/tools repository](https://github.com/nf-core/tools) and once installed can be run locally with the `nf-core pipelines lint ` command. If any failures or warnings are encountered, please follow the listed URL for more documentation. @@ -68,7 +68,7 @@ If you wish to contribute a new step, please use the following coding standards: 2. Write the process block (see below). 3. Define the output channel if needed (see below). 4. Add any new parameters to `nextflow.config` with a default (see below). -5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core schema build` tool). +5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core pipelines schema build` tool). 6. Add sanity checks and validation for all relevant parameters. 7. Perform local tests to validate that the new code works as expected. 8. If applicable, add a new test command in `.github/workflow/ci.yml`. @@ -79,11 +79,11 @@ If you wish to contribute a new step, please use the following coding standards: Parameters should be initialised / defined with default values in `nextflow.config` under the `params` scope. -Once there, use `nf-core schema build` to add to `nextflow_schema.json`. +Once there, use `nf-core pipelines schema build` to add to `nextflow_schema.json`. ### Default processes resource requirements -Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. +Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/main/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. The process resources can be passed on to the tool dynamically within the process with the `${task.cpus}` and `${task.memory}` variables in the `script:` block. @@ -96,7 +96,7 @@ Please use the following naming schemes, to make it easy to understand what is g ### Nextflow version bumping -If you are using a new feature from core Nextflow, you may bump the minimum required version of nextflow in the pipeline with: `nf-core bump-version --nextflow . [min-nf-version]` +If you are using a new feature from core Nextflow, you may bump the minimum required version of nextflow in the pipeline with: `nf-core pipelines bump-version --nextflow . [min-nf-version]` ### Images and figures diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 59f5a9e..d9a6a7b 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -9,46 +9,34 @@ body: description: A clear and concise description of what the bug is. validations: required: true + - type: textarea id: command_used attributes: label: Command used and terminal output - description: Steps to reproduce the behaviour. Please paste the command you used - to launch the pipeline and the output from your terminal. + description: Steps to reproduce the behaviour. Please paste the command you used to launch the pipeline and the output from your terminal. render: console - placeholder: "$ nextflow run ... - + placeholder: | + $ nextflow run ... Some output where something broke - " - type: textarea id: files attributes: label: Relevant files - description: "Please drag and drop the relevant files here. Create a `.zip` archive - if the extension is not allowed. - - Your verbose log file `.nextflow.log` is often useful _(this is a hidden file - in the directory where you launched the pipeline)_ as well as custom Nextflow - configuration files. + description: | + Please drag and drop the relevant files here. Create a `.zip` archive if the extension is not allowed. + Your verbose log file `.nextflow.log` is often useful _(this is a hidden file in the directory where you launched the pipeline)_ as well as custom Nextflow configuration files. - " - type: textarea id: system attributes: label: System information - description: "* Nextflow version _(eg. 23.04.0)_ - + description: | + * Nextflow version _(eg. 23.04.0)_ * Hardware _(eg. HPC, Desktop, Cloud)_ - * Executor _(eg. slurm, local, awsbatch)_ - - * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, - or Apptainer)_ - + * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ - * Version of nf-cmgg/sampletracking _(eg. 1.1, 1.5, 1.8.2)_ - - " diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 7bf1b5d..d9a041a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,8 +16,8 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-cmgg/samp - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-cmgg/sampletracking/tree/master/.github/CONTRIBUTING.md) -- [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). +- [ ] Make sure your code lints (`nf-core pipelines lint`). +- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index b706875..42e519b 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3 + uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6 with: workflow: linting.yml workflow_conclusion: completed diff --git a/.gitignore b/.gitignore index 92d1d16..e492601 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ testing* *.pyc .env .nf-test* +null diff --git a/.nf-core.yml b/.nf-core.yml index ad319de..1ffd9c7 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -13,6 +13,7 @@ lint: - docs/images/nf-core-sampletracking_logo_light.png - docs/images/nf-core-sampletracking_logo_dark.png - .github/workflows/linting.yml + - .gitignore multiqc_config: - report_comment nextflow_config: diff --git a/nextflow_schema.json b/nextflow_schema.json index 5e8bb38..52e2dc2 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -136,41 +136,6 @@ } } }, - "max_job_request_options": { - "title": "Max job request options", - "type": "object", - "fa_icon": "fab fa-acquisitions-incorporated", - "description": "Set the top limit for requested resources for any single job.", - "help_text": "If you are running on a smaller system, a pipeline step requesting more resources than are available may cause the Nextflow to stop the run with an error. These options allow you to cap the maximum resources requested by any single job so that the pipeline will run on your system.\n\nNote that you can not _increase_ the resources requested by any job using these options. For that you will need your own configuration file. See [the nf-core website](https://nf-co.re/usage/configuration) for details.", - "properties": { - "max_cpus": { - "type": "integer", - "description": "Maximum number of CPUs that can be requested for any single job.", - "default": 16, - "fa_icon": "fas fa-microchip", - "hidden": true, - "help_text": "Use to set an upper-limit for the CPU requirement for each process. Should be an integer e.g. `--max_cpus 1`" - }, - "max_memory": { - "type": "string", - "description": "Maximum amount of memory that can be requested for any single job.", - "default": "128.GB", - "fa_icon": "fas fa-memory", - "pattern": "^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$", - "hidden": true, - "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory '8.GB'`" - }, - "max_time": { - "type": "string", - "description": "Maximum amount of time that can be requested for any single job.", - "default": "240.h", - "fa_icon": "far fa-clock", - "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$", - "hidden": true, - "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" - } - } - }, "generic_options": { "title": "Generic options", "type": "object", @@ -272,9 +237,6 @@ { "$ref": "#/$defs/institutional_config_options" }, - { - "$ref": "#/$defs/max_job_request_options" - }, { "$ref": "#/$defs/generic_options" } From 6b659d60372723c781a83d8521afb2f204ff0812 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 11:28:44 +0100 Subject: [PATCH 06/14] fix wrong eclint ignore --- .editorconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index c615d7e..cbf9a83 100644 --- a/.editorconfig +++ b/.editorconfig @@ -28,7 +28,7 @@ indent_style = unset [/assets/email*] indent_size = unset -[/assets/haplotype maps/*] +[/assets/haplotype_maps/*] charset = unset end_of_line = unset insert_final_newline = unset From 2c3543e3b2ecd4ccb8d88d0600380856657cb2a4 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 11:31:43 +0100 Subject: [PATCH 07/14] use v2 of setup-nextflow --- .github/workflows/ci.yml | 2 +- .github/workflows/download_pipeline.yml | 2 +- .github/workflows/linting.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0a0e1f..e16b8e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "${{ matrix.NXF_VER }}" diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index 08622fd..f361db6 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 1445246..ecabc61 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: From a325b0b04d23e9c99badc1103f5c67c65a5ac2b1 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 11:37:22 +0100 Subject: [PATCH 08/14] bump minimal nextflow version to 23.10.0 --- .github/workflows/ci.yml | 2 +- README.md | 2 +- nextflow.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e16b8e9..bf7539f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.0" + - "23.10.0" - "latest-everything" tags: - "workflows" diff --git a/README.md b/README.md index 996b955..3924d96 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub Actions Linting Status](https://github.com/nf-cmgg/sampletracking/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-cmgg/sampletracking/actions/workflows/linting.yml)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/nextflow.config b/nextflow.config index bde11cd..5fb8933 100644 --- a/nextflow.config +++ b/nextflow.config @@ -211,7 +211,7 @@ manifest { homePage = 'https://github.com/nf-cmgg/sampletracking' description = """CMGG Sampletracking workflow""" mainScript = 'main.nf' - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.10.0' version = '1.0' doi = '' } From 1dba21f97c838b0c204ab471dd3a16f12633712d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 12:59:15 +0100 Subject: [PATCH 09/14] fix tests --- conf/base.config | 2 +- modules.json | 2 +- .../crosscheckfingerprints/environment.yml | 4 +- .../picard/crosscheckfingerprints/main.nf | 4 +- .../picard/crosscheckfingerprints/meta.yml | 102 +++++++++--------- .../picard-crosscheckfingerprints.diff | 7 ++ .../crosscheckfingerprints/tests/main.nf.test | 20 ++-- .../tests/main.nf.test.snap | 16 +-- tests/config/nf-test.config | 10 +- tests/workflows/sampletracking.nf.test | 4 + tests/workflows/sampletracking.nf.test.snap | 10 +- 11 files changed, 96 insertions(+), 85 deletions(-) diff --git a/conf/base.config b/conf/base.config index 6ed8a91..3ecf7a1 100644 --- a/conf/base.config +++ b/conf/base.config @@ -1,6 +1,6 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - nf-cmgg/germline Nextflow base config file + nf-cmgg/sampletracking Nextflow base config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A 'blank slate' config file, appropriate for general use on most high performance compute environments. Assumes that all software is installed and available on diff --git a/modules.json b/modules.json index 19ce99e..98f5732 100644 --- a/modules.json +++ b/modules.json @@ -23,7 +23,7 @@ }, "picard/crosscheckfingerprints": { "branch": "master", - "git_sha": "5154596388be562c96df3cb6bb08ff338e68c756", + "git_sha": "49f4e50534fe4b64101e62ea41d5dc43b1324358", "installed_by": ["modules"], "patch": "modules/nf-core/picard/crosscheckfingerprints/picard-crosscheckfingerprints.diff" } diff --git a/modules/nf-core/picard/crosscheckfingerprints/environment.yml b/modules/nf-core/picard/crosscheckfingerprints/environment.yml index ab81a3d..1d715d5 100644 --- a/modules/nf-core/picard/crosscheckfingerprints/environment.yml +++ b/modules/nf-core/picard/crosscheckfingerprints/environment.yml @@ -1,7 +1,5 @@ -name: picard_crosscheckfingerprints channels: - conda-forge - bioconda - - defaults dependencies: - - bioconda::picard=3.1.1 + - bioconda::picard=3.3.0 diff --git a/modules/nf-core/picard/crosscheckfingerprints/main.nf b/modules/nf-core/picard/crosscheckfingerprints/main.nf index d31ccfd..da69ad9 100644 --- a/modules/nf-core/picard/crosscheckfingerprints/main.nf +++ b/modules/nf-core/picard/crosscheckfingerprints/main.nf @@ -4,8 +4,8 @@ process PICARD_CROSSCHECKFINGERPRINTS { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/picard:3.1.1--hdfd78af_0' : - 'biocontainers/picard:3.1.1--hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/picard:3.3.0--hdfd78af_0' : + 'biocontainers/picard:3.3.0--hdfd78af_0' }" input: tuple val(meta), path(input1), path(input1_index), path(input2), path(input2_index), path(haplotype_map) diff --git a/modules/nf-core/picard/crosscheckfingerprints/meta.yml b/modules/nf-core/picard/crosscheckfingerprints/meta.yml index 1db3b86..898a11a 100644 --- a/modules/nf-core/picard/crosscheckfingerprints/meta.yml +++ b/modules/nf-core/picard/crosscheckfingerprints/meta.yml @@ -1,5 +1,6 @@ name: "picard_crosscheckfingerprints" -description: Checks that all data in the set of input files appear to come from the same individual +description: Checks that all data in the set of input files appear to come from the + same individual keywords: - alignment - metrics @@ -15,60 +16,59 @@ tools: documentation: https://broadinstitute.github.io/picard/ tool_dev_url: https://github.com/broadinstitute/picard/ licence: ["MIT"] + identifier: biotools:picard_tools input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - input1: - type: file - description: List containing 1 or more bam/vcf files or a file containing filepaths - pattern: "*.{bam,vcf,vcf.gz,txt,fofn}" - - input1_index: - type: file - description: List containing 1 or more bam/vcf files indexes - pattern: "*.{bai,csi,crai,tbi}" - - input2: - type: file - description: Optional list containing 1 or more bam/vcf files or a file containing filepaths - pattern: "*.{bam,vcf,vcf.gz,txt,fofn}" - - input2_index: - type: file - description: List containing 1 or more bam/vcf files indexes - pattern: "*.{bai,csi,crai,tbi}" - - haplotype_map: - type: file - description: Haplotype map file - pattern: "*.{txt,vcf,vcf.gz}" - - meta2: - type: map - description: | - Groovy Map containing reference information - e.g. [ id:'genome' ] - - fasta: - type: file - description: Reference genome file - pattern: "*.{fasta,fa,fasta.gz,fa.gz}" - - fasta_index: - type: file - description: Reference genome index file - pattern: "*.{fai,fasta.fai,fa.fai,fasta.gz.fai,fa.gz.fai}" + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - input1: + type: file + description: List containing 1 or more bam/vcf files or a file containing filepaths + pattern: "*.{bam,vcf,vcf.gz,txt,fofn}" + - input1_index: + type: file + description: List containing 1 or more bam/vcf files indexes + pattern: "*.{bai,csi,crai,tbi}" + - input2: + type: file + description: Optional list containing 1 or more bam/vcf files or a file containing + filepaths + pattern: "*.{bam,vcf,vcf.gz,txt,fofn}" + - input2_index: + type: file + description: List containing 1 or more bam/vcf files indexes + pattern: "*.{bai,csi,crai,tbi}" + - haplotype_map: + type: file + description: Haplotype map file + pattern: "*.{txt,vcf,vcf.gz}" + - - meta2: + type: map + description: | + Groovy Map containing reference information + e.g. [ id:'genome' ] + - fasta: + type: file + description: Reference genome file + pattern: "*.{fasta,fa,fasta.gz,fa.gz}" output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - crosscheck_metrics: - type: file - description: Metrics created by crosscheckfingerprints - pattern: "*.{crosscheck_metrics.txt}" + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.crosscheck_metrics.txt": + type: file + description: Metrics created by crosscheckfingerprints + pattern: "*.{crosscheck_metrics.txt}" - versions: - type: file - description: File containing software versions - pattern: "versions.yml" - + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@matthdsm" maintainers: diff --git a/modules/nf-core/picard/crosscheckfingerprints/picard-crosscheckfingerprints.diff b/modules/nf-core/picard/crosscheckfingerprints/picard-crosscheckfingerprints.diff index 7e94591..df8ace2 100644 --- a/modules/nf-core/picard/crosscheckfingerprints/picard-crosscheckfingerprints.diff +++ b/modules/nf-core/picard/crosscheckfingerprints/picard-crosscheckfingerprints.diff @@ -1,4 +1,5 @@ Changes in module 'nf-core/picard/crosscheckfingerprints' +Changes in 'picard/crosscheckfingerprints/main.nf': --- modules/nf-core/picard/crosscheckfingerprints/main.nf +++ modules/nf-core/picard/crosscheckfingerprints/main.nf @@ -9,7 +9,7 @@ @@ -11,4 +12,10 @@ Changes in module 'nf-core/picard/crosscheckfingerprints' output: tuple val(meta), path("*.crosscheck_metrics.txt"), emit: crosscheck_metrics +'modules/nf-core/picard/crosscheckfingerprints/environment.yml' is unchanged +'modules/nf-core/picard/crosscheckfingerprints/meta.yml' is unchanged +'modules/nf-core/picard/crosscheckfingerprints/tests/tags.yml' is unchanged +'modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test' is unchanged +'modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test.snap' is unchanged +'modules/nf-core/picard/crosscheckfingerprints/tests/nextflow.config' is unchanged ************************************************************ diff --git a/modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test b/modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test index 7a552e7..33eb40d 100644 --- a/modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test +++ b/modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test @@ -18,16 +18,16 @@ nextflow_process { input[0] = [ [ id:'test', single_end:false ], //meta [ - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_sorted_bam'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam', checkIfExists: true) ], // input1 [ - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_sorted_bam_bai'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam.bai', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam.bai', checkIfExists: true) ], // input1_index [], // input2 [], // input2_index - file(params.test_data['homo_sapiens']['genome']['haplotype_map'], checkIfExists: true) // haplotype_map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/germlineresources/haplotype_map.txt', checkIfExists: true) // haplotype_map ] input[1] = [[:],[]] """ @@ -53,16 +53,16 @@ nextflow_process { input[0] = [ [ id:'test', single_end:false ], //meta [ - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_sorted_bam'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam', checkIfExists: true) ], // input1 [ - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_sorted_bam_bai'], checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam.bai', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam.bai', checkIfExists: true) ], // input1_index [], // input2 [], // input2_index - file(params.test_data['homo_sapiens']['genome']['haplotype_map'], checkIfExists: true) // haplotype_map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/germlineresources/haplotype_map.txt', checkIfExists: true) // haplotype_map ] input[1] = [[:],[]] """ diff --git a/modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test.snap b/modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test.snap index 96fbdc8..f9e6963 100644 --- a/modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test.snap +++ b/modules/nf-core/picard/crosscheckfingerprints/tests/main.nf.test.snap @@ -22,25 +22,25 @@ "Crosscheck versions stub": { "content": [ [ - "versions.yml:md5,bd0a3212a131aee499cb7f954aeac63c" + "versions.yml:md5,baa5c1b1dc10b95cb42ea1b3d66a294f" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-03-08T14:36:24.83939" + "timestamp": "2024-10-18T11:02:39.2311109" }, "Crosscheck versions - bam": { "content": [ [ - "versions.yml:md5,bd0a3212a131aee499cb7f954aeac63c" + "versions.yml:md5,baa5c1b1dc10b95cb42ea1b3d66a294f" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "24.04.4" }, - "timestamp": "2024-03-08T14:30:50.311883" + "timestamp": "2024-10-18T11:02:13.953034883" } } \ No newline at end of file diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 2c1766d..ba50576 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -1,8 +1,10 @@ -params { +process { // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' + resourceLimits = [ + cpus : 2, + memory: 6.GB, + time : 6.h + ] } aws { diff --git a/tests/workflows/sampletracking.nf.test b/tests/workflows/sampletracking.nf.test index 9702ff2..29e97e1 100644 --- a/tests/workflows/sampletracking.nf.test +++ b/tests/workflows/sampletracking.nf.test @@ -63,6 +63,10 @@ nextflow_workflow { file("https://github.com/nf-cmgg/test-datasets/raw/sampletracking/data/genomics/homo_sapiens/genome/picard/haplotype_map.txt") ] ) + input[4] = params.outdir + input[5] = "" + input[6] = "" + input[7] = "" """ } } diff --git a/tests/workflows/sampletracking.nf.test.snap b/tests/workflows/sampletracking.nf.test.snap index 289fc8b..8f69b2e 100644 --- a/tests/workflows/sampletracking.nf.test.snap +++ b/tests/workflows/sampletracking.nf.test.snap @@ -4,14 +4,14 @@ "pool1_multiqc_report.html", "pool1.crosscheck_metrics.txt", [ - "versions.yml:md5,0e5cf92b1d741f4f366aaf67f2740640", - "versions.yml:md5,3138a3ee6f45d1d551dc6cecd3d4b0c5" + "versions.yml:md5,7442362a1e457dad7ce796c703bd6380", + "versions.yml:md5,ab16dda2f91f60705355cab06b0145bf" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.1", + "nextflow": "24.10.1" }, - "timestamp": "2024-03-25T08:37:51.702499" + "timestamp": "2024-12-10T12:55:31.080318371" } } \ No newline at end of file From 3acfb4c229b724fd9470badfd8d9e619205e0456 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 13:07:44 +0100 Subject: [PATCH 10/14] try to fix resource issues --- tests/config/nf-test.config | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index ba50576..3c03a9c 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -1,12 +1,3 @@ -process { - // Limit resources so that this can run on GitHub Actions - resourceLimits = [ - cpus : 2, - memory: 6.GB, - time : 6.h - ] -} - aws { client { endpoint = "https://s3.ugent.be" @@ -15,3 +6,12 @@ aws { connectionTimeout = 60000 } } + +process { + // Limit resources so that this can run on GitHub Actions + resourceLimits = [ + cpus : 2, + memory: 6.GB, + time : 6.h + ] +} \ No newline at end of file From be600b8a9555cdea4b0f6be8ff63fea3ffa94770 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 13:15:11 +0100 Subject: [PATCH 11/14] remove + from nf-test command --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf7539f..7442ab9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: From 4b14ef89b47c7ec7d93d4088781c20a3cf31ce19 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 13:20:57 +0100 Subject: [PATCH 12/14] remove profiles from ci --- .github/workflows/ci.yml | 4 +--- conf/test.config | 5 ----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7442ab9..c604f3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,8 +35,6 @@ jobs: tags: - "workflows" - "pipeline" - profile: - - "test,docker" steps: - name: Check out pipeline code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 @@ -76,7 +74,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose --tag ${{ matrix.tags }} --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/conf/test.config b/conf/test.config index 2483935..7471bcb 100644 --- a/conf/test.config +++ b/conf/test.config @@ -14,11 +14,6 @@ params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - input = "${projectDir}/assets/samplesheet.csv" bwa_index = "s3://test-data/genomics/homo_sapiens/genome/bwa/" fasta = "https://github.com/nf-cmgg/test-datasets/raw/sampletracking/data/genomics/homo_sapiens/genome/seq/GCA_000001405.15_GRCh38_full_plus_hs38d1_analysis_set_chr21.fna" From 981de377a91fa8178f5bcaa19b0c9deb57d642fd Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 13:28:05 +0100 Subject: [PATCH 13/14] bump min nextflow to 24.04.0 --- .github/workflows/ci.yml | 2 +- README.md | 2 +- nextflow.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c604f3f..cf08231 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.10.0" + - "24.04.0" - "latest-everything" tags: - "workflows" diff --git a/README.md b/README.md index 3924d96..58d9223 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub Actions Linting Status](https://github.com/nf-cmgg/sampletracking/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-cmgg/sampletracking/actions/workflows/linting.yml)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/nextflow.config b/nextflow.config index 5fb8933..73a85e1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -211,7 +211,7 @@ manifest { homePage = 'https://github.com/nf-cmgg/sampletracking' description = """CMGG Sampletracking workflow""" mainScript = 'main.nf' - nextflowVersion = '!>=23.10.0' + nextflowVersion = '!>=24.04.0' version = '1.0' doi = '' } From 60b2d6132389d43e50e941c81af257cfdf527f85 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 10 Dec 2024 13:47:08 +0100 Subject: [PATCH 14/14] bump min nextflow to 24.04.1 --- .github/workflows/ci.yml | 2 +- README.md | 2 +- nextflow.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf08231..cd2befd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: strategy: matrix: NXF_VER: - - "24.04.0" + - "24.04.1" - "latest-everything" tags: - "workflows" diff --git a/README.md b/README.md index 58d9223..bc63b5a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub Actions Linting Status](https://github.com/nf-cmgg/sampletracking/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-cmgg/sampletracking/actions/workflows/linting.yml)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.1-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/nextflow.config b/nextflow.config index 73a85e1..7004a5a 100644 --- a/nextflow.config +++ b/nextflow.config @@ -211,7 +211,7 @@ manifest { homePage = 'https://github.com/nf-cmgg/sampletracking' description = """CMGG Sampletracking workflow""" mainScript = 'main.nf' - nextflowVersion = '!>=24.04.0' + nextflowVersion = '!>=24.04.1' version = '1.0' doi = '' }