From 68c0e7e59334f7fb048d367bb67d6bec7ff66898 Mon Sep 17 00:00:00 2001 From: Tobias Date: Mon, 8 Apr 2019 15:32:04 +0200 Subject: [PATCH 1/9] add params_description scope to config --- .../{{cookiecutter.name_noslash}}/nextflow.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 6f0d53a0bb..3c09431ca2 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -39,6 +39,11 @@ params { // Developmental code should specify :dev process.container = '{{ cookiecutter.name_docker }}:dev' +// Specify params.settings.json settings +params_description { + path = "$baseDir/parameters.settings.json" +} + // Load base.config by default for all pipelines includeConfig 'conf/base.config' From bf617330166f5239ac594ba5fde6073b3e926637 Mon Sep 17 00:00:00 2001 From: Tobias Date: Tue, 9 Apr 2019 10:09:57 +0200 Subject: [PATCH 2/9] add parameters.settings.json file --- .../parameters.settings.json | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/parameters.settings.json diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/parameters.settings.json b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/parameters.settings.json new file mode 100644 index 0000000000..689a815609 --- /dev/null +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/parameters.settings.json @@ -0,0 +1,117 @@ +{ + "parameters": [ + { + "name": "reads", + "label": "Input files", + "usage": "Specify the location of your input FastQ files.", + "group": "Main options", + "default_value": "'data/*{1,2}.fastq.gz'", + "render": "textfield", + "pattern": ".*\\*.*", + "type": "string" + }, + { + "name": "genome", + "label": "Alignment reference iGenomes key", + "usage": "Ref. genome key for iGenomes", + "group": "Alignment", + "render": "drop-down", + "type": "string", + "choices": [ + "", + "GRCh37", + "GRCm38", + "TAIR10", + "EB2", + "UMD3.1", + "WBcel235", + "CanFam3.1", + "GRCz10", + "BDGP6", + "EquCab2", + "EB1", + "Galgal4", + "Gm01", + "Mmul_1", + "IRGSP-1.0", + "CHIMP2.1.4", + "Rnor_6.0", + "R64-1-1", + "EF2", + "Sbi1", + "Sscrofa10.2", + "AGPv3" + ], + "default_value": "" + }, + { + "name": "singleEnd", + "label": "Single-end sequencing input", + "usage": "Use single-end sequencing inputs instead of paired-end.", + "group": "Main options", + "render": "check-box", + "default_value": false, + "type": "boolean" + }, + { + "name": "fasta", + "label": "FASTA", + "usage": "Path to Fasta reference", + "group": "Alignment", + "render": "file", + "type": "string", + "pattern": ".*", + "default_value": "" + }, + { + "name": "outdir", + "label": "Output directory", + "usage": "Set where to save the results from the pipeline", + "group": "Main options", + "default_value": "./results", + "render": "textfield", + "pattern": ".*", + "type": "string" + }, + { + "name": "email", + "label": "Your email address", + "usage": "Your email address, required to receive completion notification.", + "group": "Pipeline defaults", + "render": "textfield", + "pattern": "^$|(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$)", + "type": "string", + "default_value": "" + }, + { + "name": "maxMultiqcEmailFileSize", + "label": "Maximum MultiQC email file size", + "usage": "Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached.", + "group": "Pipeline defaults", + "default_value": "25.MB", + "render": "textfield", + "pattern": "\\d+\\.[KMGT]?B", + "type": "string" + }, + { + "name": "awsregion", + "label": "AWS Region", + "usage": "The AWS region to run your job in.", + "group": "AWS cloud usage", + "default_value": "eu-west-1", + "render": "textfield", + "pattern": ".*", + "type": "string" + }, + { + "name": "awsqueue", + "label": "AWS job queue", + "usage": "The JobQueue that you intend to use on AWS Batch.", + "group": "AWS cloud usage", + "render": "textfield", + "pattern": ".*", + "type": "string", + "default_value": "" + } + ] +} From 8584d5c3b1a8b5cf9bdfaeff9ebbe02052013f2c Mon Sep 17 00:00:00 2001 From: Tobias Date: Wed, 10 Apr 2019 13:53:22 +0200 Subject: [PATCH 3/9] parameters.settings.json included in linting add check for file exists `parameters.settings.json` add check for valid JSON format --- nf_core/lint.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index cd90406464..1e59d72936 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -4,7 +4,7 @@ Tests Nextflow-based pipelines to check that they adhere to the nf-core community guidelines. """ - +import json import logging import io import os @@ -171,7 +171,8 @@ def lint_pipeline(self, release_mode=False): 'check_readme', 'check_conda_env_yaml', 'check_conda_dockerfile', - 'check_pipeline_todos' + 'check_pipeline_todos', + 'check_params_json' ] if release_mode: self.release_mode = True @@ -221,6 +222,7 @@ def check_files_exist(self): ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling 'README.md', 'CHANGELOG.md', + 'parameters.settings.json', 'docs/README.md', 'docs/output.md', 'docs/usage.md', @@ -263,6 +265,7 @@ def pf(file_path): with open(os.path.join(self.path, 'environment.yml'), 'r') as fh: self.conda_config = yaml.load(fh) + def check_docker(self): """Checks that Dockerfile contains the string ``FROM``.""" fn = os.path.join(self.path, "Dockerfile") @@ -502,6 +505,7 @@ def check_ci_config(self): elif minNextflowVersion != self.minNextflowVersion: self.failed.append((5, "Minimum NF version differed from CI and what was set in the pipelines manifest: {}".format(fn))) + def check_readme(self): """Checks the repository README file for errors. @@ -785,6 +789,18 @@ def check_pipeline_todos(self): l = '{}..'.format(l[:50-len(fname)]) self.warned.append((10, "TODO string found in '{}': {}".format(fname,l))) + + def check_params_json(self): + fn = os.path.join(self.path, 'parameters.settings.json') + with open(fn, 'r') as paramsfile: + try: + json_content = json.loads(paramsfile.read()) + except(ValueError): + self.failed.append((11, "{} file not a valid JSON file.".format(fn))) + + + return + def print_results(self): # Print results rl = "\n Using --release mode linting tests" if self.release_mode else '' From dc8171c1cc252526756ccf073cf859f18d9fa5bf Mon Sep 17 00:00:00 2001 From: Tobias Date: Wed, 10 Apr 2019 14:22:07 +0200 Subject: [PATCH 4/9] removed unused variable --- nf_core/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 1e59d72936..7f03ccd277 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -794,7 +794,7 @@ def check_params_json(self): fn = os.path.join(self.path, 'parameters.settings.json') with open(fn, 'r') as paramsfile: try: - json_content = json.loads(paramsfile.read()) + json.loads(paramsfile.read()) except(ValueError): self.failed.append((11, "{} file not a valid JSON file.".format(fn))) From 5a77fe670abd939f79025029da4f4d7210ca3769 Mon Sep 17 00:00:00 2001 From: Tobias Date: Wed, 10 Apr 2019 15:23:50 +0200 Subject: [PATCH 5/9] added parameters.settings.json to minimal working example --- .../parameters.settings.json | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tests/lint_examples/minimal_working_example/parameters.settings.json diff --git a/tests/lint_examples/minimal_working_example/parameters.settings.json b/tests/lint_examples/minimal_working_example/parameters.settings.json new file mode 100644 index 0000000000..689a815609 --- /dev/null +++ b/tests/lint_examples/minimal_working_example/parameters.settings.json @@ -0,0 +1,117 @@ +{ + "parameters": [ + { + "name": "reads", + "label": "Input files", + "usage": "Specify the location of your input FastQ files.", + "group": "Main options", + "default_value": "'data/*{1,2}.fastq.gz'", + "render": "textfield", + "pattern": ".*\\*.*", + "type": "string" + }, + { + "name": "genome", + "label": "Alignment reference iGenomes key", + "usage": "Ref. genome key for iGenomes", + "group": "Alignment", + "render": "drop-down", + "type": "string", + "choices": [ + "", + "GRCh37", + "GRCm38", + "TAIR10", + "EB2", + "UMD3.1", + "WBcel235", + "CanFam3.1", + "GRCz10", + "BDGP6", + "EquCab2", + "EB1", + "Galgal4", + "Gm01", + "Mmul_1", + "IRGSP-1.0", + "CHIMP2.1.4", + "Rnor_6.0", + "R64-1-1", + "EF2", + "Sbi1", + "Sscrofa10.2", + "AGPv3" + ], + "default_value": "" + }, + { + "name": "singleEnd", + "label": "Single-end sequencing input", + "usage": "Use single-end sequencing inputs instead of paired-end.", + "group": "Main options", + "render": "check-box", + "default_value": false, + "type": "boolean" + }, + { + "name": "fasta", + "label": "FASTA", + "usage": "Path to Fasta reference", + "group": "Alignment", + "render": "file", + "type": "string", + "pattern": ".*", + "default_value": "" + }, + { + "name": "outdir", + "label": "Output directory", + "usage": "Set where to save the results from the pipeline", + "group": "Main options", + "default_value": "./results", + "render": "textfield", + "pattern": ".*", + "type": "string" + }, + { + "name": "email", + "label": "Your email address", + "usage": "Your email address, required to receive completion notification.", + "group": "Pipeline defaults", + "render": "textfield", + "pattern": "^$|(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\\.[a-zA-Z0-9-.]+$)", + "type": "string", + "default_value": "" + }, + { + "name": "maxMultiqcEmailFileSize", + "label": "Maximum MultiQC email file size", + "usage": "Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached.", + "group": "Pipeline defaults", + "default_value": "25.MB", + "render": "textfield", + "pattern": "\\d+\\.[KMGT]?B", + "type": "string" + }, + { + "name": "awsregion", + "label": "AWS Region", + "usage": "The AWS region to run your job in.", + "group": "AWS cloud usage", + "default_value": "eu-west-1", + "render": "textfield", + "pattern": ".*", + "type": "string" + }, + { + "name": "awsqueue", + "label": "AWS job queue", + "usage": "The JobQueue that you intend to use on AWS Batch.", + "group": "AWS cloud usage", + "render": "textfield", + "pattern": ".*", + "type": "string", + "default_value": "" + } + ] +} From 728f08126d2ac97c640ef4ffa77fe64d2295d9ae Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Wed, 10 Apr 2019 15:40:33 +0200 Subject: [PATCH 6/9] Fixes unittests --- tests/test_lint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index c179547019..10f288c4c6 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -38,7 +38,7 @@ def pf(wd, path): pf(WD, 'lint_examples/license_incomplete_example')] # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 59 +MAX_PASS_CHECKS = 60 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -91,7 +91,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 4, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 5 - 1} + expectations = {"failed": 5, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 5 - 1} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): From ac1aa0ddfd131253a9f89898c94043b12067df64 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Wed, 10 Apr 2019 15:45:11 +0200 Subject: [PATCH 7/9] Adjusts expected passed checks --- tests/test_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 10f288c4c6..69700a3b60 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -91,7 +91,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 5, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 5 - 1} + expectations = {"failed": 5, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 4 - 1} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self): From 4bf8ed39de4ccc53d0ce3b5579b954acf15ecede Mon Sep 17 00:00:00 2001 From: Tobias Date: Wed, 10 Apr 2019 15:46:52 +0200 Subject: [PATCH 8/9] Dynamic Help Message add dynamic help message from parameters.settings.json file use code from nf-core/hlatyping --- .../{{cookiecutter.name_noslash}}/main.nf | 81 +++++++++++++------ 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index a3d90900ad..1e936ebd36 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -10,10 +10,58 @@ */ -def helpMessage() { - // TODO nf-core: Add to this help message with new command line parameters +def readParamsFromJsonSettings() { + List paramsWithUsage + try { + paramsWithUsage = tryReadParamsFromJsonSettings() + } catch (Exception e) { + println "Could not read parameters settings from Json. $e" + paramsWithUsage = Collections.emptyMap() + } + return paramsWithUsage +} + +def tryReadParamsFromJsonSettings() throws Exception { + def paramsContent = new File(config.params_description.path).text + def paramsWithUsage = new groovy.json.JsonSlurper().parseText(paramsContent) + return paramsWithUsage.get('parameters') +} + +def formatParameterHelpData(param) { + result = [name: param.name, value: '', usage: param.usage] + // value descibes the expected input for the param + result.value = (param.type == boolean.toString()) ? '' : param.choices ?: param.type ?: '' + return result +} + +String prettyFormatParamGroupWithPaddingAndIndent(List paramGroup, String groupName, Integer padding = 2, Integer indent = 4) { + def maxParamNameLength = paramGroup.collect { it.name.size() }.max() + def paramChoices = paramGroup.findAll { it.choices }.collect { it.choices } + def maxChoiceStringLength = paramChoices.collect { it.toString().size() }.max() + def maxTypeLength = paramGroup.collect { (it.type as String).size() }.max() + + def paramsFormattedList = paramGroup.sort { it.name }.collect { + Map param -> + paramHelpData = formatParameterHelpData(param) + sprintf("%${indent}s%-${maxParamNameLength + padding}s%-${maxChoiceStringLength + padding}s %s\n", "", "--${paramHelpData.name}", "${paramHelpData.value}", "${paramHelpData.usage}") + } + return String.format("%s:\n%s", groupName.toUpperCase(), paramsFormattedList.join()).stripIndent() +} + +String prettyFormatParamsWithPaddingAndIndent(List paramsWithUsage, Integer padding = 2, Integer indent = 4) { + + def groupedParamsWithUsage = paramsWithUsage.groupBy { it.group } + def formattedParamsGroups = groupedParamsWithUsage.collect { + prettyFormatParamGroupWithPaddingAndIndent(it.value, it.key, padding, indent) + } + return formattedParamsGroups.join('\n') +} + + +def helpMessage(paramsWithUsage) { + // TODO nf-core: Add typical command for running your pipeline log.info nfcoreHeader() - log.info""" + def usageHelp = String.format("""\ Usage: @@ -21,37 +69,22 @@ def helpMessage() { nextflow run {{ cookiecutter.name }} --reads '*_R{1,2}.fastq.gz' -profile docker - Mandatory arguments: - --reads Path to input data (must be surrounded with quotes) - -profile Configuration profile to use. Can use multiple (comma separated) - Available: conda, docker, singularity, awsbatch, test and more. - Options: - --genome Name of iGenomes reference - --singleEnd Specifies that the input is single end reads - - References If not specified in the configuration file or you wish to overwrite any of the references. - --fasta Path to Fasta reference - - Other options: - --outdir The output directory where the results will be saved - --email Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits - --maxMultiqcEmailFileSize Theshold size for MultiQC report to be attached in notification email. If file generated by pipeline exceeds the threshold, it will not be attached (Default: 25MB) - -name Name for the pipeline run. If not specified, Nextflow will automatically generate a random mnemonic. - AWSBatch options: - --awsqueue The AWSBatch JobQueue that needs to be set when running on AWSBatch - --awsregion The AWS Region for your AWS Batch job to run on - """.stripIndent() + %s + """.stripIndent(), prettyFormatParamsWithPaddingAndIndent(paramsWithUsage, 2, 4)) + log.info usageHelp } /* * SET UP CONFIGURATION VARIABLES */ +def paramsWithUsage = readParamsFromJsonSettings() + // Show help emssage if (params.help){ - helpMessage() + helpMessage(paramsWithUsage) exit 0 } From e3b57c102745e858089da01e3d9b60dc1464faab Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Wed, 10 Apr 2019 15:55:56 +0200 Subject: [PATCH 9/9] Corrects missing file ut --- tests/test_lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index 69700a3b60..ef23c54c40 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -91,7 +91,7 @@ def test_failing_missingfiles_example(self): """Tests for missing files like Dockerfile or LICENSE""" lint_obj = nf_core.lint.PipelineLint(PATH_FAILING_EXAMPLE) lint_obj.check_files_exist() - expectations = {"failed": 5, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 4 - 1} + expectations = {"failed": 5, "warned": 1, "passed": len(listfiles(PATH_WORKING_EXAMPLE)) - 6 - 1} self.assess_lint_status(lint_obj, **expectations) def test_mit_licence_example_pass(self):