diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 569cac6599..cd87a22c4e 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -48,6 +48,9 @@ jobs: - name: nf-core bump-version run: nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1 + - name: nf-core lint in release mode + run: nf-core --log-file log.txt lint nf-core-testpipeline --fail-ignored --release + - name: nf-core modules install run: nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 957714de52..d958fbee08 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -4,11 +4,12 @@ on: types: [published] workflow_dispatch: +env: + NXF_VER: 21.03.0-edge + jobs: get-pipelines: runs-on: ubuntu-latest - env: - NXF_VER: 21.03.0-edge outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index b9ca7392ac..06ac98d0bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # nf-core/tools: Changelog +## [v1.13.3 - Copper Crocodile Resurrection :crocodile:](https://github.com/nf-core/tools/releases/tag/1.13.2) - [2021-03-24] + +* Running tests twice with `nf-core modules create-test-yml` to catch unreproducible md5 sums [[#890](https://github.com/nf-core/tools/issues/890)] +* Fix sync error again where the Nextflow edge release needs to be used for some pipelines +* Fix bug with `nf-core lint --release` (`NameError: name 'os' is not defined`) +* Added linebreak to linting comment so that markdown header renders on PR comment properly +* `nf-core modules create` command - if no bioconda package is found, prompt user for a different bioconda package name +* Updated module template `main.nf` with new test data paths + ## [v1.13.2 - Copper Crocodile CPR :crocodile: :face_with_head_bandage:](https://github.com/nf-core/tools/releases/tag/1.13.2) - [2021-03-23] * Make module template pass the EC linter [[#953](https://github.com/nf-core/tools/pull/953)] diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 5ae1a28c65..fe4932759b 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -32,7 +32,7 @@ def run_nf_core(): # Set up the rich traceback - rich.traceback.install(width=200, word_wrap=True) + rich.traceback.install(width=200, word_wrap=True, extra_lines=1) # Print nf-core header to STDERR stderr = rich.console.Console(file=sys.stderr, force_terminal=nf_core.utils.rich_force_colors()) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 4d9c5de35d..548d1e8d87 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -438,7 +438,7 @@ def _get_results_md(self): f"{comment_body_text}\n\n" f"```diff{test_passed_count}{test_ignored_count}{test_fixed_count}{test_warning_count}{test_failure_count}\n" "```\n\n" - "
\n" + "
\n\n" f"{test_failures}{test_warnings}{test_ignored}{test_fixed}{test_passes}### Run details\n\n" f"* nf-core/tools version {nf_core.__version__}\n" f"* Run at `{timestamp}`\n\n" diff --git a/nf_core/lint/version_consistency.py b/nf_core/lint/version_consistency.py index fbd90394a4..2510f3e95f 100644 --- a/nf_core/lint/version_consistency.py +++ b/nf_core/lint/version_consistency.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +import os + def version_consistency(self): """Pipeline and container version number consistency. diff --git a/nf_core/module-template/tests/main.nf b/nf_core/module-template/tests/main.nf index 8438369de8..d062eba787 100644 --- a/nf_core/module-template/tests/main.nf +++ b/nf_core/module-template/tests/main.nf @@ -6,11 +6,10 @@ include { {{ tool_name|upper }} } from '../../../{{ "../" if subtool else "" }}s workflow test_{{ tool_name }} { {% if has_meta %} - def input = [] input = [ [ id:'test', single_end:false ], // meta map - file("${launchDir}/tests/data/genomics/sarscov2/bam/test_paired_end.bam", checkIfExists: true) ] + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) ] {%- else %} - def input = file("${launchDir}/tests/data/genomics/sarscov2/bam/test_single_end.bam", checkIfExists: true) + input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) {%- endif %} {{ tool_name|upper }} ( input ) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index c788e01f7b..ed7f778bfc 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -32,6 +32,7 @@ def __init__(self, directory=".", tool="", author=None, process_label=None, has_ self.has_meta = has_meta self.force_overwrite = force + self.tool_conda_name = None self.subtool = None self.tool_licence = None self.repo_type = None @@ -117,26 +118,43 @@ def create(self): self.file_paths = self.get_module_dirs() # Try to find a bioconda package for 'tool' - try: - anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) - version = anaconda_response.get("latest_version") - if not version: - version = str(max([parse_version(v) for v in anaconda_response["versions"]])) - self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) - self.tool_description = anaconda_response.get("summary", "") - self.tool_doc_url = anaconda_response.get("doc_url", "") - self.tool_dev_url = anaconda_response.get("dev_url", "") - self.bioconda = "bioconda::" + self.tool + "=" + version - log.info(f"Using Bioconda package: '{self.bioconda}'") - except (ValueError, LookupError) as e: - log.warning( - f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually." - ) + while True: + try: + if self.tool_conda_name: + anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) + else: + anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) + version = anaconda_response.get("latest_version") + if not version: + version = str(max([parse_version(v) for v in anaconda_response["versions"]])) + self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) + self.tool_description = anaconda_response.get("summary", "") + self.tool_doc_url = anaconda_response.get("doc_url", "") + self.tool_dev_url = anaconda_response.get("dev_url", "") + if self.tool_conda_name: + self.bioconda = "bioconda::" + self.tool_conda_name + "=" + version + else: + self.bioconda = "bioconda::" + self.tool + "=" + version + log.info(f"Using Bioconda package: '{self.bioconda}'") + break + except (ValueError, LookupError) as e: + log.warning(f"Could not find Conda dependency using the Anaconda API: '{self.tool}'") + if rich.prompt.Confirm.ask(f"[violet]Do you want to enter a different Bioconda package name?"): + self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip() + continue + else: + log.warning( + f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually." + ) + break # Try to get the container tag (only if bioconda package was found) if self.bioconda: try: - self.container_tag = nf_core.utils.get_biocontainer_tag(self.tool, version) + if self.tool_conda_name: + self.container_tag = nf_core.utils.get_biocontainer_tag(self.tool_conda_name, version) + else: + self.container_tag = nf_core.utils.get_biocontainer_tag(self.tool, version) log.info(f"Using Docker / Singularity container with tag: '{self.container_tag}'") except (ValueError, LookupError) as e: log.info(f"Could not find a container tag ({e})") @@ -237,7 +255,7 @@ def render_template(self): fh.write(rendered_output) # Mirror file permissions - template_stat = os.stat(os.path.join("nf_core", "module-template", template_fn)) + template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn)) os.chmod(dest_fn, template_stat.st_mode) def get_repo_type(self, directory): diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index a0453554f5..0abb5cd719 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -18,6 +18,7 @@ import subprocess import tempfile import yaml +import operator import nf_core.utils import nf_core.modules.pipeline_modules @@ -134,7 +135,7 @@ def build_all_tests(self): def build_single_test(self, entry_point): """Given the supplied cli flags, prompt for any that are missing. - Returns: False if failure, None if success. + Returns: Test command """ ep_test = { "name": "", @@ -195,6 +196,21 @@ def _md5(self, fname): md5sum = hash_md5.hexdigest() return md5sum + def create_test_file_dict(self, results_dir): + """ Walk through directory and collect md5 sums """ + test_files = [] + for root, dir, file in os.walk(results_dir): + for elem in file: + elem = os.path.join(root, elem) + elem_md5 = self._md5(elem) + # Switch out the results directory path with the expected 'output' directory + elem = elem.replace(results_dir, "output") + test_files.append({"path": elem, "md5sum": elem_md5}) + + test_files = sorted(test_files, key=operator.itemgetter("path")) + + return test_files + def get_md5_sums(self, entry_point, command): """ Recursively go through directories and subdirectories @@ -206,7 +222,7 @@ def get_md5_sums(self, entry_point, command): run_this_test = False while results_dir is None: if self.run_tests or run_this_test: - results_dir = self.run_tests_workflow(command) + results_dir, results_dir_repeat = self.run_tests_workflow(command) else: results_dir = rich.prompt.Prompt.ask( f"[violet]Test output folder with results[/] (leave blank to run test)" @@ -218,14 +234,16 @@ def get_md5_sums(self, entry_point, command): log.error(f"Directory '{results_dir}' does not exist") results_dir = None - test_files = [] - for root, dir, file in os.walk(results_dir): - for elem in file: - elem = os.path.join(root, elem) - elem_md5 = self._md5(elem) - # Switch out the results directory path with the expected 'output' directory - elem = elem.replace(results_dir, "output") - test_files.append({"path": elem, "md5sum": elem_md5}) + test_files = self.create_test_file_dict(results_dir=results_dir) + test_files_repeat = self.create_test_file_dict(results_dir=results_dir_repeat) + + # Compare both test.yml files + for i in range(len(test_files)): + if not test_files[i]["md5sum"] == test_files_repeat[i]["md5sum"]: + test_files[i].pop("md5sum") + test_files[i][ + "contains" + ] = "# TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead" if len(test_files) == 0: raise UserWarning(f"Could not find any test result files in '{results_dir}'") @@ -258,11 +276,16 @@ def run_tests_workflow(self, command): log.info(f"Setting env var '$PROFILE' to '{profile}'") tmp_dir = tempfile.mkdtemp() + tmp_dir_repeat = tempfile.mkdtemp() command += f" --outdir {tmp_dir}" + command_repeat = command + f" --outdir {tmp_dir_repeat}" log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}") try: nfconfig_raw = subprocess.check_output(shlex.split(command)) + log.info(f"Repeating test ...") + nfconfig_raw = subprocess.check_output(shlex.split(command_repeat)) + except OSError as e: if e.errno == errno.ENOENT and command.strip().startswith("nextflow "): raise AssertionError( @@ -276,7 +299,7 @@ def run_tests_workflow(self, command): log.info("Test workflow finished!") log.debug(nfconfig_raw) - return tmp_dir + return tmp_dir, tmp_dir_repeat def print_test_yml(self): """ diff --git a/setup.py b/setup.py index 9f27ad7dde..605e08406b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = "1.13.2" +version = "1.13.3" with open("README.md") as f: readme = f.read()