Master<-Dev for release of v1.13.3

.github/workflows/create-lint-wf.yml

@@ -48,6 +48,9 @@ jobs:
       - name: nf-core bump-version
         run: nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1
 
+      - name: nf-core lint in release mode
+        run: nf-core --log-file log.txt lint nf-core-testpipeline --fail-ignored --release
+
       - name: nf-core modules install
         run: nf-core --log-file log.txt modules install nf-core-testpipeline/ --tool fastqc
 

.github/workflows/sync.yml

@@ -4,11 +4,12 @@ on:
     types: [published]
   workflow_dispatch:
 
+env:
+  NXF_VER: 21.03.0-edge
+
 jobs:
   get-pipelines:
     runs-on: ubuntu-latest
-    env:
-      NXF_VER: 21.03.0-edge
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:

CHANGELOG.md

@@ -1,5 +1,14 @@
 # nf-core/tools: Changelog
 
+## [v1.13.3 - Copper Crocodile Resurrection :crocodile:](https://github.com/nf-core/tools/releases/tag/1.13.2) - [2021-03-24]
+
+* Running tests twice with `nf-core modules create-test-yml` to catch unreproducible md5 sums [[#890](https://github.com/nf-core/tools/issues/890)]
+* Fix sync error again where the Nextflow edge release needs to be used for some pipelines
+* Fix bug with `nf-core lint --release` (`NameError: name 'os' is not defined`)
+* Added linebreak to linting comment so that markdown header renders on PR comment properly
+* `nf-core modules create` command - if no bioconda package is found, prompt user for a different bioconda package name
+* Updated module template `main.nf` with new test data paths
+
 ## [v1.13.2 - Copper Crocodile CPR :crocodile: :face_with_head_bandage:](https://github.com/nf-core/tools/releases/tag/1.13.2) - [2021-03-23]
 
 * Make module template pass the EC linter [[#953](https://github.com/nf-core/tools/pull/953)]

nf_core/__main__.py

@@ -32,7 +32,7 @@
 
 def run_nf_core():
     # Set up the rich traceback
-    rich.traceback.install(width=200, word_wrap=True)
+    rich.traceback.install(width=200, word_wrap=True, extra_lines=1)
 
     # Print nf-core header to STDERR
     stderr = rich.console.Console(file=sys.stderr, force_terminal=nf_core.utils.rich_force_colors())

nf_core/lint/__init__.py

@@ -438,7 +438,7 @@ def _get_results_md(self):
             f"{comment_body_text}\n\n"
             f"```diff{test_passed_count}{test_ignored_count}{test_fixed_count}{test_warning_count}{test_failure_count}\n"
             "```\n\n"
-            "<details>\n"
+            "<details>\n\n"
             f"{test_failures}{test_warnings}{test_ignored}{test_fixed}{test_passes}### Run details\n\n"
             f"* nf-core/tools version {nf_core.__version__}\n"
             f"* Run at `{timestamp}`\n\n"

nf_core/lint/version_consistency.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+import os
+
 
 def version_consistency(self):
     """Pipeline and container version number consistency.

nf_core/module-template/tests/main.nf

@@ -6,11 +6,10 @@ include { {{ tool_name|upper }} } from '../../../{{ "../" if subtool else "" }}s
 
 workflow test_{{ tool_name }} {
     {% if has_meta %}
-    def input = []
     input = [ [ id:'test', single_end:false ], // meta map
-              file("${launchDir}/tests/data/genomics/sarscov2/bam/test_paired_end.bam", checkIfExists: true) ]
+              file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) ]
     {%- else %}
-    def input = file("${launchDir}/tests/data/genomics/sarscov2/bam/test_single_end.bam", checkIfExists: true)
+    input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true)
     {%- endif %}
 
     {{ tool_name|upper }} ( input )

nf_core/modules/create.py

@@ -32,6 +32,7 @@ def __init__(self, directory=".", tool="", author=None, process_label=None, has_
         self.has_meta = has_meta
         self.force_overwrite = force
 
+        self.tool_conda_name = None
         self.subtool = None
         self.tool_licence = None
         self.repo_type = None
@@ -117,26 +118,43 @@ def create(self):
         self.file_paths = self.get_module_dirs()
 
         # Try to find a bioconda package for 'tool'
-        try:
-            anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"])
-            version = anaconda_response.get("latest_version")
-            if not version:
-                version = str(max([parse_version(v) for v in anaconda_response["versions"]]))
-            self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version)
-            self.tool_description = anaconda_response.get("summary", "")
-            self.tool_doc_url = anaconda_response.get("doc_url", "")
-            self.tool_dev_url = anaconda_response.get("dev_url", "")
-            self.bioconda = "bioconda::" + self.tool + "=" + version
-            log.info(f"Using Bioconda package: '{self.bioconda}'")
-        except (ValueError, LookupError) as e:
-            log.warning(
-                f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually."
-            )
+        while True:
+            try:
+                if self.tool_conda_name:
+                    anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"])
+                else:
+                    anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"])
+                version = anaconda_response.get("latest_version")
+                if not version:
+                    version = str(max([parse_version(v) for v in anaconda_response["versions"]]))
+                self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version)
+                self.tool_description = anaconda_response.get("summary", "")
+                self.tool_doc_url = anaconda_response.get("doc_url", "")
+                self.tool_dev_url = anaconda_response.get("dev_url", "")
+                if self.tool_conda_name:
+                    self.bioconda = "bioconda::" + self.tool_conda_name + "=" + version
+                else:
+                    self.bioconda = "bioconda::" + self.tool + "=" + version
+                log.info(f"Using Bioconda package: '{self.bioconda}'")
+                break
+            except (ValueError, LookupError) as e:
+                log.warning(f"Could not find Conda dependency using the Anaconda API: '{self.tool}'")
+                if rich.prompt.Confirm.ask(f"[violet]Do you want to enter a different Bioconda package name?"):
+                    self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip()
+                    continue
+                else:
+                    log.warning(
+                        f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually."
+                    )
+                    break
 
         # Try to get the container tag (only if bioconda package was found)
         if self.bioconda:
             try:
-                self.container_tag = nf_core.utils.get_biocontainer_tag(self.tool, version)
+                if self.tool_conda_name:
+                    self.container_tag = nf_core.utils.get_biocontainer_tag(self.tool_conda_name, version)
+                else:
+                    self.container_tag = nf_core.utils.get_biocontainer_tag(self.tool, version)
                 log.info(f"Using Docker / Singularity container with tag: '{self.container_tag}'")
             except (ValueError, LookupError) as e:
                 log.info(f"Could not find a container tag ({e})")
@@ -237,7 +255,7 @@ def render_template(self):
                 fh.write(rendered_output)
 
             # Mirror file permissions
-            template_stat = os.stat(os.path.join("nf_core", "module-template", template_fn))
+            template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn))
             os.chmod(dest_fn, template_stat.st_mode)
 
     def get_repo_type(self, directory):

nf_core/modules/test_yml_builder.py

@@ -18,6 +18,7 @@
 import subprocess
 import tempfile
 import yaml
+import operator
 
 import nf_core.utils
 import nf_core.modules.pipeline_modules
@@ -134,7 +135,7 @@ def build_all_tests(self):
     def build_single_test(self, entry_point):
         """Given the supplied cli flags, prompt for any that are missing.
 
-        Returns: False if failure, None if success.
+        Returns: Test command
         """
         ep_test = {
             "name": "",
@@ -195,6 +196,21 @@ def _md5(self, fname):
         md5sum = hash_md5.hexdigest()
         return md5sum
 
+    def create_test_file_dict(self, results_dir):
+        """ Walk through directory and collect md5 sums """
+        test_files = []
+        for root, dir, file in os.walk(results_dir):
+            for elem in file:
+                elem = os.path.join(root, elem)
+                elem_md5 = self._md5(elem)
+                # Switch out the results directory path with the expected 'output' directory
+                elem = elem.replace(results_dir, "output")
+                test_files.append({"path": elem, "md5sum": elem_md5})
+
+        test_files = sorted(test_files, key=operator.itemgetter("path"))
+
+        return test_files
+
     def get_md5_sums(self, entry_point, command):
         """
         Recursively go through directories and subdirectories
@@ -206,7 +222,7 @@ def get_md5_sums(self, entry_point, command):
         run_this_test = False
         while results_dir is None:
             if self.run_tests or run_this_test:
-                results_dir = self.run_tests_workflow(command)
+                results_dir, results_dir_repeat = self.run_tests_workflow(command)
             else:
                 results_dir = rich.prompt.Prompt.ask(
                     f"[violet]Test output folder with results[/] (leave blank to run test)"
@@ -218,14 +234,16 @@ def get_md5_sums(self, entry_point, command):
                     log.error(f"Directory '{results_dir}' does not exist")
                     results_dir = None
 
-        test_files = []
-        for root, dir, file in os.walk(results_dir):
-            for elem in file:
-                elem = os.path.join(root, elem)
-                elem_md5 = self._md5(elem)
-                # Switch out the results directory path with the expected 'output' directory
-                elem = elem.replace(results_dir, "output")
-                test_files.append({"path": elem, "md5sum": elem_md5})
+        test_files = self.create_test_file_dict(results_dir=results_dir)
+        test_files_repeat = self.create_test_file_dict(results_dir=results_dir_repeat)
+
+        # Compare both test.yml files
+        for i in range(len(test_files)):
+            if not test_files[i]["md5sum"] == test_files_repeat[i]["md5sum"]:
+                test_files[i].pop("md5sum")
+                test_files[i][
+                    "contains"
+                ] = "# TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead"
 
         if len(test_files) == 0:
             raise UserWarning(f"Could not find any test result files in '{results_dir}'")
@@ -258,11 +276,16 @@ def run_tests_workflow(self, command):
                     log.info(f"Setting env var '$PROFILE' to '{profile}'")
 
         tmp_dir = tempfile.mkdtemp()
+        tmp_dir_repeat = tempfile.mkdtemp()
         command += f" --outdir {tmp_dir}"
+        command_repeat = command + f" --outdir {tmp_dir_repeat}"
 
         log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}")
         try:
             nfconfig_raw = subprocess.check_output(shlex.split(command))
+            log.info(f"Repeating test ...")
+            nfconfig_raw = subprocess.check_output(shlex.split(command_repeat))
+
         except OSError as e:
             if e.errno == errno.ENOENT and command.strip().startswith("nextflow "):
                 raise AssertionError(
@@ -276,7 +299,7 @@ def run_tests_workflow(self, command):
             log.info("Test workflow finished!")
             log.debug(nfconfig_raw)
 
-        return tmp_dir
+        return tmp_dir, tmp_dir_repeat
 
     def print_test_yml(self):
         """

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import setup, find_packages
 
-version = "1.13.2"
+version = "1.13.3"
 
 with open("README.md") as f:
     readme = f.read()