Merge branch 'dev' into fix-update

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -33,7 +33,7 @@ body:
  attributes:
  label: System information
  description: |
- * Nextflow version _(eg. 22.10.1)_
+ * Nextflow version _(eg. 23.04.0)_
  * Hardware _(eg. HPC, Desktop, Cloud)_
  * Executor _(eg. slurm, local, awsbatch)_
  * OS _(eg. CentOS Linux, macOS, Linux Mint)_

.github/workflows/create-lint-wf.yml

@@ -23,7 +23,7 @@ jobs:
  strategy:
  matrix:
  NXF_VER:
- - "22.10.1"
+ - "23.04.0"
  - "latest-everything"
  steps:
  # Get the repo code

.github/workflows/create-test-wf.yml

@@ -23,7 +23,7 @@ jobs:
  strategy:
  matrix:
  NXF_VER:
- - "22.10.1"
+ - "23.04.0"
  - "latest-everything"
  steps:
  - uses: actions/checkout@v3

CHANGELOG.md

@@ -12,7 +12,13 @@
 - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287)
 - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291))
 - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298))
+- Updates seqeralabs/action-tower-launch to v2.0.0 ([#2301](https://github.com/nf-core/tools/pull/2301))
 - Remove schema validation from `lib` folder and use Nextflow [nf-validation plugin](https://nextflow-io.github.io/nf-validation/) instead ([#1771](https://github.com/nf-core/tools/pull/1771/))
+- Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306))
+- Add ability to specify custom registry for linting modules, defaults to quay.io ([#2313](https://github.com/nf-core/tools/pull/2313))
+- Add `singularity.registry = 'quay.io'` in pipeline template
+- Bump minimum required NF version in pipeline template from `22.10.1` -> `23.04.0`
+- Add ability to interpret `docker.registry` from `nextflow.config` file. If not found defaults to quay.io. ([#2318](https://github.com/nf-core/tools/pull/2318))
 
 ### Download
 

README.md

@@ -941,6 +941,8 @@ before_command: sed 's/1.13a/1.10/g' modules/multiqc/main.nf > modules/multiqc/m
 
 To run unit tests of a module that you have installed or the test created by the command [`nf-core modules create-test-yml`](#create-a-module-test-config-file), you can use `nf-core modules test` command. This command runs the tests specified in `modules/tests/software/<tool>/<subtool>/test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/).
 
+> This command uses the pytest argument `--git-aware` to avoid copying the whole `.git` directory and files ignored by `git`. This means that it will only include files listed by `git ls-files`. Remember to **commit your changes** after adding a new module to add the new files to your git index.
+
 You can specify the module name in the form TOOL/SUBTOOL in command line or provide it later by prompts.
 
 <!-- RICH-CODEX
@@ -1199,6 +1201,8 @@ extra_env:
 
 To run unit tests of a subworkflow that you have installed or the test created by the command [`nf-core subworkflow create-test-yml`](#create-a-subworkflow-test-config-file), you can use `nf-core subworkflows test` command. This command runs the tests specified in `tests/subworkflows/<subworkflow_name>/test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/).
 
+> This command uses the pytest argument `--git-aware` to avoid copying the whole `.git` directory and files ignored by `git`. This means that it will only include files listed by `git ls-files`. Remember to **commit your changes** after adding a new subworkflow to add the new files to your git index.
+
 You can specify the subworkflow name in the form TOOL/SUBTOOL in command line or provide it later by prompts.
 
 <!-- RICH-CODEX

nf_core/__main__.py

@@ -807,6 +807,14 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
 @click.pass_context
 @click.argument("tool", type=str, required=False, metavar="<tool> or <tool/subtool>")
 @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="<pipeline/modules directory>")
+@click.option(
+ "-r",
+ "--registry",
+ type=str,
+ metavar="<registry>",
+ default=None,
+ help="Registry to use for containers. If not specified it will use docker.registry value in the nextflow.config file",
+)
 @click.option("-k", "--key", type=str, metavar="<test>", multiple=True, help="Run only these lint tests")
 @click.option("-a", "--all", is_flag=True, help="Run on all modules")
 @click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures")
@@ -821,7 +829,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
 )
 @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available")
 def lint(
- ctx, tool, dir, key, all, fail_warned, local, passed, sort_by, fix_version
+ ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version
 ): # pylint: disable=redefined-outer-name
  """
  Lint one or more modules in a directory.
@@ -839,13 +847,15 @@ def lint(
  module_lint = ModuleLint(
  dir,
  fail_warned=fail_warned,
+ registry=ctx.params["registry"],
  remote_url=ctx.obj["modules_repo_url"],
  branch=ctx.obj["modules_repo_branch"],
  no_pull=ctx.obj["modules_repo_no_pull"],
  hide_progress=ctx.obj["hide_progress"],
  )
  module_lint.lint(
  module=tool,
+ registry=registry,
  key=key,
  all_modules=all,
  print_results=True,

nf_core/components/components_test.py

@@ -7,6 +7,7 @@
 import pytest
 import questionary
 import rich
+from git import InvalidGitRepositoryError, Repo
 
 import nf_core.modules.modules_utils
 import nf_core.utils
@@ -179,6 +180,14 @@ def _run_pytests(self):
  console = rich.console.Console()
  console.rule(self.component_name, style="black")
 
+ # Check uncommitted changed
+ try:
+ repo = Repo(self.dir)
+ if repo.is_dirty():
+ log.warning("You have uncommitted changes. Make sure to commit last changes before running the tests.")
+ except InvalidGitRepositoryError:
+ pass
+
  # Set pytest arguments
  tag = self.component_name
  if self.component_type == "subworkflows":

nf_core/lint/readme.py

@@ -36,7 +36,7 @@ def readme(self):
 
  if "nextflow_badge" not in ignore_configs:
  # Check that there is a readme badge showing the minimum required version of Nextflow
- # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/)
+ # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/)
  # and that it has the correct version
  nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)"
  match = re.search(nf_badge_re, content)

nf_core/modules/lint/__init__.py

@@ -69,6 +69,7 @@ def __init__(
  remote_url=None,
  branch=None,
  no_pull=False,
+ registry=None,
  hide_progress=False,
  ):
  super().__init__(
@@ -114,6 +115,7 @@ def __init__(
  )
  for m in self.get_local_components()
  ]
+ self.config = nf_core.utils.fetch_wf_config(self.dir, cache_config=True)
  else:
  module_dir = Path(self.dir, self.default_modules_path)
  self.all_remote_modules = [
@@ -124,6 +126,15 @@ def __init__(
  if not self.all_remote_modules:
  raise LookupError("No modules in 'modules' directory")
 
+ # This could be better, perhaps glob for all nextflow.config files in?
+ self.config = nf_core.utils.fetch_wf_config(Path(self.dir).joinpath("tests", "config"), cache_config=True)
+
+ if registry is None:
+ self.registry = self.config.get("docker.registry", "quay.io")
+ else:
+ self.registry = registry
+ log.debug(f"Registry set to {self.registry}")
+
  self.lint_config = None
  self.modules_json = None
 
@@ -145,6 +156,7 @@ def get_all_lint_tests(is_pipeline):
  def lint(
  self,
  module=None,
+ registry="quay.io",
  key=(),
  all_modules=False,
  print_results=True,
@@ -227,11 +239,11 @@ def lint(
 
  # Lint local modules
  if local and len(local_modules) > 0:
- self.lint_modules(local_modules, local=True, fix_version=fix_version)
+ self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version)
 
  # Lint nf-core modules
  if len(remote_modules) > 0:
- self.lint_modules(remote_modules, local=False, fix_version=fix_version)
+ self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version)
 
  if print_results:
  self._print_results(show_passed=show_passed, sort_by=sort_by)
@@ -264,12 +276,13 @@ def filter_tests_by_key(self, key):
  # If -k supplied, only run these tests
  self.lint_tests = [k for k in self.lint_tests if k in key]
 
- def lint_modules(self, modules, local=False, fix_version=False):
+ def lint_modules(self, modules, registry="quay.io", local=False, fix_version=False):
  """
  Lint a list of modules
 
  Args:
  modules ([NFCoreModule]): A list of module objects
+ registry (str): The container registry to use. Should be quay.io in most situations.
  local (boolean): Whether the list consist of local or nf-core modules
  fix_version (boolean): Fix the module version if a newer version is available
  """
@@ -290,9 +303,9 @@ def lint_modules(self, modules, local=False, fix_version=False):
 
  for mod in modules:
  progress_bar.update(lint_progress, advance=1, test_name=mod.module_name)
- self.lint_module(mod, progress_bar, local=local, fix_version=fix_version)
+ self.lint_module(mod, progress_bar, registry=registry, local=local, fix_version=fix_version)
 
- def lint_module(self, mod, progress_bar, local=False, fix_version=False):
+ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=False):
  """
  Perform linting on one module
 
@@ -311,7 +324,7 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False):
 
  # Only check the main script in case of a local module
  if local:
- self.main_nf(mod, fix_version, progress_bar)
+ self.main_nf(mod, fix_version, self.registry, progress_bar)
  self.passed += [LintResult(mod, *m) for m in mod.passed]
  warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)]
  if not self.fail_warned:
@@ -323,7 +336,7 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False):
  else:
  for test_name in self.lint_tests:
  if test_name == "main_nf":
- getattr(self, test_name)(mod, fix_version, progress_bar)
+ getattr(self, test_name)(mod, fix_version, self.registry, progress_bar)
  else:
  getattr(self, test_name)(mod)
 

nf_core/modules/lint/main_nf.py

@@ -17,7 +17,7 @@
 log = logging.getLogger(__name__)
 
 
-def main_nf(module_lint_object, module, fix_version, progress_bar):
+def main_nf(module_lint_object, module, fix_version, registry, progress_bar):
  """
  Lint a ``main.nf`` module file
 
@@ -121,7 +121,7 @@ def main_nf(module_lint_object, module, fix_version, progress_bar):
  module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf))
 
  # Check the process definitions
- if check_process_section(module, process_lines, fix_version, progress_bar):
+ if check_process_section(module, process_lines, registry, fix_version, progress_bar):
  module.passed.append(("main_nf_container", "Container versions match", module.main_nf))
  else:
  module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf))
@@ -209,12 +209,20 @@ def check_when_section(self, lines):
  self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf))
 
 
-def check_process_section(self, lines, fix_version, progress_bar):
- """
- Lint the section of a module between the process definition
+def check_process_section(self, lines, registry, fix_version, progress_bar):
+ """Lint the section of a module between the process definition
  and the 'input:' definition
  Specifically checks for correct software versions
  and containers
+
+ Args:
+ lines (List[str]): Content of process.
+ registry (str): Base Docker registry for containers. Typically quay.io.
+ fix_version (bool): Fix software version
+ progress_bar (ProgressBar): Progress bar to update.
+
+ Returns:
+ Optional[bool]: True if singularity and docker containers match, False otherwise. If process definition does not exist, None.
  """
  # Check that we have a process section
  if len(lines) == 0:
@@ -223,8 +231,8 @@ def check_process_section(self, lines, fix_version, progress_bar):
  self.passed.append(("process_exist", "Process definition exists", self.main_nf))
 
  # Checks that build numbers of bioconda, singularity and docker container are matching
- singularity_tag = "singularity"
- docker_tag = "docker"
+ singularity_tag = None
+ docker_tag = None
  bioconda_packages = []
 
  # Process name should be all capital letters
@@ -240,7 +248,12 @@ def check_process_section(self, lines, fix_version, progress_bar):
  # Deprecated enable_conda
  for i, l in enumerate(lines):
  url = None
- l = l.strip(" '\"")
+ l = l.strip(" \n'\"}:")
+
+ # Catch preceeding "container "
+ if l.startswith("container"):
+ l = l.replace("container", "").strip(" \n'\"}:")
+
  if _container_type(l) == "conda":
  bioconda_packages = [b for b in l.split() if "bioconda::" in b]
  match = re.search(r"params\.enable_conda", l)
@@ -261,9 +274,10 @@ def check_process_section(self, lines, fix_version, progress_bar):
  )
  )
  if _container_type(l) == "singularity":
- # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1
- # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0
- match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?'", l)
+ # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img -> v1.2.0_cv1
+ # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0 -> 0.11.9--0
+ # Please god let's find a better way to do this than regex
+ match = re.search(r"(?:[:.])?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l)
  if match is not None:
  singularity_tag = match.group(1)
  self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf))
@@ -273,16 +287,16 @@ def check_process_section(self, lines, fix_version, progress_bar):
  url = urlparse(l.split("'")[0])
 
  if _container_type(l) == "docker":
- # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5
- # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1
- match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l)
+ # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5 -> 2.7.1--pl526_5
+ # e.g. "biocontainers/biocontainers:v1.2.0_cv1 -> v1.2.0_cv1
+ match = re.search(r":([A-Za-z\d\-_.]+)$", l)
  if match is not None:
  docker_tag = match.group(1)
  self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf))
  else:
  self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf))
- docker_tag = NoneD
- if l.startswith("quay.io/"):
+ docker_tag = None
+ if l.startswith(registry):
  l_stripped = re.sub(r"\W+$", "", l)
  self.failed.append(
  (
@@ -297,7 +311,7 @@ def check_process_section(self, lines, fix_version, progress_bar):
  # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04)
  # If so, add quay.io as default container prefix
  if l.count("/") == 1 and l.count(":") == 1:
- l = "quay.io/" + l
+ l = "/".join([registry, l]).replace("//", "/")
  url = urlparse(l.split("'")[0])
 
  # lint double quotes
@@ -320,7 +334,7 @@ def check_process_section(self, lines, fix_version, progress_bar):
  )
 
  # lint more than one container in the same line
- if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l):
+ if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or l.startswith(registry)):
  self.warned.append(
  (
  "container_links",
@@ -412,7 +426,11 @@ def check_process_section(self, lines, fix_version, progress_bar):
  else:
  self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf))
 
- return docker_tag == singularity_tag
+ # Check if a tag exists at all. If not, return None.
+ if singularity_tag is None or docker_tag is None:
+ return None
+ else:
+ return docker_tag == singularity_tag
 
 
 def check_process_labels(self, lines):
@@ -591,7 +609,7 @@ def _container_type(line):
  """Returns the container type of a build."""
  if line.startswith("conda"):
  return "conda"
- if line.startswith("https://containers") or line.startswith("https://depot"):
+ if line.startswith("https://") or line.startswith("https://depot"):
  # Look for a http download URL.
  # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980
  url_regex = (
@@ -601,5 +619,5 @@ def _container_type(line):
  if url_match:
  return "singularity"
  return None
- if line.count("/") >= 1 and line.count(":") == 1:
+ if line.count("/") >= 1 and line.count(":") == 1 and line.count(" ") == 0 and "https://" not in line:
  return "docker"

nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -42,7 +42,7 @@ body:
  attributes:
  label: System information
  description: |
- * Nextflow version _(eg. 22.10.1)_
+ * Nextflow version _(eg. 23.04.0)_
  * Hardware _(eg. HPC, Desktop, Cloud)_
  * Executor _(eg. slurm, local, awsbatch)_
  * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_