Add custom registry to nfcore modules lint

CHANGELOG.md

@@ -13,6 +13,7 @@
 - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291))
 - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298))
 - Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306))
+- Add ability to specify custom registry for linting modules, defaults to quay.io ([#2313](https://github.com/nf-core/tools/pull/2313))
 
 ### Download
 

nf_core/__main__.py

@@ -807,6 +807,9 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
 @click.pass_context
 @click.argument("tool", type=str, required=False, metavar="<tool> or <tool/subtool>")
 @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="<pipeline/modules directory>")
+@click.option(
+    "-r", "--registry", type=str, metavar="<registry>", default="quay.io", help="Registry to use for containers"
+)
 @click.option("-k", "--key", type=str, metavar="<test>", multiple=True, help="Run only these lint tests")
 @click.option("-a", "--all", is_flag=True, help="Run on all modules")
 @click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures")
@@ -821,7 +824,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
 )
 @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available")
 def lint(
-    ctx, tool, dir, key, all, fail_warned, local, passed, sort_by, fix_version
+    ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version
 ):  # pylint: disable=redefined-outer-name
     """
     Lint one or more modules in a directory.
@@ -846,6 +849,7 @@ def lint(
         )
         module_lint.lint(
             module=tool,
+            registry=registry,
             key=key,
             all_modules=all,
             print_results=True,

nf_core/modules/lint/__init__.py

@@ -145,6 +145,7 @@ def get_all_lint_tests(is_pipeline):
     def lint(
         self,
         module=None,
+        registry="quay.io",
         key=(),
         all_modules=False,
         print_results=True,
@@ -227,11 +228,11 @@ def lint(
 
         # Lint local modules
         if local and len(local_modules) > 0:
-            self.lint_modules(local_modules, local=True, fix_version=fix_version)
+            self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version)
 
         # Lint nf-core modules
         if len(remote_modules) > 0:
-            self.lint_modules(remote_modules, local=False, fix_version=fix_version)
+            self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version)
 
         if print_results:
             self._print_results(show_passed=show_passed, sort_by=sort_by)
@@ -264,12 +265,13 @@ def filter_tests_by_key(self, key):
         # If -k supplied, only run these tests
         self.lint_tests = [k for k in self.lint_tests if k in key]
 
-    def lint_modules(self, modules, local=False, fix_version=False):
+    def lint_modules(self, modules, registry="quay.io", local=False, fix_version=False):
         """
         Lint a list of modules
 
         Args:
             modules ([NFCoreModule]): A list of module objects
+            registry (str): The container registry to use. Should be quay.io in most situations.
             local (boolean): Whether the list consist of local or nf-core modules
             fix_version (boolean): Fix the module version if a newer version is available
         """
@@ -290,9 +292,9 @@ def lint_modules(self, modules, local=False, fix_version=False):
 
             for mod in modules:
                 progress_bar.update(lint_progress, advance=1, test_name=mod.module_name)
-                self.lint_module(mod, progress_bar, local=local, fix_version=fix_version)
+                self.lint_module(mod, progress_bar, registry=registry, local=local, fix_version=fix_version)
 
-    def lint_module(self, mod, progress_bar, local=False, fix_version=False):
+    def lint_module(self, mod, progress_bar, registry, local=False, fix_version=False):
         """
         Perform linting on one module
 
@@ -311,7 +313,7 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False):
 
         # Only check the main script in case of a local module
         if local:
-            self.main_nf(mod, fix_version, progress_bar)
+            self.main_nf(mod, fix_version, registry, progress_bar)
             self.passed += [LintResult(mod, *m) for m in mod.passed]
             warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)]
             if not self.fail_warned:
@@ -323,7 +325,7 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False):
         else:
             for test_name in self.lint_tests:
                 if test_name == "main_nf":
-                    getattr(self, test_name)(mod, fix_version, progress_bar)
+                    getattr(self, test_name)(mod, fix_version, registry, progress_bar)
                 else:
                     getattr(self, test_name)(mod)
 

nf_core/modules/lint/main_nf.py

@@ -17,7 +17,7 @@
 log = logging.getLogger(__name__)
 
 
-def main_nf(module_lint_object, module, fix_version, progress_bar):
+def main_nf(module_lint_object, module, fix_version, registry, progress_bar):
     """
     Lint a ``main.nf`` module file
 
@@ -121,7 +121,7 @@ def main_nf(module_lint_object, module, fix_version, progress_bar):
         module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf))
 
     # Check the process definitions
-    if check_process_section(module, process_lines, fix_version, progress_bar):
+    if check_process_section(module, process_lines, registry, fix_version, progress_bar):
         module.passed.append(("main_nf_container", "Container versions match", module.main_nf))
     else:
         module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf))
@@ -209,12 +209,20 @@ def check_when_section(self, lines):
     self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf))
 
 
-def check_process_section(self, lines, fix_version, progress_bar):
-    """
-    Lint the section of a module between the process definition
+def check_process_section(self, lines, registry, fix_version, progress_bar):
+    """Lint the section of a module between the process definition
     and the 'input:' definition
     Specifically checks for correct software versions
     and containers
+
+    Args:
+        lines (List[str]): Content of process.
+        registry (str): Base Docker registry for containers. Typically quay.io.
+        fix_version (bool): Fix software version
+        progress_bar (ProgressBar): Progress bar to update.
+
+    Returns:
+        Optional[bool]: True if singularity and docker containers match, False otherwise. If process definition does not exist, None.
     """
     # Check that we have a process section
     if len(lines) == 0:
@@ -288,7 +296,7 @@ def check_process_section(self, lines, fix_version, progress_bar):
             else:
                 self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf))
                 docker_tag = None
-            if l.startswith("quay.io/"):
+            if l.startswith(registry):
                 l_stripped = re.sub(r"\W+$", "", l)
                 self.failed.append(
                     (
@@ -303,7 +311,7 @@ def check_process_section(self, lines, fix_version, progress_bar):
             # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04)
             # If so, add quay.io as default container prefix
             if l.count("/") == 1 and l.count(":") == 1:
-                l = "quay.io/" + l
+                l = "/".join([registry, l]).replace("//", "/")
             url = urlparse(l.split("'")[0])
 
         # lint double quotes
@@ -326,7 +334,7 @@ def check_process_section(self, lines, fix_version, progress_bar):
                 )
 
         # lint more than one container in the same line
-        if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l):
+        if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or l.startswith(registry)):
             self.warned.append(
                 (
                     "container_links",

tests/modules/lint.py

@@ -82,6 +82,16 @@ def test_modules_lint_multiple_remotes(self):
     assert len(module_lint.warned) >= 0
 
 
+def test_modules_lint_registry(self):
+    """Test linting the samtools module and alternative registry"""
+    self.mods_install.install("samtools")
+    module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir)
+    module_lint.lint(print_results=False, registry="public.ecr.aws", module="samtools")
+    assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}"
+    assert len(module_lint.passed) > 0
+    assert len(module_lint.warned) >= 0
+
+
 def test_modules_lint_patched_modules(self):
     """
     Test creating a patch file and applying it to a new version of the the files