Merge pull request #2318 from nf-core/2311-interpret-docker.registry-…

…automatically

2311 interpret docker.registry automatically

CHANGELOG.md

@@ -18,6 +18,7 @@
 - Add ability to specify custom registry for linting modules, defaults to quay.io ([#2313](https://github.com/nf-core/tools/pull/2313))
 - Add `singularity.registry = 'quay.io'` in pipeline template
 - Bump minimum required NF version in pipeline template from `22.10.1` -> `23.04.0`
+- Add ability to interpret `docker.registry` from `nextflow.config` file. If not found defaults to quay.io. ([#2318](https://github.com/nf-core/tools/pull/2318))
 
 ### Download
 

nf_core/__main__.py

@@ -808,7 +808,12 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
 @click.argument("tool", type=str, required=False, metavar="<tool> or <tool/subtool>")
 @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="<pipeline/modules directory>")
 @click.option(
- "-r", "--registry", type=str, metavar="<registry>", default="quay.io", help="Registry to use for containers"
+ "-r",
+ "--registry",
+ type=str,
+ metavar="<registry>",
+ default=None,
+ help="Registry to use for containers. If not specified it will use docker.registry value in the nextflow.config file",
 )
 @click.option("-k", "--key", type=str, metavar="<test>", multiple=True, help="Run only these lint tests")
 @click.option("-a", "--all", is_flag=True, help="Run on all modules")
@@ -842,6 +847,7 @@ def lint(
  module_lint = ModuleLint(
  dir,
  fail_warned=fail_warned,
+ registry=ctx.params["registry"],
  remote_url=ctx.obj["modules_repo_url"],
  branch=ctx.obj["modules_repo_branch"],
  no_pull=ctx.obj["modules_repo_no_pull"],

nf_core/modules/lint/__init__.py

@@ -69,6 +69,7 @@ def __init__(
  remote_url=None,
  branch=None,
  no_pull=False,
+ registry=None,
  hide_progress=False,
  ):
  super().__init__(
@@ -114,6 +115,7 @@ def __init__(
  )
  for m in self.get_local_components()
  ]
+ self.config = nf_core.utils.fetch_wf_config(self.dir, cache_config=True)
  else:
  module_dir = Path(self.dir, self.default_modules_path)
  self.all_remote_modules = [
@@ -124,6 +126,15 @@ def __init__(
  if not self.all_remote_modules:
  raise LookupError("No modules in 'modules' directory")
 
+ # This could be better, perhaps glob for all nextflow.config files in?
+ self.config = nf_core.utils.fetch_wf_config(Path(self.dir).joinpath("tests", "config"), cache_config=True)
+
+ if registry is None:
+ self.registry = self.config.get("docker.registry", "quay.io")
+ else:
+ self.registry = registry
+ log.debug(f"Registry set to {self.registry}")
+
  self.lint_config = None
  self.modules_json = None
 
@@ -313,7 +324,7 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals
 
  # Only check the main script in case of a local module
  if local:
- self.main_nf(mod, fix_version, registry, progress_bar)
+ self.main_nf(mod, fix_version, self.registry, progress_bar)
  self.passed += [LintResult(mod, *m) for m in mod.passed]
  warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)]
  if not self.fail_warned:
@@ -325,7 +336,7 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals
  else:
  for test_name in self.lint_tests:
  if test_name == "main_nf":
- getattr(self, test_name)(mod, fix_version, registry, progress_bar)
+ getattr(self, test_name)(mod, fix_version, self.registry, progress_bar)
  else:
  getattr(self, test_name)(mod)
 

nf_core/utils.py

@@ -258,7 +258,7 @@ def fetch_wf_config(wf_path, cache_config=True):
 
  if cache_basedir and cache_fn:
  cache_path = os.path.join(cache_basedir, cache_fn)
- if os.path.isfile(cache_path):
+ if os.path.isfile(cache_path) and cache_config is True:
  log.debug(f"Found a config cache, loading: {cache_path}")
  with open(cache_path, "r") as fh:
  try:
@@ -274,7 +274,7 @@ def fetch_wf_config(wf_path, cache_config=True):
  ul = l.decode("utf-8")
  try:
  k, v = ul.split(" = ", 1)
- config[k] = v
+ config[k] = v.strip("'\"")
  except ValueError:
  log.debug(f"Couldn't find key=value config pair:\n {ul}")
 

tests/modules/lint.py

@@ -85,12 +85,13 @@ def test_modules_lint_multiple_remotes(self):
 def test_modules_lint_registry(self):
  """Test linting the samtools module and alternative registry"""
  self.mods_install.install("samtools")
- module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir)
- module_lint.lint(print_results=False, registry="public.ecr.aws", module="samtools")
+ module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, registry="public.ecr.aws")
+ module_lint.lint(print_results=False, module="samtools")
  assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}"
  assert len(module_lint.passed) > 0
  assert len(module_lint.warned) >= 0
- module_lint.lint(print_results=False, registry="quay.io", module="samtools")
+ module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir)
+ module_lint.lint(print_results=False, module="samtools")
  assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}"
  assert len(module_lint.passed) > 0
  assert len(module_lint.warned) >= 0

tests/test_download.py

@@ -128,7 +128,7 @@ def test_wf_use_local_configs(self, tmp_path):
  # Test the function
  download_obj.wf_use_local_configs("workflow")
  wf_config = nf_core.utils.fetch_wf_config(os.path.join(test_outdir, "workflow"), cache_config=False)
- assert wf_config["params.custom_config_base"] == f"'{test_outdir}/workflow/../configs/'"
+ assert wf_config["params.custom_config_base"] == f"{test_outdir}/workflow/../configs/"
 
  #
  # Tests for 'find_container_images'