Merge pull request #1655 from ErikDanielsson/custom-file-path

Add `--base-path` option to modules commands (and refactor `modules.json` code)

CHANGELOG.md

@@ -31,6 +31,7 @@
 - Update how we interface with git remotes. ([#1626](https://github.com/nf-core/tools/issues/1626))
 - Add prompt for module name to `nf-core modules info` ([#1644](https://github.com/nf-core/tools/issues/1644))
 - Update docs with example of custom git remote ([#1645](https://github.com/nf-core/tools/issues/1645))
+- Add `--base-path` flag to `nf-core modules` to specify the base path for the modules in a remote. Also refactored `modules.json` code. ([#1643](https://github.com/nf-core/tools/issues/1643))
 
 ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16]
 

README.md

@@ -920,6 +920,8 @@ For example, if you want to install the `fastqc` module from the repository `nf-
 nf-core modules --git-remote git@gitlab.com:nf-core/modules-test.git install fastqc
 ```
 
+If the modules in your custom remote are stored in another directory than `modules`, you can specify the path by using the `--base-path <path>` flag. This will default to `modules`.
+
 Note that a custom remote must follow a similar directory structure to that of `nf-core/moduleś` for the `nf-core modules` commands to work properly.
 
 The modules commands will during initalisation try to pull changes from the remote repositories. If you want to disable this, for example

nf_core/__main__.py

@@ -4,6 +4,7 @@
 import os
 import re
 import sys
+from email.policy import default
 
 import rich.console
 import rich.logging
@@ -357,8 +358,14 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, fail_warned, markdow
     default=False,
     help="Do not pull in latest changes to local clone of modules repository.",
 )
+@click.option(
+    "--base-path",
+    type=str,
+    default=None,
+    help="Specify where the modules are stored in the remote",
+)
 @click.pass_context
-def modules(ctx, git_remote, branch, no_pull):
+def modules(ctx, git_remote, branch, no_pull, base_path):
     """
     Commands to manage Nextflow DSL2 modules (tool wrappers).
     """
@@ -370,6 +377,7 @@ def modules(ctx, git_remote, branch, no_pull):
     ctx.obj["modules_repo_url"] = git_remote
     ctx.obj["modules_repo_branch"] = branch
     ctx.obj["modules_repo_no_pull"] = no_pull
+    ctx.obj["modules_repo_base_path"] = base_path
 
 
 # nf-core modules list subcommands
@@ -393,7 +401,12 @@ def remote(ctx, keywords, json):
     """
     try:
         module_list = nf_core.modules.ModuleList(
-            None, True, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+            None,
+            True,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+            ctx.obj["modules_repo_base_path"],
         )
         print(module_list.list_modules(keywords, json))
     except (UserWarning, LookupError) as e:
@@ -419,7 +432,12 @@ def local(ctx, keywords, json, dir):
     """
     try:
         module_list = nf_core.modules.ModuleList(
-            dir, False, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+            dir,
+            False,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+            ctx.obj["modules_repo_base_path"],
         )
         print(module_list.list_modules(keywords, json))
     except (UserWarning, LookupError) as e:
@@ -456,6 +474,7 @@ def install(ctx, tool, dir, prompt, force, sha):
             ctx.obj["modules_repo_url"],
             ctx.obj["modules_repo_branch"],
             ctx.obj["modules_repo_no_pull"],
+            ctx.obj["modules_repo_base_path"],
         )
         exit_status = module_install.install(tool)
         if not exit_status and all:
@@ -513,6 +532,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff):
             ctx.obj["modules_repo_url"],
             ctx.obj["modules_repo_branch"],
             ctx.obj["modules_repo_no_pull"],
+            ctx.obj["modules_repo_base_path"],
         )
         exit_status = module_install.update(tool)
         if not exit_status and all:
@@ -539,7 +559,11 @@ def remove(ctx, dir, tool):
     """
     try:
         module_remove = nf_core.modules.ModuleRemove(
-            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+            dir,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+            ctx.obj["modules_repo_base_path"],
         )
         module_remove.remove(tool)
     except (UserWarning, LookupError) as e:
@@ -637,7 +661,11 @@ def lint(ctx, tool, dir, key, all, local, passed, fix_version):
     """
     try:
         module_lint = nf_core.modules.ModuleLint(
-            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+            dir,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+            ctx.obj["modules_repo_base_path"],
         )
         module_lint.lint(
             module=tool,
@@ -683,7 +711,12 @@ def info(ctx, tool, dir):
     """
     try:
         module_info = nf_core.modules.ModuleInfo(
-            dir, tool, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+            dir,
+            tool,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+            ctx.obj["modules_repo_base_path"],
         )
         print(module_info.get_module_info())
     except (UserWarning, LookupError) as e:
@@ -705,7 +738,11 @@ def bump_versions(ctx, tool, dir, all, show_all):
     """
     try:
         version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(
-            dir, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"]
+            dir,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+            ctx.obj["modules_repo_base_path"],
         )
         version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all)
     except nf_core.modules.module_utils.ModuleException as e:

nf_core/lint/modules_json.py

@@ -3,6 +3,7 @@
 from logging import warn
 
 from nf_core.modules.modules_command import ModuleCommand
+from nf_core.modules.modules_json import ModulesJson
 
 
 def modules_json(self):
@@ -19,22 +20,24 @@ def modules_json(self):
 
     # Load pipeline modules and modules.json
     modules_command = ModuleCommand(self.wf_path)
-    modules_json = modules_command.load_modules_json()
+    modules_json = ModulesJson(self.wf_path)
+    modules_json.load_modules_json()
+    modules_json_dict = modules_json.modules_json
 
     if modules_json:
         modules_command.get_pipeline_modules()
 
         all_modules_passed = True
 
-        for repo in modules_json["repos"].keys():
+        for repo in modules_json_dict["repos"].keys():
             # Check if the modules.json has been updated to keep the
-            if "modules" not in modules_json["repos"][repo] or "git_url" not in modules_json["repos"][repo]:
+            if "modules" not in modules_json_dict["repos"][repo] or "git_url" not in modules_json_dict["repos"][repo]:
                 failed.append(
                     f"Your `modules.json` file is outdated. Please remove it and reinstall it by running any module command"
                 )
                 continue
 
-            for key in modules_json["repos"][repo]["modules"]:
+            for key in modules_json_dict["repos"][repo]["modules"]:
                 if not key in modules_command.module_names[repo]:
                     failed.append(f"Entry for `{key}` found in `modules.json` but module is not installed in pipeline.")
                     all_modules_passed = False

nf_core/modules/__init__.py

@@ -6,6 +6,7 @@
 from .list import ModuleList
 from .module_test import ModulesTest
 from .module_utils import ModuleException
+from .modules_json import ModulesJson
 from .modules_repo import ModulesRepo
 from .mulled import MulledImageNameGenerator
 from .remove import ModuleRemove

nf_core/modules/bump_versions.py

@@ -28,8 +28,8 @@
 
 
 class ModuleVersionBumper(ModuleCommand):
-    def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False):
-        super().__init__(pipeline_dir, remote_url, branch, no_pull)
+    def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False, base_path=None):
+        super().__init__(pipeline_dir, remote_url, branch, no_pull, base_path)
 
         self.up_to_date = None
         self.updated = None

nf_core/modules/install.py

@@ -5,6 +5,7 @@
 
 import nf_core.modules.module_utils
 import nf_core.utils
+from nf_core.modules.modules_json import ModulesJson
 
 from .modules_command import ModuleCommand
 from .modules_repo import NF_CORE_MODULES_NAME
@@ -13,8 +14,28 @@
 
 
 class ModuleInstall(ModuleCommand):
-    def __init__(self, pipeline_dir, force=False, prompt=False, sha=None, remote_url=None, branch=None, no_pull=False):
-        super().__init__(pipeline_dir, remote_url, branch, no_pull)
+    def __init__(
+        self,
+        pipeline_dir,
+        force=False,
+        prompt=False,
+        sha=None,
+        remote_url=None,
+        branch=None,
+        no_pull=False,
+        base_path=None,
+    ):
+        # Check if we are given a base path, otherwise look in the modules.json
+        if base_path is None:
+            try:
+                modules_json = ModulesJson(pipeline_dir)
+                repo_name = nf_core.modules.module_utils.path_from_remote(remote_url)
+                base_path = modules_json.get_base_path(repo_name)
+            except:
+                # We don't want to fail yet if the modules.json is not found
+                pass
+
+        super().__init__(pipeline_dir, remote_url, branch, no_pull, base_path)
         self.force = force
         self.prompt = prompt
         self.sha = sha
@@ -28,7 +49,8 @@ def install(self, module):
             return False
 
         # Verify that 'modules.json' is consistent with the installed modules
-        self.modules_json_up_to_date()
+        modules_json = ModulesJson(self.dir)
+        modules_json.modules_json_up_to_date()
 
         if self.prompt and self.sha is not None:
             log.error("Cannot use '--sha' and '--prompt' at the same time!")
@@ -53,22 +75,14 @@ def install(self, module):
             log.info("Use the command 'nf-core modules list' to view available software")
             return False
 
-        # Load 'modules.json'
-        modules_json = self.load_modules_json()
-        if not modules_json:
-            return False
-
         if not self.modules_repo.module_exists(module):
             warn_msg = (
-                f"Module '{module}' not found in remote '{self.modules_repo.fullname}' ({self.modules_repo.branch})"
+                f"Module '{module}' not found in remote '{self.modules_repo.remote_url}' ({self.modules_repo.branch})"
             )
             log.warning(warn_msg)
             return False
 
-        if self.modules_repo.fullname in modules_json["repos"]:
-            current_entry = modules_json["repos"][self.modules_repo.fullname]["modules"].get(module)
-        else:
-            current_entry = None
+        current_version = modules_json.get_module_version(module, self.modules_repo.fullname)
 
         # Set the install folder based on the repository name
         install_folder = [self.dir, "modules"]
@@ -78,11 +92,11 @@ def install(self, module):
         module_dir = os.path.join(*install_folder, module)
 
         # Check that the module is not already installed
-        if (current_entry is not None and os.path.exists(module_dir)) and not self.force:
+        if (current_version is not None and os.path.exists(module_dir)) and not self.force:
 
             log.error("Module is already installed.")
             repo_flag = (
-                "" if self.modules_repo.fullname == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.fullname} "
+                "" if self.modules_repo.fullname == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} "
             )
             branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} "
 
@@ -97,7 +111,7 @@ def install(self, module):
             try:
                 version = nf_core.modules.module_utils.prompt_module_version_sha(
                     module,
-                    installed_sha=current_entry["git_sha"] if not current_entry is None else None,
+                    installed_sha=current_version,
                     modules_repo=self.modules_repo,
                 )
             except SystemError as e:
@@ -113,7 +127,7 @@ def install(self, module):
             self.clear_module_dir(module, module_dir)
 
         log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'")
-        log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.fullname}")
+        log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}")
 
         # Download module files
         if not self.install_module_files(module, version, self.modules_repo, install_folder):
@@ -124,5 +138,5 @@ def install(self, module):
         log.info(f"Include statement: include {{ {module_name} }} from '.{os.path.join(*install_folder, module)}/main'")
 
         # Update module.json with newly installed module
-        self.update_modules_json(modules_json, self.modules_repo, module, version)
+        modules_json.update_modules_json(self.modules_repo, module, version)
         return True

nf_core/modules/lint/__init__.py

@@ -29,6 +29,7 @@
 from nf_core.lint.pipeline_todos import pipeline_todos
 from nf_core.lint_utils import console
 from nf_core.modules.modules_command import ModuleCommand
+from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
 from nf_core.modules.nfcore_module import NFCoreModule
 from nf_core.utils import plural_s as _s
@@ -69,7 +70,7 @@ class ModuleLint(ModuleCommand):
     from .module_todos import module_todos
     from .module_version import module_version
 
-    def __init__(self, dir, remote_url=None, branch=None, no_pull=False):
+    def __init__(self, dir, remote_url=None, branch=None, no_pull=False, base_path=None):
         self.dir = dir
         try:
             self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
@@ -79,7 +80,7 @@ def __init__(self, dir, remote_url=None, branch=None, no_pull=False):
         self.passed = []
         self.warned = []
         self.failed = []
-        self.modules_repo = ModulesRepo(remote_url, branch, no_pull)
+        self.modules_repo = ModulesRepo(remote_url, branch, no_pull, base_path)
         self.lint_tests = self._get_all_lint_tests()
         # Get lists of modules install in directory
         self.all_local_modules, self.all_nfcore_modules = self.get_installed_modules()
@@ -196,7 +197,8 @@ def lint(
 
     def set_up_pipeline_files(self):
         self.load_lint_config()
-        self.modules_json = self.load_modules_json()
+        self.modules_json = ModulesJson(self.dir)
+        self.modules_json.load_modules_json()
 
         # Only continue if a lint config has been loaded
         if self.lint_config:

nf_core/modules/lint/module_version.py

@@ -30,24 +30,23 @@ def module_version(module_lint_object, module):
     modules_json_path = os.path.join(module_lint_object.dir, "modules.json")
 
     # Verify that a git_sha exists in the `modules.json` file for this module
-    try:
-        module_entry = module_lint_object.modules_json["repos"][module_lint_object.modules_repo.fullname]["modules"][
-            module.module_name
-        ]
-        git_sha = module_entry["git_sha"]
-        module.git_sha = git_sha
-        module.passed.append(("git_sha", "Found git_sha entry in `modules.json`", modules_json_path))
-
-        # Check whether a new version is available
-        try:
-            modules_repo = nf_core.modules.modules_repo.ModulesRepo()
-            module_git_log = modules_repo.get_module_git_log(module.module_name)
-            if git_sha == next(module_git_log)["git_sha"]:
-                module.passed.append(("module_version", "Module is the latest version", module.module_dir))
-            else:
-                module.warned.append(("module_version", "New version available", module.module_dir))
-        except UserWarning:
-            module.warned.append(("module_version", "Failed to fetch git log", module.module_dir))
-
-    except KeyError:
+    module_version = module_lint_object.modules_json.get_module_version(
+        module.module_name, module_lint_object.modules_repo.fullname
+    )
+    if module_version is None:
         module.failed.append(("git_sha", "No git_sha entry in `modules.json`", modules_json_path))
+        return
+
+    module.git_sha = module_version
+    module.passed.append(("git_sha", "Found git_sha entry in `modules.json`", modules_json_path))
+
+    # Check whether a new version is available
+    try:
+        modules_repo = nf_core.modules.modules_repo.ModulesRepo()
+        module_git_log = modules_repo.get_module_git_log(module.module_name)
+        if module_version == next(module_git_log)["git_sha"]:
+            module.passed.append(("module_version", "Module is the latest version", module.module_dir))
+        else:
+            module.warned.append(("module_version", "New version available", module.module_dir))
+    except UserWarning:
+        module.warned.append(("module_version", "Failed to fetch git log", module.module_dir))