Refactor nf-core modules commands

CHANGELOG.md

@@ -19,6 +19,7 @@
 * Changed working directory to temporary directory for `nf-core modules create-test-yml` [[#908](https://github.com/nf-core/tools/issues/908)]
 * Use Biocontainers API instead of quayi.io API for `nf-core modules create` [[#875](https://github.com/nf-core/tools/issues/875)]
 * Update `nf-core modules install` to handle different versions of modules  [#1116](https://github.com/nf-core/tools/pull/1116)
+* Refactored `nf-core modules` command into one file per command [#1124](https://github.com/nf-core/tools/pull/1124)
 
 #### Sync
 

nf_core/__main__.py

@@ -370,10 +370,9 @@ def list(ctx, pipeline_dir, json):
     If no pipeline directory is given, lists all currently available
     software wrappers in the nf-core/modules repository.
     """
-    mods = nf_core.modules.PipelineModules()
-    mods.modules_repo = ctx.obj["modules_repo_obj"]
-    mods.pipeline_dir = pipeline_dir
-    print(mods.list_modules(json))
+    module_list = nf_core.modules.ModuleList(pipeline_dir)
+    module_list.modules_repo = ctx.obj["modules_repo_obj"]
+    print(module_list.list_modules(json))
 
 
 @modules.command(help_priority=2)
@@ -393,13 +392,9 @@ def install(ctx, pipeline_dir, tool, latest, force, sha):
     along with associated metadata.
     """
     try:
-        mods = nf_core.modules.PipelineModules()
-        mods.modules_repo = ctx.obj["modules_repo_obj"]
-        mods.pipeline_dir = pipeline_dir
-        mods.force = force
-        mods.latest = latest
-        mods.sha = sha
-        mods.install(tool)
+        module_install = nf_core.modules.ModuleInstall(pipeline_dir, force=force, latest=latest, sha=sha)
+        module_install.modules_repo = ctx.obj["modules_repo_obj"]
+        module_install.install(tool)
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)
@@ -435,10 +430,9 @@ def remove(ctx, pipeline_dir, tool):
     Remove a software wrapper from a pipeline.
     """
     try:
-        mods = nf_core.modules.PipelineModules()
-        mods.modules_repo = ctx.obj["modules_repo_obj"]
-        mods.pipeline_dir = pipeline_dir
-        mods.remove(tool)
+        module_remove = nf_core.modules.ModuleRemove(pipeline_dir)
+        module_remove.modules_repo = ctx.obj["modules_repo_obj"]
+        module_remove.remove(tool)
     except UserWarning as e:
         log.critical(e)
         sys.exit(1)

nf_core/modules/__init__.py

@@ -1,4 +1,7 @@
-from .pipeline_modules import ModulesRepo, PipelineModules
+from .modules_repo import ModulesRepo
 from .create import ModuleCreate
 from .test_yml_builder import ModulesTestYmlBuilder
 from .lint import ModuleLint
+from .list import ModuleList
+from .install import ModuleInstall
+from .remove import ModuleRemove

nf_core/modules/install.py

@@ -0,0 +1,173 @@
+import os
+import sys
+import json
+import questionary
+import logging
+
+import nf_core.utils
+
+from .modules_command import ModuleCommand
+from .module_utils import get_module_git_log
+
+log = logging.getLogger(__name__)
+
+
+class ModuleInstall(ModuleCommand):
+    def __init__(self, pipeline_dir, module=None, force=False, latest=False, sha=None):
+        super().__init__(pipeline_dir)
+        self.force = force
+        self.latest = latest
+        self.sha = sha
+        module = module
+
+    def install(self, module):
+        if self.repo_type == "modules":
+            log.error("You cannot install a module in a clone of nf-core/modules")
+            return False
+        # Check whether pipelines is valid
+        self.has_valid_directory()
+
+        # Get the available modules
+        self.modules_repo.get_modules_file_tree()
+        if self.latest and self.sha is not None:
+            log.error("Cannot use '--sha' and '--latest' at the same time!")
+            return False
+
+        if module is None:
+            module = questionary.autocomplete(
+                "Tool name:",
+                choices=self.modules_repo.modules_avail_module_names,
+                style=nf_core.utils.nfcore_question_style,
+            ).unsafe_ask()
+
+        # Check that the supplied name is an available module
+        if module not in self.modules_repo.modules_avail_module_names:
+            log.error("Module '{}' not found in list of available modules.".format(module))
+            log.info("Use the command 'nf-core modules list' to view available software")
+            return False
+        # Set the install folder based on the repository name
+        install_folder = ["nf-core", "software"]
+        if not self.modules_repo.name == "nf-core/modules":
+            install_folder = ["external"]
+
+        # Compute the module directory
+        module_dir = os.path.join(self.dir, "modules", *install_folder, module)
+
+        # Load 'modules.json'
+        modules_json_path = os.path.join(self.dir, "modules.json")
+        with open(modules_json_path, "r") as fh:
+            modules_json = json.load(fh)
+
+        current_entry = modules_json["modules"].get(module)
+
+        if current_entry is not None and self.sha is None:
+            # Fetch the latest commit for the module
+            current_version = current_entry["git_sha"]
+            git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+            if len(git_log) == 0:
+                log.error(f"Was unable to fetch version of module '{module}'")
+                return False
+            latest_version = git_log[0]["git_sha"]
+            if current_version == latest_version and not self.force:
+                log.info("Already up to date")
+                return True
+            elif not self.force:
+                log.error("Found newer version of module.")
+                self.latest = self.force = questionary.confirm(
+                    "Do you want install it? (--force --latest)", default=False
+                ).unsafe_ask()
+                if not self.latest:
+                    return False
+        else:
+            latest_version = None
+
+        # Check that we don't already have a folder for this module
+        if not self.check_module_files_installed(module, module_dir):
+            return False
+
+        if self.sha:
+            if not current_entry is None and not self.force:
+                return False
+            if self.download_module_file(module, self.sha, install_folder, module_dir):
+                self.update_modules_json(modules_json, modules_json_path, module, self.sha)
+                return True
+            else:
+                try:
+                    version = self.prompt_module_version_sha(
+                        installed_sha=current_entry["git_sha"] if not current_entry is None else None
+                    )
+                except SystemError as e:
+                    log.error(e)
+                    return False
+        else:
+            if self.latest:
+                # Fetch the latest commit for the module
+                if latest_version is None:
+                    git_log = get_module_git_log(module, per_page=1, page_nbr=1)
+                    if len(git_log) == 0:
+                        log.error(f"Was unable to fetch version of module '{module}'")
+                        return False
+                    latest_version = git_log[0]["git_sha"]
+                version = latest_version
+            else:
+                try:
+                    version = self.prompt_module_version_sha(
+                        installed_sha=current_entry["git_sha"] if not current_entry is None else None
+                    )
+                except SystemError as e:
+                    log.error(e)
+                    return False
+
+        log.info("Installing {}".format(module))
+        log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash))
+
+        # Download module files
+        if not self.download_module_file(module, version, install_folder, module_dir):
+            return False
+
+        # Update module.json with newly installed module
+        self.update_modules_json(modules_json, modules_json_path, module, version)
+        return True
+
+    def check_module_files_installed(self, module_name, module_dir):
+        """Checks if a module is already installed"""
+        if os.path.exists(module_dir):
+            if not self.force:
+                log.error(f"Module directory '{module_dir}' already exists.")
+                self.force = questionary.confirm(
+                    "Do you want to overwrite local files? (--force)", default=False
+                ).unsafe_ask()
+            if self.force:
+                log.info(f"Removing old version of module '{module_name}'")
+                return self.clear_module_dir(module_name, module_dir)
+            else:
+                return False
+        else:
+            return True
+
+    def prompt_module_version_sha(self, module, installed_sha=None):
+        older_commits_choice = questionary.Choice(
+            title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value=""
+        )
+        git_sha = ""
+        page_nbr = 1
+        next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr)
+        while git_sha is "":
+            commits = next_page_commits
+            next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1)
+            choices = []
+            for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits):
+
+                display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired"
+                message = f"{title} {sha}"
+                if installed_sha == sha:
+                    message += " (installed version)"
+                commit_display = [(display_color, message), ("class:choice-default", "")]
+                choices.append(questionary.Choice(title=commit_display, value=sha))
+            if len(next_page_commits) > 0:
+                choices += [older_commits_choice]
+            git_sha = questionary.select(
+                f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style
+            ).unsafe_ask()
+            page_nbr += 1
+        return git_sha

nf_core/modules/lint/__init__.py

@@ -25,9 +25,11 @@
 import sys
 
 import nf_core.utils
-from nf_core.modules.pipeline_modules import ModulesRepo
+import nf_core.modules.module_utils
+from nf_core.modules.modules_repo import ModulesRepo
 from nf_core.modules.nfcore_module import NFCoreModule
 
+
 log = logging.getLogger(__name__)
 
 
@@ -64,7 +66,7 @@ class ModuleLint(object):
 
     def __init__(self, dir, key=()):
         self.dir = dir
-        self.repo_type = self.get_repo_type()
+        self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
         self.passed = []
         self.warned = []
         self.failed = []
@@ -227,25 +229,6 @@ def lint_nfcore_modules(self, nfcore_modules):
                 progress_bar.update(lint_progress, advance=1, test_name=mod.module_name)
                 self.lint_module(mod)
 
-    def get_repo_type(self):
-        """
-        Determine whether this is a pipeline repository or a clone of
-        nf-core/modules
-        """
-        # Verify that the pipeline dir exists
-        if self.dir is None or not os.path.exists(self.dir):
-            log.error("Could not find directory: {}".format(self.dir))
-            sys.exit(1)
-
-        # Determine repository type
-        if os.path.exists(os.path.join(self.dir, "main.nf")):
-            return "pipeline"
-        elif os.path.exists(os.path.join(self.dir, "software")):
-            return "modules"
-        else:
-            log.error("Could not determine repository type of {}".format(self.dir))
-            sys.exit(1)
-
     def get_installed_modules(self):
         """
         Make a list of all modules installed in this repository

nf_core/modules/list.py

@@ -0,0 +1,60 @@
+import json
+from os import pipe
+import rich
+import logging
+
+from .modules_command import ModuleCommand
+
+log = logging.getLogger(__name__)
+
+
+class ModuleList(ModuleCommand):
+    def __init__(self, pipeline_dir):
+        super().__init__(pipeline_dir)
+
+    def list_modules(self, print_json=False):
+        """
+        Get available module names from GitHub tree for repo
+        and print as list to stdout
+        """
+
+        # Initialise rich table
+        table = rich.table.Table()
+        table.add_column("Module Name")
+        modules = []
+
+        # No pipeline given - show all remote
+        if self.dir is None:
+            log.info(f"Modules available from {self.modules_repo.name} ({self.modules_repo.branch}):\n")
+
+            # Get the list of available modules
+            self.modules_repo.get_modules_file_tree()
+            modules = self.modules_repo.modules_avail_module_names
+            # Nothing found
+            if len(modules) == 0:
+                log.info(f"No available modules found in {self.modules_repo.name} ({self.modules_repo.branch})")
+                return ""
+
+        # We have a pipeline - list what's installed
+        else:
+            log.info(f"Modules installed in '{self.dir}':\n")
+
+            # Check whether pipelines is valid
+            try:
+                self.has_valid_directory()
+            except UserWarning as e:
+                log.error(e)
+                return ""
+            # Get installed modules
+            self.get_pipeline_modules()
+            modules = self.module_names
+            # Nothing found
+            if len(modules) == 0:
+                log.info(f"No nf-core modules found in '{self.dir}'")
+                return ""
+
+        for mod in sorted(modules):
+            table.add_row(mod)
+        if print_json:
+            return json.dumps(modules, sort_keys=True, indent=4)
+        return table

nf_core/modules/module_utils.py

@@ -158,29 +158,21 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_
     return all(files_are_equal)
 
 
-def prompt_module_version_sha(module, installed_sha=None):
-    older_commits_choice = questionary.Choice(
-        title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value=""
-    )
-    git_sha = ""
-    page_nbr = 1
-    next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr)
-    while git_sha is "":
-        commits = next_page_commits
-        next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1)
-        choices = []
-        for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits):
-
-            display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired"
-            message = f"{title} {sha}"
-            if installed_sha == sha:
-                message += " (installed version)"
-            commit_display = [(display_color, message), ("class:choice-default", "")]
-            choices.append(questionary.Choice(title=commit_display, value=sha))
-        if len(next_page_commits) > 0:
-            choices += [older_commits_choice]
-        git_sha = questionary.select(
-            f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style
-        ).unsafe_ask()
-        page_nbr += 1
-    return git_sha
+def get_repo_type(dir):
+    """
+    Determine whether this is a pipeline repository or a clone of
+    nf-core/modules
+    """
+    # Verify that the pipeline dir exists
+    if dir is None or not os.path.exists(dir):
+        log.error("Could not find directory: {}".format(dir))
+        sys.exit(1)
+
+    # Determine repository type
+    if os.path.exists(os.path.join(dir, "main.nf")):
+        return "pipeline"
+    elif os.path.exists(os.path.join(dir, "software")):
+        return "modules"
+    else:
+        log.error("Could not determine repository type of {}".format(dir))
+        sys.exit(1)