diff --git a/CHANGELOG.md b/CHANGELOG.md index 62c2548eae..2317c19e12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ * Changed working directory to temporary directory for `nf-core modules create-test-yml` [[#908](https://github.com/nf-core/tools/issues/908)] * Use Biocontainers API instead of quayi.io API for `nf-core modules create` [[#875](https://github.com/nf-core/tools/issues/875)] * Update `nf-core modules install` to handle different versions of modules [#1116](https://github.com/nf-core/tools/pull/1116) +* Refactored `nf-core modules` command into one file per command [#1124](https://github.com/nf-core/tools/pull/1124) #### Sync diff --git a/nf_core/__main__.py b/nf_core/__main__.py index f32fdf4d60..cb53b7efc7 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -370,10 +370,9 @@ def list(ctx, pipeline_dir, json): If no pipeline directory is given, lists all currently available software wrappers in the nf-core/modules repository. """ - mods = nf_core.modules.PipelineModules() - mods.modules_repo = ctx.obj["modules_repo_obj"] - mods.pipeline_dir = pipeline_dir - print(mods.list_modules(json)) + module_list = nf_core.modules.ModuleList(pipeline_dir) + module_list.modules_repo = ctx.obj["modules_repo_obj"] + print(module_list.list_modules(json)) @modules.command(help_priority=2) @@ -393,13 +392,9 @@ def install(ctx, pipeline_dir, tool, latest, force, sha): along with associated metadata. """ try: - mods = nf_core.modules.PipelineModules() - mods.modules_repo = ctx.obj["modules_repo_obj"] - mods.pipeline_dir = pipeline_dir - mods.force = force - mods.latest = latest - mods.sha = sha - mods.install(tool) + module_install = nf_core.modules.ModuleInstall(pipeline_dir, force=force, latest=latest, sha=sha) + module_install.modules_repo = ctx.obj["modules_repo_obj"] + module_install.install(tool) except UserWarning as e: log.critical(e) sys.exit(1) @@ -435,10 +430,9 @@ def remove(ctx, pipeline_dir, tool): Remove a software wrapper from a pipeline. """ try: - mods = nf_core.modules.PipelineModules() - mods.modules_repo = ctx.obj["modules_repo_obj"] - mods.pipeline_dir = pipeline_dir - mods.remove(tool) + module_remove = nf_core.modules.ModuleRemove(pipeline_dir) + module_remove.modules_repo = ctx.obj["modules_repo_obj"] + module_remove.remove(tool) except UserWarning as e: log.critical(e) sys.exit(1) diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index a3e92b96cc..efd9184143 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -1,4 +1,7 @@ -from .pipeline_modules import ModulesRepo, PipelineModules +from .modules_repo import ModulesRepo from .create import ModuleCreate from .test_yml_builder import ModulesTestYmlBuilder from .lint import ModuleLint +from .list import ModuleList +from .install import ModuleInstall +from .remove import ModuleRemove diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py new file mode 100644 index 0000000000..8f1b484d59 --- /dev/null +++ b/nf_core/modules/install.py @@ -0,0 +1,173 @@ +import os +import sys +import json +import questionary +import logging + +import nf_core.utils + +from .modules_command import ModuleCommand +from .module_utils import get_module_git_log + +log = logging.getLogger(__name__) + + +class ModuleInstall(ModuleCommand): + def __init__(self, pipeline_dir, module=None, force=False, latest=False, sha=None): + super().__init__(pipeline_dir) + self.force = force + self.latest = latest + self.sha = sha + module = module + + def install(self, module): + if self.repo_type == "modules": + log.error("You cannot install a module in a clone of nf-core/modules") + return False + # Check whether pipelines is valid + self.has_valid_directory() + + # Get the available modules + self.modules_repo.get_modules_file_tree() + if self.latest and self.sha is not None: + log.error("Cannot use '--sha' and '--latest' at the same time!") + return False + + if module is None: + module = questionary.autocomplete( + "Tool name:", + choices=self.modules_repo.modules_avail_module_names, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + # Check that the supplied name is an available module + if module not in self.modules_repo.modules_avail_module_names: + log.error("Module '{}' not found in list of available modules.".format(module)) + log.info("Use the command 'nf-core modules list' to view available software") + return False + # Set the install folder based on the repository name + install_folder = ["nf-core", "software"] + if not self.modules_repo.name == "nf-core/modules": + install_folder = ["external"] + + # Compute the module directory + module_dir = os.path.join(self.dir, "modules", *install_folder, module) + + # Load 'modules.json' + modules_json_path = os.path.join(self.dir, "modules.json") + with open(modules_json_path, "r") as fh: + modules_json = json.load(fh) + + current_entry = modules_json["modules"].get(module) + + if current_entry is not None and self.sha is None: + # Fetch the latest commit for the module + current_version = current_entry["git_sha"] + git_log = get_module_git_log(module, per_page=1, page_nbr=1) + if len(git_log) == 0: + log.error(f"Was unable to fetch version of module '{module}'") + return False + latest_version = git_log[0]["git_sha"] + if current_version == latest_version and not self.force: + log.info("Already up to date") + return True + elif not self.force: + log.error("Found newer version of module.") + self.latest = self.force = questionary.confirm( + "Do you want install it? (--force --latest)", default=False + ).unsafe_ask() + if not self.latest: + return False + else: + latest_version = None + + # Check that we don't already have a folder for this module + if not self.check_module_files_installed(module, module_dir): + return False + + if self.sha: + if not current_entry is None and not self.force: + return False + if self.download_module_file(module, self.sha, install_folder, module_dir): + self.update_modules_json(modules_json, modules_json_path, module, self.sha) + return True + else: + try: + version = self.prompt_module_version_sha( + installed_sha=current_entry["git_sha"] if not current_entry is None else None + ) + except SystemError as e: + log.error(e) + return False + else: + if self.latest: + # Fetch the latest commit for the module + if latest_version is None: + git_log = get_module_git_log(module, per_page=1, page_nbr=1) + if len(git_log) == 0: + log.error(f"Was unable to fetch version of module '{module}'") + return False + latest_version = git_log[0]["git_sha"] + version = latest_version + else: + try: + version = self.prompt_module_version_sha( + installed_sha=current_entry["git_sha"] if not current_entry is None else None + ) + except SystemError as e: + log.error(e) + return False + + log.info("Installing {}".format(module)) + log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash)) + + # Download module files + if not self.download_module_file(module, version, install_folder, module_dir): + return False + + # Update module.json with newly installed module + self.update_modules_json(modules_json, modules_json_path, module, version) + return True + + def check_module_files_installed(self, module_name, module_dir): + """Checks if a module is already installed""" + if os.path.exists(module_dir): + if not self.force: + log.error(f"Module directory '{module_dir}' already exists.") + self.force = questionary.confirm( + "Do you want to overwrite local files? (--force)", default=False + ).unsafe_ask() + if self.force: + log.info(f"Removing old version of module '{module_name}'") + return self.clear_module_dir(module_name, module_dir) + else: + return False + else: + return True + + def prompt_module_version_sha(self, module, installed_sha=None): + older_commits_choice = questionary.Choice( + title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value="" + ) + git_sha = "" + page_nbr = 1 + next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr) + while git_sha is "": + commits = next_page_commits + next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1) + choices = [] + for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits): + + display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired" + message = f"{title} {sha}" + if installed_sha == sha: + message += " (installed version)" + commit_display = [(display_color, message), ("class:choice-default", "")] + choices.append(questionary.Choice(title=commit_display, value=sha)) + if len(next_page_commits) > 0: + choices += [older_commits_choice] + git_sha = questionary.select( + f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + page_nbr += 1 + return git_sha diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index d7da26c12d..34b9c3fa7c 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -25,9 +25,11 @@ import sys import nf_core.utils -from nf_core.modules.pipeline_modules import ModulesRepo +import nf_core.modules.module_utils +from nf_core.modules.modules_repo import ModulesRepo from nf_core.modules.nfcore_module import NFCoreModule + log = logging.getLogger(__name__) @@ -64,7 +66,7 @@ class ModuleLint(object): def __init__(self, dir, key=()): self.dir = dir - self.repo_type = self.get_repo_type() + self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) self.passed = [] self.warned = [] self.failed = [] @@ -227,25 +229,6 @@ def lint_nfcore_modules(self, nfcore_modules): progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) self.lint_module(mod) - def get_repo_type(self): - """ - Determine whether this is a pipeline repository or a clone of - nf-core/modules - """ - # Verify that the pipeline dir exists - if self.dir is None or not os.path.exists(self.dir): - log.error("Could not find directory: {}".format(self.dir)) - sys.exit(1) - - # Determine repository type - if os.path.exists(os.path.join(self.dir, "main.nf")): - return "pipeline" - elif os.path.exists(os.path.join(self.dir, "software")): - return "modules" - else: - log.error("Could not determine repository type of {}".format(self.dir)) - sys.exit(1) - def get_installed_modules(self): """ Make a list of all modules installed in this repository diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py new file mode 100644 index 0000000000..f74ad26196 --- /dev/null +++ b/nf_core/modules/list.py @@ -0,0 +1,60 @@ +import json +from os import pipe +import rich +import logging + +from .modules_command import ModuleCommand + +log = logging.getLogger(__name__) + + +class ModuleList(ModuleCommand): + def __init__(self, pipeline_dir): + super().__init__(pipeline_dir) + + def list_modules(self, print_json=False): + """ + Get available module names from GitHub tree for repo + and print as list to stdout + """ + + # Initialise rich table + table = rich.table.Table() + table.add_column("Module Name") + modules = [] + + # No pipeline given - show all remote + if self.dir is None: + log.info(f"Modules available from {self.modules_repo.name} ({self.modules_repo.branch}):\n") + + # Get the list of available modules + self.modules_repo.get_modules_file_tree() + modules = self.modules_repo.modules_avail_module_names + # Nothing found + if len(modules) == 0: + log.info(f"No available modules found in {self.modules_repo.name} ({self.modules_repo.branch})") + return "" + + # We have a pipeline - list what's installed + else: + log.info(f"Modules installed in '{self.dir}':\n") + + # Check whether pipelines is valid + try: + self.has_valid_directory() + except UserWarning as e: + log.error(e) + return "" + # Get installed modules + self.get_pipeline_modules() + modules = self.module_names + # Nothing found + if len(modules) == 0: + log.info(f"No nf-core modules found in '{self.dir}'") + return "" + + for mod in sorted(modules): + table.add_row(mod) + if print_json: + return json.dumps(modules, sort_keys=True, indent=4) + return table diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 6e65a8e1b9..85083066a4 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -158,29 +158,21 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_ return all(files_are_equal) -def prompt_module_version_sha(module, installed_sha=None): - older_commits_choice = questionary.Choice( - title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value="" - ) - git_sha = "" - page_nbr = 1 - next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr) - while git_sha is "": - commits = next_page_commits - next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1) - choices = [] - for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits): - - display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired" - message = f"{title} {sha}" - if installed_sha == sha: - message += " (installed version)" - commit_display = [(display_color, message), ("class:choice-default", "")] - choices.append(questionary.Choice(title=commit_display, value=sha)) - if len(next_page_commits) > 0: - choices += [older_commits_choice] - git_sha = questionary.select( - f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - page_nbr += 1 - return git_sha +def get_repo_type(dir): + """ + Determine whether this is a pipeline repository or a clone of + nf-core/modules + """ + # Verify that the pipeline dir exists + if dir is None or not os.path.exists(dir): + log.error("Could not find directory: {}".format(dir)) + sys.exit(1) + + # Determine repository type + if os.path.exists(os.path.join(dir, "main.nf")): + return "pipeline" + elif os.path.exists(os.path.join(dir, "software")): + return "modules" + else: + log.error("Could not determine repository type of {}".format(dir)) + sys.exit(1) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py new file mode 100644 index 0000000000..bec67a6bec --- /dev/null +++ b/nf_core/modules/modules_command.py @@ -0,0 +1,105 @@ +import os +import glob +import shutil +import json +import logging +import questionary + +import nf_core.modules.module_utils +from nf_core.modules.modules_repo import ModulesRepo + +log = logging.getLogger(__name__) + + +class ModuleCommand: + """ + Base class for the 'nf-core modules' commands + """ + + def __init__(self, dir): + """ + Initialise the ModulesCommand object + """ + self.modules_repo = ModulesRepo() + self.dir = dir + self.module_names = [] + if self.dir: + self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + else: + self.repo_type = None + + def get_pipeline_modules(self): + """Get list of modules installed in the current directory""" + self.module_names = [] + if self.repo_type == "pipeline": + module_base_path = f"{self.dir}/modules/nf-core/software" + elif self.repo_type == "modules": + module_base_path = f"{self.dir}/software" + else: + log.error("Directory is neither a clone of nf-core/modules nor a pipeline") + raise SystemError + module_mains_path = f"{module_base_path}/**/main.nf" + module_mains = glob.glob(module_mains_path, recursive=True) + for mod in module_mains: + self.module_names.append(os.path.dirname(os.path.relpath(mod, module_base_path))) + + def has_valid_directory(self): + """Check that we were given a pipeline or clone of nf-core/modules""" + if self.repo_type == "modules": + return True + if self.dir is None or not os.path.exists(self.dir): + log.error("Could not find pipeline: {}".format(self.dir)) + return False + main_nf = os.path.join(self.dir, "main.nf") + nf_config = os.path.join(self.dir, "nextflow.config") + if not os.path.exists(main_nf) and not os.path.exists(nf_config): + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + self.has_modules_file() + return True + + def has_modules_file(self): + """Checks whether a module.json file has been created and creates one if it is missing""" + modules_json_path = os.path.join(self.dir, "modules.json") + if not os.path.exists(modules_json_path): + log.info("Creating missing 'module.json' file.") + nf_core.modules.module_utils.create_modules_json(self.dir) + + def clear_module_dir(self, module_name, module_dir): + """Removes all files in the module directory""" + try: + shutil.rmtree(module_dir) + # Try cleaning up empty parent if tool/subtool and tool/ is empty + if module_name.count("/") > 0: + parent_dir = os.path.dirname(module_dir) + try: + os.rmdir(parent_dir) + except OSError: + log.debug(f"Parent directory not empty: '{parent_dir}'") + else: + log.debug(f"Deleted orphan tool directory: '{parent_dir}'") + log.debug("Successfully removed {} module".format(module_name)) + return True + except OSError as e: + log.error("Could not remove module: {}".format(e)) + return False + + def download_module_file(self, module_name, module_version, install_folder, module_dir): + """Downloads the files of a module from the remote repo""" + files = self.modules_repo.get_module_file_urls(module_name, module_version) + log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys()))) + for filename, api_url in files.items(): + split_filename = filename.split("/") + dl_filename = os.path.join(self.dir, "modules", *install_folder, *split_filename[1:]) + try: + self.modules_repo.download_gh_file(dl_filename, api_url) + except SystemError as e: + log.error(e) + return False + log.info("Downloaded {} files to {}".format(len(files), module_dir)) + return True + + def update_modules_json(self, modules_json, modules_json_path, module_name, module_version): + """Updates the 'module.json' file with new module info""" + modules_json["modules"][module_name] = {"git_sha": module_version} + with open(modules_json_path, "w") as fh: + json.dump(modules_json, fh, indent=4) diff --git a/nf_core/modules/pipeline_modules.py b/nf_core/modules/pipeline_modules.py deleted file mode 100644 index 3154a29c5b..0000000000 --- a/nf_core/modules/pipeline_modules.py +++ /dev/null @@ -1,329 +0,0 @@ -#!/usr/bin/env python -""" -Code to handle several functions in order to deal with nf-core/modules in -nf-core pipelines - -* list modules -* install modules -* remove modules -* update modules (TODO) -* -""" - -from __future__ import print_function -import base64 -import glob -import json -import logging -import os -import re -import hashlib -import questionary -import requests -import rich -import shutil -import yaml -from rich.console import Console -from rich.table import Table -from rich.markdown import Markdown -import rich -from nf_core.utils import rich_force_colors -from nf_core.lint.pipeline_todos import pipeline_todos -import sys - -import nf_core.utils -from .module_utils import create_modules_json, get_module_git_log, prompt_module_version_sha -from .modules_repo import ModulesRepo - -log = logging.getLogger(__name__) - - -class PipelineModules(object): - def __init__(self): - """ - Initialise the PipelineModules object - """ - self.modules_repo = ModulesRepo() - self.pipeline_dir = None - self.pipeline_module_names = [] - self.force = False - self.latest = False - self.sha = None - - def list_modules(self, print_json=False): - """ - Get available module names from GitHub tree for repo - and print as list to stdout - """ - - # Initialise rich table - table = rich.table.Table() - table.add_column("Module Name") - modules = [] - - # No pipeline given - show all remote - if self.pipeline_dir is None: - log.info(f"Modules available from {self.modules_repo.name} ({self.modules_repo.branch}):\n") - - # Get the list of available modules - self.modules_repo.get_modules_file_tree() - modules = self.modules_repo.modules_avail_module_names - # Nothing found - if len(modules) == 0: - log.info(f"No available modules found in {self.modules_repo.name} ({self.modules_repo.branch})") - return "" - - # We have a pipeline - list what's installed - else: - log.info(f"Modules installed in '{self.pipeline_dir}':\n") - - # Check whether pipelines is valid - try: - self.has_valid_pipeline() - except UserWarning as e: - log.error(e) - return "" - # Get installed modules - self.get_pipeline_modules() - modules = self.pipeline_module_names - # Nothing found - if len(modules) == 0: - log.info(f"No nf-core modules found in '{self.pipeline_dir}'") - return "" - - for mod in sorted(modules): - table.add_row(mod) - if print_json: - return json.dumps(modules, sort_keys=True, indent=4) - return table - - def install(self, module=None): - - # Check whether pipelines is valid - self.has_valid_pipeline() - - # Get the available modules - self.modules_repo.get_modules_file_tree() - if self.latest and self.sha is not None: - log.error("Cannot use '--sha' and '--latest' at the same time!") - return False - - if module is None: - module = questionary.autocomplete( - "Tool name:", - choices=self.modules_repo.modules_avail_module_names, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Check that the supplied name is an available module - if module not in self.modules_repo.modules_avail_module_names: - log.error("Module '{}' not found in list of available modules.".format(module)) - log.info("Use the command 'nf-core modules list' to view available software") - return False - # Set the install folder based on the repository name - install_folder = ["nf-core", "software"] - if not self.modules_repo.name == "nf-core/modules": - install_folder = ["external"] - - # Compute the module directory - module_dir = os.path.join(self.pipeline_dir, "modules", *install_folder, module) - - # Load 'modules.json' - modules_json_path = os.path.join(self.pipeline_dir, "modules.json") - with open(modules_json_path, "r") as fh: - modules_json = json.load(fh) - - current_entry = modules_json["modules"].get(module) - - if current_entry is not None and self.sha is None: - # Fetch the latest commit for the module - current_version = current_entry["git_sha"] - git_log = get_module_git_log(module, per_page=1, page_nbr=1) - if len(git_log) == 0: - log.error(f"Was unable to fetch version of module '{module}'") - return False - latest_version = git_log[0]["git_sha"] - if current_version == latest_version and not self.force: - log.info("Already up to date") - return True - elif not self.force: - log.error("Found newer version of module.") - self.latest = self.force = questionary.confirm( - "Do you want install it? (--force --latest)", default=False - ).unsafe_ask() - if not self.latest: - return False - else: - latest_version = None - - # Check that we don't already have a folder for this module - if not self.check_module_files_installed(module, module_dir): - return False - - if self.sha: - if not current_entry is None and not self.force: - return False - if self.download_module_file(module, self.sha, install_folder, module_dir): - self.update_modules_json(modules_json, modules_json_path, module, self.sha) - return True - else: - try: - version = prompt_module_version_sha(module, installed_sha=current_entry["git_sha"]) - except SystemError as e: - log.error(e) - return False - else: - if self.latest: - # Fetch the latest commit for the module - if latest_version is None: - git_log = get_module_git_log(module, per_page=1, page_nbr=1) - if len(git_log) == 0: - log.error(f"Was unable to fetch version of module '{module}'") - return False - latest_version = git_log[0]["git_sha"] - version = latest_version - else: - try: - version = prompt_module_version_sha( - module, installed_sha=current_entry["git_sha"] if not current_entry is None else None - ) - except SystemError as e: - log.error(e) - return False - - log.info("Installing {}".format(module)) - log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash)) - - # Download module files - if not self.download_module_file(module, version, install_folder, module_dir): - return False - - # Update module.json with newly installed module - self.update_modules_json(modules_json, modules_json_path, module, version) - return True - - def update(self, module, force=False): - log.error("This command is not yet implemented") - pass - - def remove(self, module): - """ - Remove an already installed module - This command only works for modules that are installed from 'nf-core/modules' - """ - - # Check whether pipelines is valid - self.has_valid_pipeline() - - # Get the installed modules - self.get_pipeline_modules() - - if module is None: - if len(self.pipeline_module_names) == 0: - log.error("No installed modules found in pipeline") - return False - module = questionary.autocomplete( - "Tool name:", choices=self.pipeline_module_names, style=nf_core.utils.nfcore_question_style - ).ask() - - # Set the install folder based on the repository name - install_folder = ["nf-core", "software"] - if not self.modules_repo.name == "nf-core/modules": - install_folder = ["external"] - - # Get the module directory - module_dir = os.path.join(self.pipeline_dir, "modules", *install_folder, module) - - # Verify that the module is actually installed - if not os.path.exists(module_dir): - log.error("Module directory is not installed: {}".format(module_dir)) - log.info("The module you want to remove does not seem to be installed") - return False - - log.info("Removing {}".format(module)) - - # Remove the module - return self.clear_module_dir(module_name=module, module_dir=module_dir) - - def get_pipeline_modules(self): - """Get list of modules installed in the current pipeline""" - self.pipeline_module_names = [] - module_mains = glob.glob(f"{self.pipeline_dir}/modules/nf-core/software/**/main.nf", recursive=True) - for mod in module_mains: - self.pipeline_module_names.append( - os.path.dirname(os.path.relpath(mod, f"{self.pipeline_dir}/modules/nf-core/software/")) - ) - - def has_valid_pipeline(self): - """Check that we were given a pipeline""" - if self.pipeline_dir is None or not os.path.exists(self.pipeline_dir): - log.error("Could not find pipeline: {}".format(self.pipeline_dir)) - return False - main_nf = os.path.join(self.pipeline_dir, "main.nf") - nf_config = os.path.join(self.pipeline_dir, "nextflow.config") - if not os.path.exists(main_nf) and not os.path.exists(nf_config): - raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.pipeline_dir}'") - self.has_modules_file() - return True - - def has_modules_file(self): - """Checks whether a module.json file has been created and creates one if it is missing""" - modules_json_path = os.path.join(self.pipeline_dir, "modules.json") - if not os.path.exists(modules_json_path): - log.info("Creating missing 'module.json' file.") - create_modules_json(self.pipeline_dir) - - def clear_module_dir(self, module_name, module_dir): - try: - shutil.rmtree(module_dir) - # Try cleaning up empty parent if tool/subtool and tool/ is empty - if module_name.count("/") > 0: - parent_dir = os.path.dirname(module_dir) - try: - os.rmdir(parent_dir) - except OSError: - log.debug(f"Parent directory not empty: '{parent_dir}'") - else: - log.debug(f"Deleted orphan tool directory: '{parent_dir}'") - log.debug("Successfully removed {} module".format(module_name)) - return True - except OSError as e: - log.error("Could not remove module: {}".format(e)) - return False - - def download_module_file(self, module_name, module_version, install_folder, module_dir): - """Downloads the files of a module from the remote repo""" - files = self.modules_repo.get_module_file_urls(module_name, module_version) - log.debug("Fetching module files:\n - {}".format("\n - ".join(files.keys()))) - for filename, api_url in files.items(): - split_filename = filename.split("/") - dl_filename = os.path.join(self.pipeline_dir, "modules", *install_folder, *split_filename[1:]) - try: - self.modules_repo.download_gh_file(dl_filename, api_url) - except SystemError as e: - log.error(e) - return False - log.info("Downloaded {} files to {}".format(len(files), module_dir)) - return True - - def check_module_files_installed(self, module_name, module_dir): - """Checks if a module is already installed""" - if os.path.exists(module_dir): - if not self.force: - log.error(f"Module directory '{module_dir}' already exists.") - self.force = questionary.confirm( - "Do you want to overwrite local files? (--force)", default=False - ).unsafe_ask() - if self.force: - log.info(f"Removing old version of module '{module_name}'") - return self.clear_module_dir(module_name, module_dir) - else: - return False - else: - return True - - def update_modules_json(self, modules_json, modules_json_path, module_name, module_version): - """Updates the 'module.json' file with new module info""" - modules_json["modules"][module_name] = {"git_sha": module_version} - with open(modules_json_path, "w") as fh: - json.dump(modules_json, fh, indent=4) diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py new file mode 100644 index 0000000000..3313343a8b --- /dev/null +++ b/nf_core/modules/remove.py @@ -0,0 +1,62 @@ +import os +import sys +import questionary +import logging + + +import nf_core.utils + +from .modules_command import ModuleCommand + +log = logging.getLogger(__name__) + + +class ModuleRemove(ModuleCommand): + def __init__(self, pipeline_dir): + """ + Initialise the ModulesRemove object and run remove command + """ + super().__init__(pipeline_dir) + + def remove(self, module): + """ + Remove an already installed module + This command only works for modules that are installed from 'nf-core/modules' + """ + if self.repo_type == "modules": + log.error("You cannot remove a module in a clone of nf-core/modules") + return False + + # Check whether pipelines is valid + self.has_valid_directory() + + # Get the installed modules + self.get_pipeline_modules() + + if module is None: + if len(self.module_names) == 0: + log.error("No installed modules found in pipeline") + return False + module = questionary.autocomplete( + "Tool name:", choices=self.module_names, style=nf_core.utils.nfcore_question_style + ).ask() + + # Set the install folder based on the repository name + install_folder = ["nf-core", "software"] + print(self.modules_repo.name) + if not self.modules_repo.name == "nf-core/modules": + install_folder = ["external"] + + # Get the module directory + module_dir = os.path.join(self.dir, "modules", *install_folder, module) + + # Verify that the module is actually installed + if not os.path.exists(module_dir): + log.error("Module directory is not installed: {}".format(module_dir)) + log.info("The module you want to remove does not seem to be installed") + return False + + log.info("Removing {}".format(module)) + + # Remove the module + return self.clear_module_dir(module_name=module, module_dir=module_dir) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 8912784e38..5b3ca41078 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -21,7 +21,8 @@ import operator import nf_core.utils -import nf_core.modules.pipeline_modules + +from .modules_repo import ModulesRepo log = logging.getLogger(__name__) @@ -62,7 +63,7 @@ def check_inputs(self): # Get the tool name if not specified if self.module_name is None: - modules_repo = nf_core.modules.pipeline_modules.ModulesRepo() + modules_repo = ModulesRepo() modules_repo.get_modules_file_tree() self.module_name = questionary.autocomplete( "Tool name:", diff --git a/tests/test_modules.py b/tests/test_modules.py index d792d297e7..5f8f3d89e2 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -38,13 +38,18 @@ def setUp(self): self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") self.pipeline_dir = os.path.join(tempfile.mkdtemp(), "mypipeline") shutil.copytree(self.template_dir, self.pipeline_dir) - self.mods = nf_core.modules.PipelineModules() - self.mods.pipeline_dir = self.pipeline_dir - self.mods.latest = self.mods.force = True - self.mods_alt = nf_core.modules.PipelineModules() - self.mods_alt.pipeline_dir = self.pipeline_dir - self.mods_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") - self.mods_alt.latest = self.mods_alt.force = True + + # Set up install objects + print("Setting up install objects") + self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, latest=True, force=True) + self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, latest=True, force=True) + self.mods_install_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") + + # Set up remove objects + print("Setting up remove objects") + self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) + self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir) + self.mods_remove_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy() @@ -57,8 +62,8 @@ def test_modulesrepo_class(self): def test_modules_list(self): """Test listing available modules""" - self.mods.pipeline_dir = None - listed_mods = self.mods.list_modules() + mods_list = nf_core.modules.ModuleList(None) + listed_mods = mods_list.list_modules() console = Console(record=True) console.print(listed_mods) output = console.export_text() @@ -66,58 +71,58 @@ def test_modules_list(self): def test_modules_install_nopipeline(self): """Test installing a module - no pipeline given""" - self.mods.pipeline_dir = None - assert self.mods.install("foo") is False + self.mods_install.dir = None + assert self.mods_install.install("foo") is False def test_modules_install_emptypipeline(self): """Test installing a module - empty dir given""" - self.mods.pipeline_dir = tempfile.mkdtemp() + self.mods_install.dir = tempfile.mkdtemp() with pytest.raises(UserWarning) as excinfo: - self.mods.install("foo") + self.mods_install.install("foo") assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) def test_modules_install_nomodule(self): """Test installing a module - unrecognised module given""" - assert self.mods.install("foo") is False + assert self.mods_install.install("foo") is False def test_modules_install_trimgalore(self): """Test installing a module - TrimGalore!""" - assert self.mods.install("trimgalore") is not False - module_path = os.path.join(self.mods.pipeline_dir, "modules", "nf-core", "software", "trimgalore") + assert self.mods_install.install("trimgalore") is not False + module_path = os.path.join(self.mods_install.dir, "modules", "nf-core", "software", "trimgalore") assert os.path.exists(module_path) def test_modules_install_trimgalore_alternative_source(self): """Test installing a module from a different source repository - TrimGalore!""" - assert self.mods_alt.install("trimgalore") is not False - module_path = os.path.join(self.mods.pipeline_dir, "modules", "external", "trimgalore") + assert self.mods_install_alt.install("trimgalore") is not False + module_path = os.path.join(self.mods_install.dir, "modules", "external", "trimgalore") assert os.path.exists(module_path) def test_modules_install_trimgalore_twice(self): """Test installing a module - TrimGalore! already there""" - self.mods.install("trimgalore") - assert self.mods.install("trimgalore") is True + self.mods_install.install("trimgalore") + assert self.mods_install.install("trimgalore") is True def test_modules_remove_trimgalore(self): """Test removing TrimGalore! module after installing it""" - self.mods.install("trimgalore") - module_path = os.path.join(self.mods.pipeline_dir, "modules", "nf-core", "software", "trimgalore") - assert self.mods.remove("trimgalore") + self.mods_install.install("trimgalore") + module_path = os.path.join(self.mods_install.dir, "modules", "nf-core", "software", "trimgalore") + assert self.mods_remove.remove("trimgalore") assert os.path.exists(module_path) is False def test_modules_remove_trimgalore_alternative_source(self): """Test removing TrimGalore! module after installing it from an alternative source""" - self.mods_alt.install("trimgalore") - module_path = os.path.join(self.mods.pipeline_dir, "modules", "external", "trimgalore") - assert self.mods_alt.remove("trimgalore") + self.mods_install_alt.install("trimgalore") + module_path = os.path.join(self.mods_install.dir, "modules", "external", "trimgalore") + assert self.mods_remove_alt.remove("trimgalore") assert os.path.exists(module_path) is False def test_modules_remove_trimgalore_uninstalled(self): """Test removing TrimGalore! module without installing it""" - assert self.mods.remove("trimgalore") is False + assert self.mods_remove.remove("trimgalore") is False def test_modules_lint_trimgalore(self): """Test linting the TrimGalore! module""" - self.mods.install("trimgalore") + self.mods_install.install("trimgalore") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) module_lint.lint(print_results=False, module="trimgalore") assert len(module_lint.passed) == 20 @@ -126,8 +131,8 @@ def test_modules_lint_trimgalore(self): def test_modules_lint_empty(self): """Test linting a pipeline with no modules installed""" - self.mods.remove("fastqc") - self.mods.remove("multiqc") + self.mods_remove.remove("fastqc") + self.mods_remove.remove("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) module_lint.lint(print_results=False, all_modules=True) assert len(module_lint.passed) == 0