-
Notifications
You must be signed in to change notification settings - Fork 192
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1124 from ErikDanielsson/refactor-module-commands
Refactor `nf-core modules` commands
- Loading branch information
Showing
12 changed files
with
474 additions
and
424 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,7 @@ | ||
| from .pipeline_modules import ModulesRepo, PipelineModules | ||
| from .modules_repo import ModulesRepo | ||
| from .create import ModuleCreate | ||
| from .test_yml_builder import ModulesTestYmlBuilder | ||
| from .lint import ModuleLint | ||
| from .list import ModuleList | ||
| from .install import ModuleInstall | ||
| from .remove import ModuleRemove |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,173 @@ | ||
| import os | ||
| import sys | ||
| import json | ||
| import questionary | ||
| import logging | ||
|
|
||
| import nf_core.utils | ||
|
|
||
| from .modules_command import ModuleCommand | ||
| from .module_utils import get_module_git_log | ||
|
|
||
| log = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class ModuleInstall(ModuleCommand): | ||
| def __init__(self, pipeline_dir, module=None, force=False, latest=False, sha=None): | ||
| super().__init__(pipeline_dir) | ||
| self.force = force | ||
| self.latest = latest | ||
| self.sha = sha | ||
| module = module | ||
|
|
||
| def install(self, module): | ||
| if self.repo_type == "modules": | ||
| log.error("You cannot install a module in a clone of nf-core/modules") | ||
| return False | ||
| # Check whether pipelines is valid | ||
| self.has_valid_directory() | ||
|
|
||
| # Get the available modules | ||
| self.modules_repo.get_modules_file_tree() | ||
| if self.latest and self.sha is not None: | ||
| log.error("Cannot use '--sha' and '--latest' at the same time!") | ||
| return False | ||
|
|
||
| if module is None: | ||
| module = questionary.autocomplete( | ||
| "Tool name:", | ||
| choices=self.modules_repo.modules_avail_module_names, | ||
| style=nf_core.utils.nfcore_question_style, | ||
| ).unsafe_ask() | ||
|
|
||
| # Check that the supplied name is an available module | ||
| if module not in self.modules_repo.modules_avail_module_names: | ||
| log.error("Module '{}' not found in list of available modules.".format(module)) | ||
| log.info("Use the command 'nf-core modules list' to view available software") | ||
| return False | ||
| # Set the install folder based on the repository name | ||
| install_folder = ["nf-core", "software"] | ||
| if not self.modules_repo.name == "nf-core/modules": | ||
| install_folder = ["external"] | ||
|
|
||
| # Compute the module directory | ||
| module_dir = os.path.join(self.dir, "modules", *install_folder, module) | ||
|
|
||
| # Load 'modules.json' | ||
| modules_json_path = os.path.join(self.dir, "modules.json") | ||
| with open(modules_json_path, "r") as fh: | ||
| modules_json = json.load(fh) | ||
|
|
||
| current_entry = modules_json["modules"].get(module) | ||
|
|
||
| if current_entry is not None and self.sha is None: | ||
| # Fetch the latest commit for the module | ||
| current_version = current_entry["git_sha"] | ||
| git_log = get_module_git_log(module, per_page=1, page_nbr=1) | ||
| if len(git_log) == 0: | ||
| log.error(f"Was unable to fetch version of module '{module}'") | ||
| return False | ||
| latest_version = git_log[0]["git_sha"] | ||
| if current_version == latest_version and not self.force: | ||
| log.info("Already up to date") | ||
| return True | ||
| elif not self.force: | ||
| log.error("Found newer version of module.") | ||
| self.latest = self.force = questionary.confirm( | ||
| "Do you want install it? (--force --latest)", default=False | ||
| ).unsafe_ask() | ||
| if not self.latest: | ||
| return False | ||
| else: | ||
| latest_version = None | ||
|
|
||
| # Check that we don't already have a folder for this module | ||
| if not self.check_module_files_installed(module, module_dir): | ||
| return False | ||
|
|
||
| if self.sha: | ||
| if not current_entry is None and not self.force: | ||
| return False | ||
| if self.download_module_file(module, self.sha, install_folder, module_dir): | ||
| self.update_modules_json(modules_json, modules_json_path, module, self.sha) | ||
| return True | ||
| else: | ||
| try: | ||
| version = self.prompt_module_version_sha( | ||
| installed_sha=current_entry["git_sha"] if not current_entry is None else None | ||
| ) | ||
| except SystemError as e: | ||
| log.error(e) | ||
| return False | ||
| else: | ||
| if self.latest: | ||
| # Fetch the latest commit for the module | ||
| if latest_version is None: | ||
| git_log = get_module_git_log(module, per_page=1, page_nbr=1) | ||
| if len(git_log) == 0: | ||
| log.error(f"Was unable to fetch version of module '{module}'") | ||
| return False | ||
| latest_version = git_log[0]["git_sha"] | ||
| version = latest_version | ||
| else: | ||
| try: | ||
| version = self.prompt_module_version_sha( | ||
| installed_sha=current_entry["git_sha"] if not current_entry is None else None | ||
| ) | ||
| except SystemError as e: | ||
| log.error(e) | ||
| return False | ||
|
|
||
| log.info("Installing {}".format(module)) | ||
| log.debug("Installing module '{}' at modules hash {}".format(module, self.modules_repo.modules_current_hash)) | ||
|
|
||
| # Download module files | ||
| if not self.download_module_file(module, version, install_folder, module_dir): | ||
| return False | ||
|
|
||
| # Update module.json with newly installed module | ||
| self.update_modules_json(modules_json, modules_json_path, module, version) | ||
| return True | ||
|
|
||
| def check_module_files_installed(self, module_name, module_dir): | ||
| """Checks if a module is already installed""" | ||
| if os.path.exists(module_dir): | ||
| if not self.force: | ||
| log.error(f"Module directory '{module_dir}' already exists.") | ||
| self.force = questionary.confirm( | ||
| "Do you want to overwrite local files? (--force)", default=False | ||
| ).unsafe_ask() | ||
| if self.force: | ||
| log.info(f"Removing old version of module '{module_name}'") | ||
| return self.clear_module_dir(module_name, module_dir) | ||
| else: | ||
| return False | ||
| else: | ||
| return True | ||
|
|
||
| def prompt_module_version_sha(self, module, installed_sha=None): | ||
| older_commits_choice = questionary.Choice( | ||
| title=[("fg:ansiyellow", "older commits"), ("class:choice-default", "")], value="" | ||
| ) | ||
| git_sha = "" | ||
| page_nbr = 1 | ||
| next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr) | ||
| while git_sha is "": | ||
| commits = next_page_commits | ||
| next_page_commits = get_module_git_log(module, per_page=10, page_nbr=page_nbr + 1) | ||
| choices = [] | ||
| for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits): | ||
|
|
||
| display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired" | ||
| message = f"{title} {sha}" | ||
| if installed_sha == sha: | ||
| message += " (installed version)" | ||
| commit_display = [(display_color, message), ("class:choice-default", "")] | ||
| choices.append(questionary.Choice(title=commit_display, value=sha)) | ||
| if len(next_page_commits) > 0: | ||
| choices += [older_commits_choice] | ||
| git_sha = questionary.select( | ||
| f"Select '{module}' version", choices=choices, style=nf_core.utils.nfcore_question_style | ||
| ).unsafe_ask() | ||
| page_nbr += 1 | ||
| return git_sha |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| import json | ||
| from os import pipe | ||
| import rich | ||
| import logging | ||
|
|
||
| from .modules_command import ModuleCommand | ||
|
|
||
| log = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class ModuleList(ModuleCommand): | ||
| def __init__(self, pipeline_dir): | ||
| super().__init__(pipeline_dir) | ||
|
|
||
| def list_modules(self, print_json=False): | ||
| """ | ||
| Get available module names from GitHub tree for repo | ||
| and print as list to stdout | ||
| """ | ||
|
|
||
| # Initialise rich table | ||
| table = rich.table.Table() | ||
| table.add_column("Module Name") | ||
| modules = [] | ||
|
|
||
| # No pipeline given - show all remote | ||
| if self.dir is None: | ||
| log.info(f"Modules available from {self.modules_repo.name} ({self.modules_repo.branch}):\n") | ||
|
|
||
| # Get the list of available modules | ||
| self.modules_repo.get_modules_file_tree() | ||
| modules = self.modules_repo.modules_avail_module_names | ||
| # Nothing found | ||
| if len(modules) == 0: | ||
| log.info(f"No available modules found in {self.modules_repo.name} ({self.modules_repo.branch})") | ||
| return "" | ||
|
|
||
| # We have a pipeline - list what's installed | ||
| else: | ||
| log.info(f"Modules installed in '{self.dir}':\n") | ||
|
|
||
| # Check whether pipelines is valid | ||
| try: | ||
| self.has_valid_directory() | ||
| except UserWarning as e: | ||
| log.error(e) | ||
| return "" | ||
| # Get installed modules | ||
| self.get_pipeline_modules() | ||
| modules = self.module_names | ||
| # Nothing found | ||
| if len(modules) == 0: | ||
| log.info(f"No nf-core modules found in '{self.dir}'") | ||
| return "" | ||
|
|
||
| for mod in sorted(modules): | ||
| table.add_row(mod) | ||
| if print_json: | ||
| return json.dumps(modules, sort_keys=True, indent=4) | ||
| return table |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.