Merge pull request #1995 from mashehu/migrate-sw-list

Migrate list command to components

nf_core/__main__.py

@@ -454,7 +454,7 @@ def remote(ctx, keywords, json):
             ctx.obj["modules_repo_branch"],
             ctx.obj["modules_repo_no_pull"],
         )
-        stdout.print(module_list.list_modules(keywords, json))
+        stdout.print(module_list.list_components(keywords, json))
     except (UserWarning, LookupError) as e:
         log.critical(e)
         sys.exit(1)
@@ -484,7 +484,7 @@ def local(ctx, keywords, json, dir):  # pylint: disable=redefined-builtin
             ctx.obj["modules_repo_branch"],
             ctx.obj["modules_repo_no_pull"],
         )
-        stdout.print(module_list.list_modules(keywords, json))
+        stdout.print(module_list.list_components(keywords, json))
     except (UserWarning, LookupError) as e:
         log.error(e)
         sys.exit(1)
@@ -999,6 +999,70 @@ def install(ctx, subworkflow, dir, prompt, force, sha):
         sys.exit(1)
 
 
+# nf-core subworkflows list subcommands
+@subworkflows.group()
+@click.pass_context
+def list(ctx):
+    """
+    List modules in a local pipeline or remote repository.
+    """
+    pass
+
+
+# nf-core subworkflows list remote
+@list.command()
+@click.pass_context
+@click.argument("keywords", required=False, nargs=-1, metavar="<filter keywords>")
+@click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout")
+def remote(ctx, keywords, json):
+    """
+    List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
+    """
+    try:
+        subworkflow_list = nf_core.subworkflows.SubworkflowList(
+            None,
+            True,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+        )
+
+        stdout.print(subworkflow_list.list_components(keywords, json))
+    except (UserWarning, LookupError) as e:
+        log.critical(e)
+        sys.exit(1)
+
+
+# nf-core subworkflows list local
+@list.command()
+@click.pass_context
+@click.argument("keywords", required=False, nargs=-1, metavar="<filter keywords>")
+@click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout")
+@click.option(
+    "-d",
+    "--dir",
+    type=click.Path(exists=True),
+    default=".",
+    help=r"Pipeline directory. [dim]\[default: Current working directory][/]",
+)
+def local(ctx, keywords, json, dir):  # pylint: disable=redefined-builtin
+    """
+    List subworkflows installed locally in a pipeline
+    """
+    try:
+        subworkflow_list = nf_core.subworkflows.SubworkflowList(
+            dir,
+            False,
+            ctx.obj["modules_repo_url"],
+            ctx.obj["modules_repo_branch"],
+            ctx.obj["modules_repo_no_pull"],
+        )
+        stdout.print(subworkflow_list.list_components(keywords, json))
+    except (UserWarning, LookupError) as e:
+        log.error(e)
+        sys.exit(1)
+
+
 # nf-core schema subcommands
 @nf_core_cli.group()
 def schema():

nf_core/components/components_command.py

@@ -157,3 +157,40 @@ def load_lint_config(self):
                 self.lint_config = yaml.safe_load(fh)
         except FileNotFoundError:
             log.debug(f"No lint config file found: {config_fn}")
+
+    def check_component_structure(self, component_name):
+        """
+        Check that the structure of the modules/subworkflow directory in a pipeline is the correct one:
+            modules/nf-core/TOOL/SUBTOOL | subworkflows/nf-core/SUBWORKFLOW
+
+        Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting:
+            modules/nf-core/modules/TOOL/SUBTOOL
+        """
+        if self.repo_type == "pipeline":
+            wrong_location_modules = []
+            for directory, _, files in os.walk(Path(self.dir, component_name)):
+                if "main.nf" in files:
+                    module_path = Path(directory).relative_to(Path(self.dir, component_name))
+                    parts = module_path.parts
+                    # Check that there are modules installed directly under the 'modules' directory
+                    if parts[1] == component_name:
+                        wrong_location_modules.append(module_path)
+            # If there are modules installed in the wrong location
+            if len(wrong_location_modules) > 0:
+                log.info("The modules folder structure is outdated. Reinstalling modules.")
+                # Remove the local copy of the modules repository
+                log.info(f"Updating '{self.modules_repo.local_repo_dir}'")
+                self.modules_repo.setup_local_repo(
+                    self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress
+                )
+                # Move wrong modules to the right directory
+                for module in wrong_location_modules:
+                    modules_dir = Path(component_name).resolve()
+                    correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:]))
+                    wrong_dir = Path(modules_dir, module)
+                    shutil.move(wrong_dir, correct_dir)
+                    log.info(f"Moved {wrong_dir} to {correct_dir}.")
+                shutil.rmtree(Path(self.dir, component_name, self.modules_repo.repo_path, component_name))
+                # Regenerate modules.json file
+                modules_json = ModulesJson(self.dir)
+                modules_json.check_up_to_date()

nf_core/components/list.py

@@ -0,0 +1,144 @@
+import json
+import logging
+
+import rich
+
+import nf_core.modules.modules_utils
+from nf_core.components.components_command import ComponentCommand
+from nf_core.modules.modules_json import ModulesJson
+
+# from .modules_command import ModulesRepo
+from nf_core.modules.modules_repo import ModulesRepo
+
+log = logging.getLogger(__name__)
+
+
+class ComponentList(ComponentCommand):
+    def __init__(self, component_type, pipeline_dir, remote=True, remote_url=None, branch=None, no_pull=False):
+        super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
+        self.remote = remote
+
+    def list_components(self, keywords=None, print_json=False):
+        keywords = keywords or []
+        """
+        Get available modules/subworkflows names from GitHub tree for repo
+        and print as list to stdout
+        """
+        # Check modules directory structure
+        # self.check_component_structure(self.component_type)
+
+        # Initialise rich table
+        table = rich.table.Table()
+        table.add_column(f"{self.component_type[:-1].capitalize()} Name")
+        components = []
+
+        def pattern_msg(keywords):
+            if len(keywords) == 0:
+                return ""
+            if len(keywords) == 1:
+                return f" matching pattern '{keywords[0]}'"
+            else:
+                quoted_keywords = (f"'{key}'" for key in keywords)
+                return f" matching patterns {', '.join(quoted_keywords)}"
+
+        # No pipeline given - show all remote
+        if self.remote:
+            # Filter the modules/subworkflows by keywords
+            components = [
+                comp
+                for comp in self.modules_repo.get_avail_components(self.component_type)
+                if all(k in comp for k in keywords)
+            ]
+
+            # Nothing found
+            if len(components) == 0:
+                log.info(
+                    f"No available {self.component_type} found in {self.modules_repo.remote_url} ({self.modules_repo.branch})"
+                    f"{pattern_msg(keywords)}"
+                )
+                return ""
+
+            for comp in sorted(components):
+                table.add_row(comp)
+
+        # We have a pipeline - list what's installed
+        else:
+            # Check that we are in a pipeline directory
+
+            try:
+                _, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir)
+                if repo_type != "pipeline":
+                    raise UserWarning(
+                        f"The command 'nf-core {self.component_type} list local' must be run from a pipeline directory.",
+                    )
+            except UserWarning as e:
+                log.error(e)
+                return ""
+            # Check whether pipelines is valid
+            try:
+                self.has_valid_directory()
+            except UserWarning as e:
+                log.error(e)
+                return ""
+
+            # Verify that 'modules.json' is consistent with the installed modules
+            modules_json = ModulesJson(self.dir)
+            modules_json.check_up_to_date()
+
+            # Filter by keywords
+            repos_with_comps = {
+                repo_url: [comp for comp in components if all(k in comp[1] for k in keywords)]
+                for repo_url, components in modules_json.get_all_components(self.component_type).items()
+            }
+
+            # Nothing found
+            if sum(map(len, repos_with_comps)) == 0:
+                log.info(f"No nf-core {self.component_type} found in '{self.dir}'{pattern_msg(keywords)}")
+                return ""
+
+            table.add_column("Repository")
+            table.add_column("Version SHA")
+            table.add_column("Message")
+            table.add_column("Date")
+
+            # Load 'modules.json'
+            modules_json = modules_json.modules_json
+
+            for repo_url, component_with_dir in sorted(repos_with_comps.items()):
+                repo_entry = modules_json["repos"].get(repo_url, {})
+                for install_dir, component in sorted(component_with_dir):
+                    repo_modules = repo_entry.get(self.component_type)
+                    component_entry = repo_modules.get(install_dir).get(component)
+
+                    if component_entry:
+                        version_sha = component_entry["git_sha"]
+                        try:
+                            # pass repo_name to get info on modules even outside nf-core/modules
+                            message, date = ModulesRepo(
+                                remote_url=repo_url,
+                                branch=component_entry["branch"],
+                            ).get_commit_info(version_sha)
+                        except LookupError as e:
+                            log.warning(e)
+                            date = "[red]Not Available"
+                            message = "[red]Not Available"
+                    else:
+                        log.warning(
+                            f"Commit SHA for {self.component_type[:-1]} '{install_dir}/{self.component_type}' is missing from 'modules.json'"
+                        )
+                        version_sha = "[red]Not Available"
+                        date = "[red]Not Available"
+                        message = "[red]Not Available"
+                    table.add_row(component, repo_url, version_sha, message, date)
+
+        if print_json:
+            return json.dumps(components, sort_keys=True, indent=4)
+
+        if self.remote:
+            log.info(
+                f"{self.component_type.capitalize()} available from {self.modules_repo.remote_url} ({self.modules_repo.branch})"
+                f"{pattern_msg(keywords)}:\n"
+            )
+        else:
+            log.info(f"{self.component_type.capitalize()} installed in '{self.dir}'{pattern_msg(keywords)}:\n")
+        return table