Merge pull request #2000 from mirpedrol/componentscommand

Substitute `ModulesCommand` and `SubworkflowsCommand` by `ComponentsCommand`

CHANGELOG.md

@@ -27,6 +27,7 @@
 - Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out`
 - Updated GitHub actions ([#1998](https://github.com/nf-core/tools/pull/1998), [#2001](https://github.com/nf-core/tools/pull/2001))
 - Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999))
+- Substitute ModulesCommand and SubworkflowsCommand by ComponentsCommand ([#2000](https://github.com/nf-core/tools/pull/2000))
 
 ### Modules
 

nf_core/components/components_command.py

@@ -158,22 +158,22 @@ def load_lint_config(self):
         except FileNotFoundError:
             log.debug(f"No lint config file found: {config_fn}")
 
-    def check_component_structure(self, component_name):
+    def check_modules_structure(self):
         """
-        Check that the structure of the modules/subworkflow directory in a pipeline is the correct one:
-            modules/nf-core/TOOL/SUBTOOL | subworkflows/nf-core/SUBWORKFLOW
+        Check that the structure of the modules directory in a pipeline is the correct one:
+            modules/nf-core/TOOL/SUBTOOL
 
         Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting:
             modules/nf-core/modules/TOOL/SUBTOOL
         """
         if self.repo_type == "pipeline":
             wrong_location_modules = []
-            for directory, _, files in os.walk(Path(self.dir, component_name)):
+            for directory, _, files in os.walk(Path(self.dir, "modules")):
                 if "main.nf" in files:
-                    module_path = Path(directory).relative_to(Path(self.dir, component_name))
+                    module_path = Path(directory).relative_to(Path(self.dir, "modules"))
                     parts = module_path.parts
                     # Check that there are modules installed directly under the 'modules' directory
-                    if parts[1] == component_name:
+                    if parts[1] == "modules":
                         wrong_location_modules.append(module_path)
             # If there are modules installed in the wrong location
             if len(wrong_location_modules) > 0:
@@ -185,12 +185,43 @@ def check_component_structure(self, component_name):
                 )
                 # Move wrong modules to the right directory
                 for module in wrong_location_modules:
-                    modules_dir = Path(component_name).resolve()
+                    modules_dir = Path("modules").resolve()
                     correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:]))
                     wrong_dir = Path(modules_dir, module)
                     shutil.move(wrong_dir, correct_dir)
                     log.info(f"Moved {wrong_dir} to {correct_dir}.")
-                shutil.rmtree(Path(self.dir, component_name, self.modules_repo.repo_path, component_name))
+                shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules"))
                 # Regenerate modules.json file
                 modules_json = ModulesJson(self.dir)
                 modules_json.check_up_to_date()
+
+    def check_patch_paths(self, patch_path, module_name):
+        """
+        Check that paths in patch files are updated to the new modules path
+        """
+        if patch_path.exists():
+            log.info(f"Modules {module_name} contains a patch file.")
+            rewrite = False
+            with open(patch_path, "r") as fh:
+                lines = fh.readlines()
+                for index, line in enumerate(lines):
+                    # Check if there are old paths in the patch file and replace
+                    if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line:
+                        rewrite = True
+                        lines[index] = line.replace(
+                            f"modules/{self.modules_repo.repo_path}/modules/{module_name}/",
+                            f"modules/{self.modules_repo.repo_path}/{module_name}/",
+                        )
+            if rewrite:
+                log.info(f"Updating paths in {patch_path}")
+                with open(patch_path, "w") as fh:
+                    for line in lines:
+                        fh.write(line)
+                # Update path in modules.json if the file is in the correct format
+                modules_json = ModulesJson(self.dir)
+                modules_json.load()
+                if modules_json.has_git_url_and_modules():
+                    modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][
+                        self.modules_repo.repo_path
+                    ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve()))
+                modules_json.dump()

nf_core/components/components_test.py

@@ -1,12 +1,30 @@
+import logging
+import os
+import sys
+from pathlib import Path
+from shutil import which
+
+import pytest
+import questionary
+import rich
+
+import nf_core.modules.modules_utils
+import nf_core.utils
+from nf_core.components.components_command import ComponentCommand
+from nf_core.modules.modules_json import ModulesJson
+
+log = logging.getLogger(__name__)
+
+
 class ComponentsTest(ComponentCommand):
     """
-    Class to run module pytests.
+    Class to run module and subworkflow pytests.
 
     ...
 
     Attributes
     ----------
-    module_name : str
+    component_name : str
         name of the tool to run tests for
     no_prompts : bool
         flat indicating if prompts are used
@@ -18,7 +36,7 @@ class ComponentsTest(ComponentCommand):
     run():
         Run test steps
     _check_inputs():
-        Check inputs. Ask for module_name if not provided and check that the directory exists
+        Check inputs. Ask for component_name if not provided and check that the directory exists
     _set_profile():
         Set software profile
     _run_pytests(self):
@@ -27,19 +45,19 @@ class ComponentsTest(ComponentCommand):
 
     def __init__(
         self,
-        module_name=None,
+        component_type,
+        component_name=None,
         no_prompts=False,
         pytest_args="",
         remote_url=None,
         branch=None,
         no_pull=False,
     ):
-        self.module_name = module_name
+        super().__init__(component_type=component_type, dir=".", remote_url=remote_url, branch=branch, no_pull=no_pull)
+        self.component_name = component_name
         self.no_prompts = no_prompts
         self.pytest_args = pytest_args
 
-        super().__init__(".", remote_url, branch, no_pull)
-
     def run(self):
         """Run test steps"""
         if not self.no_prompts:
@@ -58,27 +76,34 @@ def _check_inputs(self):
 
         # Retrieving installed modules
         if self.repo_type == "modules":
-            installed_modules = self.get_modules_clone_modules()
+            installed_components = self.get_components_clone_modules()
         else:
             modules_json = ModulesJson(self.dir)
             modules_json.check_up_to_date()
-            installed_modules = modules_json.get_all_modules().get(self.modules_repo.remote_url)
+            if self.component_type == "modules":
+                installed_components = modules_json.get_all_modules().get(self.modules_repo.remote_url)
+            elif self.component_type == "subworkflows":
+                modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url)
 
-        # Get the tool name if not specified
-        if self.module_name is None:
+        # Get the component name if not specified
+        if self.component_name is None:
             if self.no_prompts:
                 raise UserWarning(
-                    "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL."
+                    f"{self.component_type[:-1].title()} name not provided and prompts deactivated. Please provide the {self.component_type[:-1]} name{' as TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}."
                 )
-            if not installed_modules:
+            if not installed_components:
+                if self.component_type == "modules":
+                    dir_structure_message = f"modules/{self.modules_repo.repo_path}/TOOL/SUBTOOL/ and tests/modules/{self.modules_repo.repo_path}/TOOLS/SUBTOOL/"
+                elif self.component_type == "subworkflows":
+                    dir_structure_message = f"subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/ and tests/subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/"
                 raise UserWarning(
-                    f"No installed modules were found from '{self.modules_repo.remote_url}'.\n"
-                    f"Are you running the tests inside the nf-core/modules main directory?\n"
-                    f"Otherwise, make sure that the directory structure is modules/TOOL/SUBTOOL/ and tests/modules/TOOLS/SUBTOOL/"
+                    f"No installed {self.component_type} were found from '{self.modules_repo.remote_url}'.\n"
+                    f"Are you running the tests inside the repository root directory?\n"
+                    f"Make sure that the directory structure is {dir_structure_message}"
                 )
-            self.module_name = questionary.autocomplete(
+            self.component_name = questionary.autocomplete(
                 "Tool name:",
-                choices=installed_modules,
+                choices=installed_components,
                 style=nf_core.utils.nfcore_question_style,
             ).unsafe_ask()
 
@@ -89,19 +114,25 @@ def _validate_folder_structure(self):
         """Validate that the modules follow the correct folder structure to run the tests:
         - modules/nf-core/TOOL/SUBTOOL/
         - tests/modules/nf-core/TOOL/SUBTOOL/
-
+        or
+        - subworkflows/nf-core/SUBWORKFLOW/
+        - tests/subworkflows/nf-core/SUBWORKFLOW/
         """
-        module_path = Path(self.default_modules_path) / self.module_name
-        test_path = Path(self.default_tests_path) / self.module_name
-
-        if not (self.dir / module_path).is_dir():
+        if self.component_type == "modules":
+            component_path = Path(self.default_modules_path) / self.component_name
+            test_path = Path(self.default_tests_path) / self.component_name
+        elif self.component_type == "subworkflows":
+            component_path = Path(self.default_subworkflows_path) / self.component_name
+            test_path = Path(self.default_subworkflows_tests_path) / self.component_name
+
+        if not (self.dir / component_path).is_dir():
             raise UserWarning(
-                f"Cannot find directory '{module_path}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?"
+                f"Cannot find directory '{component_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. Are you running the tests inside the modules repository root directory?"
             )
         if not (self.dir / test_path).is_dir():
             raise UserWarning(
-                f"Cannot find directory '{test_path}'. Should be TOOL/SUBTOOL or TOOL. "
-                "Are you running the tests inside the nf-core/modules main directory? "
+                f"Cannot find directory '{test_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. "
+                "Are you running the tests inside the modules repository root directory? "
                 "Do you have tests for the specified module?"
             )
 
@@ -144,15 +175,15 @@ def _check_profile(self):
             )
 
     def _run_pytests(self):
-        """Given a module name, run tests."""
+        """Given a module/subworkflow name, run tests."""
         # Print nice divider line
         console = rich.console.Console()
-        console.rule(self.module_name, style="black")
+        console.rule(self.component_name, style="black")
 
         # Set pytest arguments
-        command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"]
+        command_args = ["--tag", f"{self.component_name}", "--symlink", "--keep-workflow-wd", "--git-aware"]
         command_args += self.pytest_args
 
         # Run pytest
-        log.info(f"Running pytest for module '{self.module_name}'")
+        log.info(f"Running pytest for {self.component_type[:-1]} '{self.component_name}'")
         sys.exit(pytest.main(command_args))

nf_core/components/list.py

@@ -6,8 +6,6 @@
 import nf_core.modules.modules_utils
 from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson
-
-# from .modules_command import ModulesRepo
 from nf_core.modules.modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)

nf_core/modules/bump_versions.py

@@ -18,17 +18,16 @@
 
 import nf_core.modules.modules_utils
 import nf_core.utils
+from nf_core.components.components_command import ComponentCommand
 from nf_core.utils import plural_s as _s
 from nf_core.utils import rich_force_colors
 
-from .modules_command import ModuleCommand
-
 log = logging.getLogger(__name__)
 
 
-class ModuleVersionBumper(ModuleCommand):
+class ModuleVersionBumper(ComponentCommand):
     def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False):
-        super().__init__(pipeline_dir, remote_url, branch, no_pull)
+        super().__init__("modules", pipeline_dir, remote_url, branch, no_pull)
 
         self.up_to_date = None
         self.updated = None

nf_core/modules/create.py

@@ -17,13 +17,12 @@
 import nf_core.components.components_create
 import nf_core.modules.modules_utils
 import nf_core.utils
-
-from .modules_command import ModuleCommand
+from nf_core.components.components_command import ComponentCommand
 
 log = logging.getLogger(__name__)
 
 
-class ModuleCreate(ModuleCommand):
+class ModuleCreate(ComponentCommand):
     def __init__(
         self,
         directory=".",
@@ -36,7 +35,7 @@ def __init__(
         conda_version=None,
         repo_type=None,
     ):
-        super().__init__(directory)
+        super().__init__("modules", directory)
         self.directory = directory
         self.tool = tool
         self.author = author
@@ -157,7 +156,7 @@ def create(self):
                 pytest_modules_yml = dict(sorted(pytest_modules_yml.items()))
                 with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh:
                     yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper())
-            except FileNotFoundError as e:
+            except FileNotFoundError:
                 raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!")
 
         new_files = list(self.file_paths.values())

nf_core/modules/info.py

@@ -11,16 +11,16 @@
 from rich.text import Text
 
 import nf_core.utils
+from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson
 
-from .modules_command import ModuleCommand
 from .modules_repo import NF_CORE_MODULES_REMOTE
 from .modules_utils import get_repo_type
 
 log = logging.getLogger(__name__)
 
 
-class ModuleInfo(ModuleCommand):
+class ModuleInfo(ComponentCommand):
     """
     Class to print information of a module.
 
@@ -56,7 +56,7 @@ class ModuleInfo(ModuleCommand):
     """
 
     def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull):
-        super().__init__(pipeline_dir, remote_url, branch, no_pull)
+        super().__init__("modules", pipeline_dir, remote_url, branch, no_pull)
         self.meta = None
         self.local_path = None
         self.remote_location = None
@@ -139,7 +139,6 @@ def get_local_yaml(self):
 
         if self.repo_type == "pipeline":
             # Try to find and load the meta.yml file
-            repo_name = self.modules_repo.repo_path
             module_base_path = os.path.join(self.dir, "modules")
             # Check that we have any modules installed from this repo
             modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url)

nf_core/modules/install.py

@@ -4,14 +4,13 @@
 import nf_core.components.components_install
 import nf_core.modules.modules_utils
 import nf_core.utils
+from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson
 
-from .modules_command import ModuleCommand
-
 log = logging.getLogger(__name__)
 
 
-class ModuleInstall(ModuleCommand):
+class ModuleInstall(ComponentCommand):
     def __init__(
         self,
         pipeline_dir,
@@ -23,7 +22,7 @@ def __init__(
         no_pull=False,
         installed_by=False,
     ):
-        super().__init__(pipeline_dir, remote_url, branch, no_pull)
+        super().__init__("modules", pipeline_dir, remote_url, branch, no_pull)
         self.force = force
         self.prompt = prompt
         self.sha = sha