Substitute ModulesCommand and SubworkflowsCommand by ComponentsCommand

CHANGELOG.md

@@ -27,6 +27,7 @@
 - Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out`
 - Updated GitHub actions ([#1998](https://github.com/nf-core/tools/pull/1998), [#2001](https://github.com/nf-core/tools/pull/2001))
 - Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999))
+- Substitute ModulesCommand and SubworkflowsCommand by ComponentsCommand ([#2000](https://github.com/nf-core/tools/pull/2000))
 
 ### Modules
 

nf_core/components/components_command.py

@@ -158,22 +158,22 @@ def load_lint_config(self):
  except FileNotFoundError:
  log.debug(f"No lint config file found: {config_fn}")
 
- def check_component_structure(self, component_name):
+ def check_modules_structure(self):
  """
- Check that the structure of the modules/subworkflow directory in a pipeline is the correct one:
- modules/nf-core/TOOL/SUBTOOL | subworkflows/nf-core/SUBWORKFLOW
+ Check that the structure of the modules directory in a pipeline is the correct one:
+ modules/nf-core/TOOL/SUBTOOL
 
  Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting:
  modules/nf-core/modules/TOOL/SUBTOOL
  """
  if self.repo_type == "pipeline":
  wrong_location_modules = []
- for directory, _, files in os.walk(Path(self.dir, component_name)):
+ for directory, _, files in os.walk(Path(self.dir, "modules")):
  if "main.nf" in files:
- module_path = Path(directory).relative_to(Path(self.dir, component_name))
+ module_path = Path(directory).relative_to(Path(self.dir, "modules"))
  parts = module_path.parts
  # Check that there are modules installed directly under the 'modules' directory
- if parts[1] == component_name:
+ if parts[1] == "modules":
  wrong_location_modules.append(module_path)
  # If there are modules installed in the wrong location
  if len(wrong_location_modules) > 0:
@@ -185,12 +185,43 @@ def check_component_structure(self, component_name):
  )
  # Move wrong modules to the right directory
  for module in wrong_location_modules:
- modules_dir = Path(component_name).resolve()
+ modules_dir = Path("modules").resolve()
  correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:]))
  wrong_dir = Path(modules_dir, module)
  shutil.move(wrong_dir, correct_dir)
  log.info(f"Moved {wrong_dir} to {correct_dir}.")
- shutil.rmtree(Path(self.dir, component_name, self.modules_repo.repo_path, component_name))
+ shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules"))
  # Regenerate modules.json file
  modules_json = ModulesJson(self.dir)
  modules_json.check_up_to_date()
+
+ def check_patch_paths(self, patch_path, module_name):
+ """
+ Check that paths in patch files are updated to the new modules path
+ """
+ if patch_path.exists():
+ log.info(f"Modules {module_name} contains a patch file.")
+ rewrite = False
+ with open(patch_path, "r") as fh:
+ lines = fh.readlines()
+ for index, line in enumerate(lines):
+ # Check if there are old paths in the patch file and replace
+ if f"modules/{self.modules_repo.repo_path}/modules/{module_name}/" in line:
+ rewrite = True
+ lines[index] = line.replace(
+ f"modules/{self.modules_repo.repo_path}/modules/{module_name}/",
+ f"modules/{self.modules_repo.repo_path}/{module_name}/",
+ )
+ if rewrite:
+ log.info(f"Updating paths in {patch_path}")
+ with open(patch_path, "w") as fh:
+ for line in lines:
+ fh.write(line)
+ # Update path in modules.json if the file is in the correct format
+ modules_json = ModulesJson(self.dir)
+ modules_json.load()
+ if modules_json.has_git_url_and_modules():
+ modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][
+ self.modules_repo.repo_path
+ ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve()))
+ modules_json.dump()

nf_core/components/components_test.py

@@ -1,12 +1,30 @@
+import logging
+import os
+import sys
+from pathlib import Path
+from shutil import which
+
+import pytest
+import questionary
+import rich
+
+import nf_core.modules.modules_utils
+import nf_core.utils
+from nf_core.components.components_command import ComponentCommand
+from nf_core.modules.modules_json import ModulesJson
+
+log = logging.getLogger(__name__)
+
+
 class ComponentsTest(ComponentCommand):
  """
- Class to run module pytests.
+ Class to run module and subworkflow pytests.
 
  ...
 
  Attributes
  ----------
- module_name : str
+ component_name : str
  name of the tool to run tests for
  no_prompts : bool
  flat indicating if prompts are used
@@ -18,7 +36,7 @@ class ComponentsTest(ComponentCommand):
  run():
  Run test steps
  _check_inputs():
- Check inputs. Ask for module_name if not provided and check that the directory exists
+ Check inputs. Ask for component_name if not provided and check that the directory exists
  _set_profile():
  Set software profile
  _run_pytests(self):
@@ -27,19 +45,19 @@ class ComponentsTest(ComponentCommand):
 
  def __init__(
  self,
- module_name=None,
+ component_type,
+ component_name=None,
  no_prompts=False,
  pytest_args="",
  remote_url=None,
  branch=None,
  no_pull=False,
  ):
- self.module_name = module_name
+ super().__init__(component_type=component_type, dir=".", remote_url=remote_url, branch=branch, no_pull=no_pull)
+ self.component_name = component_name
  self.no_prompts = no_prompts
  self.pytest_args = pytest_args
 
- super().__init__(".", remote_url, branch, no_pull)
-
  def run(self):
  """Run test steps"""
  if not self.no_prompts:
@@ -58,27 +76,34 @@ def _check_inputs(self):
 
  # Retrieving installed modules
  if self.repo_type == "modules":
- installed_modules = self.get_modules_clone_modules()
+ installed_components = self.get_components_clone_modules()
  else:
  modules_json = ModulesJson(self.dir)
  modules_json.check_up_to_date()
- installed_modules = modules_json.get_all_modules().get(self.modules_repo.remote_url)
+ if self.component_type == "modules":
+ installed_components = modules_json.get_all_modules().get(self.modules_repo.remote_url)
+ elif self.component_type == "subworkflows":
+ modules_json.get_installed_subworkflows().get(self.modules_repo.remote_url)
 
- # Get the tool name if not specified
- if self.module_name is None:
+ # Get the component name if not specified
+ if self.component_name is None:
  if self.no_prompts:
  raise UserWarning(
- "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL."
+ f"{self.component_type[:-1].title()} name not provided and prompts deactivated. Please provide the {self.component_type[:-1]} name{' as TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}."
  )
- if not installed_modules:
+ if not installed_components:
+ if self.component_type == "modules":
+ dir_structure_message = f"modules/{self.modules_repo.repo_path}/TOOL/SUBTOOL/ and tests/modules/{self.modules_repo.repo_path}/TOOLS/SUBTOOL/"
+ elif self.component_type == "subworkflows":
+ dir_structure_message = f"subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/ and tests/subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/"
  raise UserWarning(
- f"No installed modules were found from '{self.modules_repo.remote_url}'.\n"
- f"Are you running the tests inside the nf-core/modules main directory?\n"
- f"Otherwise, make sure that the directory structure is modules/TOOL/SUBTOOL/ and tests/modules/TOOLS/SUBTOOL/"
+ f"No installed {self.component_type} were found from '{self.modules_repo.remote_url}'.\n"
+ f"Are you running the tests inside the repository root directory?\n"
+ f"Make sure that the directory structure is {dir_structure_message}"
  )
- self.module_name = questionary.autocomplete(
+ self.component_name = questionary.autocomplete(
  "Tool name:",
- choices=installed_modules,
+ choices=installed_components,
  style=nf_core.utils.nfcore_question_style,
  ).unsafe_ask()
 
@@ -89,19 +114,25 @@ def _validate_folder_structure(self):
  """Validate that the modules follow the correct folder structure to run the tests:
  - modules/nf-core/TOOL/SUBTOOL/
  - tests/modules/nf-core/TOOL/SUBTOOL/
-
+ or
+ - subworkflows/nf-core/SUBWORKFLOW/
+ - tests/subworkflows/nf-core/SUBWORKFLOW/
  """
- module_path = Path(self.default_modules_path) / self.module_name
- test_path = Path(self.default_tests_path) / self.module_name
-
- if not (self.dir / module_path).is_dir():
+ if self.component_type == "modules":
+ component_path = Path(self.default_modules_path) / self.component_name
+ test_path = Path(self.default_tests_path) / self.component_name
+ elif self.component_type == "subworkflows":
+ component_path = Path(self.default_subworkflows_path) / self.component_name
+ test_path = Path(self.default_subworkflows_tests_path) / self.component_name
+
+ if not (self.dir / component_path).is_dir():
  raise UserWarning(
- f"Cannot find directory '{module_path}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?"
+ f"Cannot find directory '{component_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. Are you running the tests inside the modules repository root directory?"
  )
  if not (self.dir / test_path).is_dir():
  raise UserWarning(
- f"Cannot find directory '{test_path}'. Should be TOOL/SUBTOOL or TOOL. "
- "Are you running the tests inside the nf-core/modules main directory? "
+ f"Cannot find directory '{test_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. "
+ "Are you running the tests inside the modules repository root directory? "
  "Do you have tests for the specified module?"
  )
 
@@ -144,15 +175,15 @@ def _check_profile(self):
  )
 
  def _run_pytests(self):
- """Given a module name, run tests."""
+ """Given a module/subworkflow name, run tests."""
  # Print nice divider line
  console = rich.console.Console()
- console.rule(self.module_name, style="black")
+ console.rule(self.component_name, style="black")
 
  # Set pytest arguments
- command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"]
+ command_args = ["--tag", f"{self.component_name}", "--symlink", "--keep-workflow-wd", "--git-aware"]
  command_args += self.pytest_args
 
  # Run pytest
- log.info(f"Running pytest for module '{self.module_name}'")
+ log.info(f"Running pytest for {self.component_type[:-1]} '{self.component_name}'")
  sys.exit(pytest.main(command_args))

nf_core/components/list.py

@@ -6,8 +6,6 @@
 import nf_core.modules.modules_utils
 from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson
-
-# from .modules_command import ModulesRepo
 from nf_core.modules.modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)

nf_core/modules/bump_versions.py

@@ -18,17 +18,16 @@
 
 import nf_core.modules.modules_utils
 import nf_core.utils
+from nf_core.components.components_command import ComponentCommand
 from nf_core.utils import plural_s as _s
 from nf_core.utils import rich_force_colors
 
-from .modules_command import ModuleCommand
-
 log = logging.getLogger(__name__)
 
 
-class ModuleVersionBumper(ModuleCommand):
+class ModuleVersionBumper(ComponentCommand):
  def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False):
- super().__init__(pipeline_dir, remote_url, branch, no_pull)
+ super().__init__("modules", pipeline_dir, remote_url, branch, no_pull)
 
  self.up_to_date = None
  self.updated = None

nf_core/modules/create.py

@@ -17,13 +17,12 @@
 import nf_core.components.components_create
 import nf_core.modules.modules_utils
 import nf_core.utils
-
-from .modules_command import ModuleCommand
+from nf_core.components.components_command import ComponentCommand
 
 log = logging.getLogger(__name__)
 
 
-class ModuleCreate(ModuleCommand):
+class ModuleCreate(ComponentCommand):
  def __init__(
  self,
  directory=".",
@@ -36,7 +35,7 @@ def __init__(
  conda_version=None,
  repo_type=None,
  ):
- super().__init__(directory)
+ super().__init__("modules", directory)
  self.directory = directory
  self.tool = tool
  self.author = author
@@ -157,7 +156,7 @@ def create(self):
  pytest_modules_yml = dict(sorted(pytest_modules_yml.items()))
  with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh:
  yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper())
- except FileNotFoundError as e:
+ except FileNotFoundError:
  raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!")
 
  new_files = list(self.file_paths.values())

nf_core/modules/info.py

@@ -11,16 +11,16 @@
 from rich.text import Text
 
 import nf_core.utils
+from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson
 
-from .modules_command import ModuleCommand
 from .modules_repo import NF_CORE_MODULES_REMOTE
 from .modules_utils import get_repo_type
 
 log = logging.getLogger(__name__)
 
 
-class ModuleInfo(ModuleCommand):
+class ModuleInfo(ComponentCommand):
  """
  Class to print information of a module.
 
@@ -56,7 +56,7 @@ class ModuleInfo(ModuleCommand):
  """
 
  def __init__(self, pipeline_dir, tool, remote_url, branch, no_pull):
- super().__init__(pipeline_dir, remote_url, branch, no_pull)
+ super().__init__("modules", pipeline_dir, remote_url, branch, no_pull)
  self.meta = None
  self.local_path = None
  self.remote_location = None
@@ -139,7 +139,6 @@ def get_local_yaml(self):
 
  if self.repo_type == "pipeline":
  # Try to find and load the meta.yml file
- repo_name = self.modules_repo.repo_path
  module_base_path = os.path.join(self.dir, "modules")
  # Check that we have any modules installed from this repo
  modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url)

nf_core/modules/install.py

@@ -4,14 +4,13 @@
 import nf_core.components.components_install
 import nf_core.modules.modules_utils
 import nf_core.utils
+from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson
 
-from .modules_command import ModuleCommand
-
 log = logging.getLogger(__name__)
 
 
-class ModuleInstall(ModuleCommand):
+class ModuleInstall(ComponentCommand):
  def __init__(
  self,
  pipeline_dir,
@@ -23,7 +22,7 @@ def __init__(
  no_pull=False,
  installed_by=False,
  ):
- super().__init__(pipeline_dir, remote_url, branch, no_pull)
+ super().__init__("modules", pipeline_dir, remote_url, branch, no_pull)
  self.force = force
  self.prompt = prompt
  self.sha = sha