refact: Use same org_path as modulesrepo first (#22)

* refact: Use same org_path as modulesrepo first

* refact: Change current_remote to a modulesrepo instance

* refact: Try using modulesrepo in get_comps

* refact: Try using utility functions to grab the yml

* fix: Restore old regex behaviour

* refact: Use parent path instead of regex in comp_utils

* refact: Use modulesrepo for current_repo in install

* refact: Move constants to new file and use MRepo

Moves constants from component_utils to constants.py
so that ModulesRepo can be used from inside of component_utils,
thereby avoiding a circular import

* Revert "refact: Use modulesrepo for current_repo in install"

This reverts commit f4cc0fe.

* Reapply "refact: Use modulesrepo for current_repo in install"

This reverts commit c5bebcf.

* Revert "refact: Move constants to new file and use MRepo"

This reverts commit c0a7a00.

* refact: Change constants to diff file, use MRepo

* fix: Change import in test_update

nf_core/__main__.py

@@ -54,7 +54,7 @@
     subworkflows_test,
     subworkflows_update,
 )
-from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE
+from nf_core.components.constants import NF_CORE_MODULES_REMOTE
 from nf_core.pipelines.download import DownloadError
 from nf_core.utils import check_if_outdated, nfcore_logo, rich_force_colors, setup_nfcore_dir
 

nf_core/components/components_utils.py

@@ -1,25 +1,18 @@
 import logging
 import re
 from pathlib import Path
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import questionary
 import requests
 import rich.prompt
 import yaml
 
-if TYPE_CHECKING:
-    from nf_core.modules.modules_repo import ModulesRepo
-
 import nf_core.utils
+from nf_core.modules.modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)
 
-# Constants for the nf-core/modules repo used throughout the module files
-NF_CORE_MODULES_NAME = "nf-core"
-NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git"
-NF_CORE_MODULES_DEFAULT_BRANCH = "master"
-
 
 def get_repo_info(directory: Path, use_prompt: Optional[bool] = True) -> Tuple[Path, Optional[str], str]:
     """
@@ -181,20 +174,15 @@ def get_components_to_install(
                 for component in components:
                     if isinstance(component, dict):
                         component_name = list(component.keys())[0].lower()
+                        branch = component[component_name].get("branch")
                         git_remote = component[component_name]["git_remote"]
-                        org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", git_remote)
-                        if org_path_match:
-                            org_path = org_path_match.group(1)
-                        else:
-                            raise UserWarning(
-                                f"The organisation path of {component_name} could not be established from '{git_remote}'"
-                            )
+                        modules_repo = ModulesRepo(git_remote, branch=branch)
                         current_comp_dict = subworkflows if component_name in subworkflows else modules
 
                         component_dict = {
-                            "org_path": org_path,
+                            "org_path": modules_repo.repo_path,
                             "git_remote": git_remote,
-                            "branch": component[component_name].get("branch"),
+                            "branch": branch,
                         }
 
                         current_comp_dict[component_name].update(component_dict)

nf_core/components/constants.py

@@ -0,0 +1,4 @@
+# Constants for the nf-core/modules repo used throughout the module files
+NF_CORE_MODULES_NAME = "nf-core"
+NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git"
+NF_CORE_MODULES_DEFAULT_BRANCH = "master"

nf_core/components/info.py

@@ -15,7 +15,7 @@
 
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
-from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE
+from nf_core.components.constants import NF_CORE_MODULES_REMOTE
 from nf_core.modules.modules_json import ModulesJson
 
 log = logging.getLogger(__name__)

nf_core/components/install.py

@@ -15,10 +15,12 @@
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
 from nf_core.components.components_utils import (
-    NF_CORE_MODULES_NAME,
     get_components_to_install,
     prompt_component_version_sha,
 )
+from nf_core.components.constants import (
+    NF_CORE_MODULES_NAME,
+)
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
 
@@ -39,7 +41,7 @@ def __init__(
         installed_by: Optional[List[str]] = None,
     ):
         super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
-        self.current_remote = remote_url
+        self.current_remote = ModulesRepo(remote_url, branch)
         self.branch = branch
         self.force = force
         self.prompt = prompt
@@ -53,15 +55,15 @@ def __init__(
     def install(self, component: Union[str, Dict[str, str]], silent: bool = False) -> bool:
         if isinstance(component, dict):
             # Override modules_repo when the component to install is a dependency from a subworkflow.
-            remote_url = component.get("git_remote", self.current_remote)
+            remote_url = component.get("git_remote", self.current_remote.remote_url)
             branch = component.get("branch", self.branch)
             self.modules_repo = ModulesRepo(remote_url, branch)
             component = component["name"]
 
         if self.current_remote is None:
-            self.current_remote = self.modules_repo.remote_url
+            self.current_remote = self.modules_repo
 
-        if self.current_remote == self.modules_repo.remote_url and self.sha is not None:
+        if self.current_remote.remote_url == self.modules_repo.remote_url and self.sha is not None:
             self.current_sha = self.sha
         else:
             self.current_sha = None
@@ -244,7 +246,7 @@ def collect_and_verify_name(
 
             raise ValueError
 
-        if self.current_remote == modules_repo.remote_url:
+        if self.current_remote.remote_url == modules_repo.remote_url:
             if not modules_repo.component_exists(component, self.component_type, commit=self.current_sha):
                 warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})"
                 log.warning(warn_msg)

nf_core/components/update.py

@@ -1,6 +1,5 @@
 import logging
 import os
-import re
 import shutil
 import tempfile
 from pathlib import Path
@@ -42,7 +41,7 @@ def __init__(
         limit_output=False,
     ):
         super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
-        self.current_remote = remote_url
+        self.current_remote = ModulesRepo(remote_url, branch)
         self.branch = branch
         self.force = force
         self.prompt = prompt
@@ -97,7 +96,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
         """
         if isinstance(component, dict):
             # Override modules_repo when the component to install is a dependency from a subworkflow.
-            remote_url = component.get("git_remote", self.current_remote)
+            remote_url = component.get("git_remote", self.current_remote.remote_url)
             branch = component.get("branch", self.branch)
             self.modules_repo = ModulesRepo(remote_url, branch)
             component = component["name"]
@@ -892,7 +891,7 @@ def get_components_to_update(self, component):
         if self.component_type == "modules":
             # All subworkflow names in the installed_by section of a module are subworkflows using this module
             # We need to update them too
-            git_remote = self.current_remote
+            git_remote = self.current_remote.remote_url
             for subworkflow in installed_by:
                 if subworkflow != component:
                     for remote_url, content in mods_json["repos"].items():
@@ -965,9 +964,7 @@ def update_linked_components(
     def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update):
         """Check for linked components added or removed in the new subworkflow version"""
         if self.component_type == "subworkflows":
-            org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", self.current_remote)
-            if org_path_match:
-                org_path = org_path_match.group(1)
+            org_path = self.current_remote.repo_path
 
             subworkflow_directory = Path(self.directory, self.component_type, org_path, component)
             included_modules, included_subworkflows = get_components_to_install(subworkflow_directory)
@@ -991,15 +988,15 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw
             # If a new module/subworkflow is included in the subworklfow and wasn't included before
             for module in included_modules:
                 module_name = module["name"]
-                module["git_remote"] = module.get("git_remote", self.current_remote)
+                module["git_remote"] = module.get("git_remote", self.current_remote.remote_url)
                 module["branch"] = module.get("branch", self.branch)
                 if module_name not in modules_to_update:
                     log.info(f"Installing newly included module '{module_name}' for '{component}'")
                     install_module_object = ComponentInstall(self.directory, "modules", installed_by=component)
                     install_module_object.install(module, silent=True)
             for subworkflow in included_subworkflows:
                 subworkflow_name = subworkflow["name"]
-                subworkflow["git_remote"] = subworkflow.get("git_remote", self.current_remote)
+                subworkflow["git_remote"] = subworkflow.get("git_remote", self.current_remote.remote_url)
                 subworkflow["branch"] = subworkflow.get("branch", self.branch)
                 if subworkflow_name not in subworkflows_to_update:
                     log.info(f"Installing newly included subworkflow '{subworkflow_name}' for '{component}'")
@@ -1022,4 +1019,4 @@ def _reset_component_type(self, original_component_type, original_update_all):
         self.modules_json.pipeline_components = None
         self.update_all = original_update_all
         if self.current_remote is None:
-            self.current_remote = self.modules_repo.remote_url
+            self.current_remote = self.modules_repo

nf_core/modules/modules_json.py

@@ -15,7 +15,8 @@
 from typing_extensions import NotRequired, TypedDict  # for py<3.11
 
 import nf_core.utils
-from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE, get_components_to_install
+from nf_core.components.components_utils import get_components_to_install
+from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
 from nf_core.modules.modules_repo import ModulesRepo
 from nf_core.pipelines.lint_utils import dump_json_with_prettier
 

nf_core/modules/modules_repo.py

@@ -10,9 +10,8 @@
 import rich.prompt
 from git.exc import GitCommandError, InvalidGitRepositoryError
 
-import nf_core.modules.modules_json
 import nf_core.modules.modules_utils
-from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
+from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
 from nf_core.synced_repo import RemoteProgressbar, SyncedRepo
 from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, load_tools_config
 

nf_core/synced_repo.py

@@ -9,7 +9,7 @@
 import git
 from git.exc import GitCommandError
 
-from nf_core.components.components_utils import (
+from nf_core.components.constants import (
     NF_CORE_MODULES_NAME,
     NF_CORE_MODULES_REMOTE,
 )

tests/modules/test_modules_json.py

@@ -3,7 +3,7 @@
 import shutil
 from pathlib import Path
 
-from nf_core.components.components_utils import (
+from nf_core.components.constants import (
     NF_CORE_MODULES_DEFAULT_BRANCH,
     NF_CORE_MODULES_NAME,
     NF_CORE_MODULES_REMOTE,

tests/modules/test_update.py

@@ -8,7 +8,7 @@
 import yaml
 
 import nf_core.utils
-from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
+from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
 from nf_core.modules.install import ModuleInstall
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.patch import ModulePatch

tests/subworkflows/test_update.py

@@ -8,7 +8,7 @@
 import yaml
 
 import nf_core.utils
-from nf_core.components.components_utils import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
+from nf_core.components.constants import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.update import ModuleUpdate
 from nf_core.subworkflows.install import SubworkflowInstall