Modules restructure

.github/workflows/create-lint-wf.yml

@@ -102,7 +102,7 @@ jobs:
         run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force
 
       - name: nf-core modules install gitlab
-        run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git install fastqc --dir nf-core-testpipeline/
+        run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main-restructure install fastqc --force --dir nf-core-testpipeline/
 
       - name: nf-core modules list local
         run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/

CHANGELOG.md

@@ -27,6 +27,7 @@
 - Use contextlib to temporarily change working directories ([#1819](https://github.com/nf-core/tools/pull/1819))
 - More helpful error messages if `nf-core download` can't parse a singularity image download
 - Modules: If something is wrong with the local repo cache, offer to delete it and try again ([#1850](https://github.com/nf-core/tools/issues/1850))
+- Restructure code to work with the directory restructuring in [modules](https://github.com/nf-core/modules/pull/2141) ([#1859](https://github.com/nf-core/tools/pull/1859))
 
 ### Modules
 

nf_core/__main__.py

@@ -661,7 +661,13 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts):
     the required `test.yml` file based on the output files.
     """
     try:
-        meta_builder = nf_core.modules.ModulesTestYmlBuilder(tool, run_tests, output, force, no_prompts)
+        meta_builder = nf_core.modules.ModulesTestYmlBuilder(
+            module_name=tool,
+            run_tests=run_tests,
+            test_yml_output_path=output,
+            force_overwrite=force,
+            no_prompts=no_prompts,
+        )
         meta_builder.run()
     except (UserWarning, LookupError) as e:
         log.critical(e)

nf_core/lint/modules_json.py

@@ -28,24 +28,25 @@ def modules_json(self):
 
         for repo in modules_json_dict["repos"].keys():
             # Check if the modules.json has been updated to keep the
-            if "modules" not in modules_json_dict["repos"][repo] or "git_url" not in modules_json_dict["repos"][repo]:
+            if "modules" not in modules_json_dict["repos"][repo] or not repo.startswith("http"):
                 failed.append(
                     "Your `modules.json` file is outdated. "
-                    "Please remove it and reinstall it by running any module command."
+                    "It will be automatically generated by running any module command."
                 )
                 continue
 
-            for module, module_entry in modules_json_dict["repos"][repo]["modules"].items():
-                if not Path(modules_dir, repo, module).exists():
-                    failed.append(
-                        f"Entry for `{Path(repo, module)}` found in `modules.json` but module is not installed in "
-                        "pipeline."
-                    )
-                    all_modules_passed = False
-                if module_entry.get("branch") is None:
-                    failed.append(f"Entry for `{Path(repo, module)}` is missing branch information.")
-                if module_entry.get("git_sha") is None:
-                    failed.append(f"Entry for `{Path(repo, module)}` is missing version information.")
+            for dir in modules_json_dict["repos"][repo]["modules"].keys():
+                for module, module_entry in modules_json_dict["repos"][repo]["modules"][dir].items():
+                    if not Path(modules_dir, dir, module).exists():
+                        failed.append(
+                            f"Entry for `{Path(modules_dir, dir, module)}` found in `modules.json` but module is not installed in "
+                            "pipeline."
+                        )
+                        all_modules_passed = False
+                    if module_entry.get("branch") is None:
+                        failed.append(f"Entry for `{Path(modules_dir, dir, module)}` is missing branch information.")
+                    if module_entry.get("git_sha") is None:
+                        failed.append(f"Entry for `{Path(modules_dir, dir, module)}` is missing version information.")
         if all_modules_passed:
             passed.append("Only installed modules found in `modules.json`")
     else:

nf_core/module-template/tests/main.nf

@@ -2,7 +2,7 @@
 
 nextflow.enable.dsl = 2
 
-include { {{ tool_name_underscore|upper }} } from '../../../{{ "../" if subtool else "" }}modules/{{ tool_dir }}/main.nf'
+include { {{ tool_name_underscore|upper }} } from '../../../../{{ "../" if subtool else "" }}modules/nf-core/{{ tool_dir }}/main.nf'
 
 workflow test_{{ tool_name_underscore }} {
     {% if has_meta %}

nf_core/modules/bump_versions.py

@@ -8,6 +8,7 @@
 
 import logging
 import re
+from pathlib import Path
 
 import questionary
 import rich

nf_core/modules/create.py

@@ -22,10 +22,12 @@
 import nf_core.modules.module_utils
 import nf_core.utils
 
+from .modules_command import ModuleCommand
+
 log = logging.getLogger(__name__)
 
 
-class ModuleCreate(object):
+class ModuleCreate(ModuleCommand):
     def __init__(
         self,
         directory=".",
@@ -38,6 +40,7 @@ def __init__(
         conda_version=None,
         repo_type=None,
     ):
+        super().__init__(directory)
         self.directory = directory
         self.tool = tool
         self.author = author
@@ -72,10 +75,10 @@ def create(self):
 
         If <directory> is a clone of nf-core/modules, it creates or modifies the following files:
 
-        modules/modules/tool/subtool/
+        modules/modules/nf-core/tool/subtool/
             * main.nf
             * meta.yml
-        modules/tests/modules/tool/subtool/
+        modules/tests/modules/nf-core/tool/subtool/
             * main.nf
             * test.yml
             * nextflow.config
@@ -250,13 +253,13 @@ def create(self):
                     pytest_modules_yml = yaml.safe_load(fh)
                 if self.subtool:
                     pytest_modules_yml[self.tool_name] = [
-                        f"modules/{self.tool}/{self.subtool}/**",
-                        f"tests/modules/{self.tool}/{self.subtool}/**",
+                        f"modules/nf-core/{self.tool}/{self.subtool}/**",
+                        f"tests/modules/nf-core/{self.tool}/{self.subtool}/**",
                     ]
                 else:
                     pytest_modules_yml[self.tool_name] = [
-                        f"modules/{self.tool}/**",
-                        f"tests/modules/{self.tool}/**",
+                        f"modules/nf-core/{self.tool}/**",
+                        f"tests/modules/nf-core/{self.tool}/**",
                     ]
                 pytest_modules_yml = dict(sorted(pytest_modules_yml.items()))
                 with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh:
@@ -323,8 +326,8 @@ def get_module_dirs(self):
             file_paths[os.path.join("modules", "main.nf")] = module_file
 
         if self.repo_type == "modules":
-            software_dir = os.path.join(self.directory, "modules", self.tool_dir)
-            test_dir = os.path.join(self.directory, "tests", "modules", self.tool_dir)
+            software_dir = os.path.join(self.directory, self.default_modules_path, self.tool_dir)
+            test_dir = os.path.join(self.directory, self.default_tests_path, self.tool_dir)
 
             # Check if module directories exist already
             if os.path.exists(software_dir) and not self.force_overwrite:
@@ -334,8 +337,8 @@ def get_module_dirs(self):
                 raise UserWarning(f"Module test directory exists: '{test_dir}'. Use '--force' to overwrite")
 
             # If a subtool, check if there is a module called the base tool name already
-            parent_tool_main_nf = os.path.join(self.directory, "modules", self.tool, "main.nf")
-            parent_tool_test_nf = os.path.join(self.directory, "tests", "modules", self.tool, "main.nf")
+            parent_tool_main_nf = os.path.join(self.directory, self.default_modules_path, self.tool, "main.nf")
+            parent_tool_test_nf = os.path.join(self.directory, self.default_tests_path, self.tool, "main.nf")
             if self.subtool and os.path.exists(parent_tool_main_nf):
                 raise UserWarning(
                     f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.tool_name}'"
@@ -346,7 +349,7 @@ def get_module_dirs(self):
                 )
 
             # If no subtool, check that there isn't already a tool/subtool
-            tool_glob = glob.glob(f"{os.path.join(self.directory, 'modules', self.tool)}/*/main.nf")
+            tool_glob = glob.glob(f"{os.path.join(self.directory, self.default_modules_path, self.tool)}/*/main.nf")
             if not self.subtool and tool_glob:
                 raise UserWarning(
                     f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.tool_name}'"

nf_core/modules/info.py

@@ -93,7 +93,8 @@ def init_mod_name(self, module):
                 if self.repo_type == "modules":
                     modules = self.get_modules_clone_modules()
                 else:
-                    modules = self.modules_json.get_all_modules().get(self.modules_repo.fullname)
+                    modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url)
+                    modules = [module if dir == "nf-core" else f"{dir}/{module}" for dir, module in modules]
                     if modules is None:
                         raise UserWarning(f"No modules installed from '{self.modules_repo.remote_url}'")
             else:
@@ -135,15 +136,17 @@ def get_local_yaml(self):
 
         if self.repo_type == "pipeline":
             # Try to find and load the meta.yml file
-            repo_name = self.modules_repo.fullname
-            module_base_path = os.path.join(self.dir, "modules", repo_name)
+            repo_name = self.modules_repo.repo_path
+            module_base_path = os.path.join(self.dir, "modules")
             # Check that we have any modules installed from this repo
-            modules = self.modules_json.get_all_modules().get(repo_name)
+            modules = self.modules_json.get_all_modules().get(self.modules_repo.remote_url)
+            module_names = [module for _, module in modules]
             if modules is None:
                 raise LookupError(f"No modules installed from {self.modules_repo.remote_url}")
 
-            if self.module in modules:
-                mod_dir = os.path.join(module_base_path, self.module)
+            if self.module in module_names:
+                install_dir = [dir for dir, module in modules if module == self.module][0]
+                mod_dir = os.path.join(module_base_path, install_dir, self.module)
                 meta_fn = os.path.join(mod_dir, "meta.yml")
                 if os.path.exists(meta_fn):
                     log.debug(f"Found local file: {meta_fn}")
@@ -153,7 +156,7 @@ def get_local_yaml(self):
 
             log.debug(f"Module '{self.module}' meta.yml not found locally")
         else:
-            module_base_path = os.path.join(self.dir, "modules")
+            module_base_path = os.path.join(self.dir, "modules", "nf-core")
             if self.module in os.listdir(module_base_path):
                 mod_dir = os.path.join(module_base_path, self.module)
                 meta_fn = os.path.join(mod_dir, "meta.yml")

nf_core/modules/install.py

@@ -48,7 +48,7 @@ def install(self, module):
         # Verify that the provided SHA exists in the repo
         if self.sha:
             if not self.modules_repo.sha_exists_on_branch(self.sha):
-                log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.fullname}'")
+                log.error(f"Commit SHA '{self.sha}' doesn't exist in '{self.modules_repo.remote_url}'")
                 return False
 
         if module is None:
@@ -71,10 +71,12 @@ def install(self, module):
             log.warning(warn_msg)
             return False
 
-        current_version = modules_json.get_module_version(module, self.modules_repo.fullname)
+        current_version = modules_json.get_module_version(
+            module, self.modules_repo.remote_url, self.modules_repo.repo_path
+        )
 
         # Set the install folder based on the repository name
-        install_folder = os.path.join(self.dir, "modules", self.modules_repo.fullname)
+        install_folder = os.path.join(self.dir, "modules", self.modules_repo.repo_path)
 
         # Compute the module directory
         module_dir = os.path.join(install_folder, module)
@@ -84,7 +86,7 @@ def install(self, module):
 
             log.error("Module is already installed.")
             repo_flag = (
-                "" if self.modules_repo.fullname == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} "
+                "" if self.modules_repo.repo_path == NF_CORE_MODULES_NAME else f"-g {self.modules_repo.remote_url} "
             )
             branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} "
 
@@ -110,8 +112,18 @@ def install(self, module):
             version = self.modules_repo.get_latest_module_version(module)
 
         if self.force:
-            log.info(f"Removing installed version of '{self.modules_repo.fullname}/{module}'")
+            log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'")
             self.clear_module_dir(module, module_dir)
+            for repo_url, repo_content in modules_json.modules_json["repos"].items():
+                for dir, dir_modules in repo_content["modules"].items():
+                    for name, _ in dir_modules.items():
+                        if name == module and dir == self.modules_repo.repo_path:
+                            repo_to_remove = repo_url
+                            log.info(
+                                f"Removing module '{self.modules_repo.repo_path}/{module}' from repo '{repo_to_remove}' from modules.json"
+                            )
+                            modules_json.remove_entry(module, repo_to_remove, self.modules_repo.repo_path)
+                            break
 
         log.info(f"{'Rei' if self.force else 'I'}nstalling '{module}'")
         log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.remote_url}")

nf_core/modules/lint/__init__.py

@@ -85,12 +85,12 @@ def __init__(
             modules_json = ModulesJson(self.dir)
             modules_json.check_up_to_date()
             all_pipeline_modules = modules_json.get_all_modules()
-            if self.modules_repo.fullname in all_pipeline_modules:
-                module_dir = Path(self.dir, "modules", self.modules_repo.fullname)
+            if self.modules_repo.remote_url in all_pipeline_modules:
+                module_dir = Path(self.dir, "modules", "nf-core")
                 self.all_remote_modules = [
-                    NFCoreModule(m, self.modules_repo.fullname, module_dir / m, self.repo_type, Path(self.dir))
-                    for m in all_pipeline_modules[self.modules_repo.fullname]
-                ]
+                    NFCoreModule(m[1], self.modules_repo.remote_url, module_dir / m[1], self.repo_type, Path(self.dir))
+                    for m in all_pipeline_modules[self.modules_repo.remote_url]
+                ]  # m = (module_dir, module_name)
                 if not self.all_remote_modules:
                     raise LookupError(f"No modules from {self.modules_repo.remote_url} installed in pipeline.")
                 local_module_dir = Path(self.dir, "modules", "local")
@@ -102,7 +102,7 @@ def __init__(
             else:
                 raise LookupError(f"No modules from {self.modules_repo.remote_url} installed in pipeline.")
         else:
-            module_dir = Path(self.dir, "modules")
+            module_dir = Path(self.dir, self.default_modules_path)
             self.all_remote_modules = [
                 NFCoreModule(m, None, module_dir / m, self.repo_type, Path(self.dir))
                 for m in self.get_modules_clone_modules()
@@ -144,7 +144,7 @@ def lint(
         Lint all or one specific module
 
         First gets a list of all local modules (in modules/local/process) and all modules
-        installed from nf-core (in modules/nf-core/modules)
+        installed from nf-core (in modules/nf-core)
 
         For all nf-core modules, the correct file structure is assured and important
         file content is verified. If directory subject to linting is a clone of 'nf-core/modules',

nf_core/modules/lint/main_nf.py

@@ -46,7 +46,7 @@ def main_nf(module_lint_object, module, fix_version, progress_bar):
     if module.is_patched:
         lines = ModulesDiffer.try_apply_patch(
             module.module_name,
-            module_lint_object.modules_repo.fullname,
+            module_lint_object.modules_repo.repo_path,
             module.patch_path,
             Path(module.module_dir).relative_to(module.base_dir),
             reverse=True,

nf_core/modules/lint/meta_yml.py

@@ -28,7 +28,7 @@ def meta_yml(module_lint_object, module):
     if module.is_patched:
         lines = ModulesDiffer.try_apply_patch(
             module.module_name,
-            module_lint_object.modules_repo.fullname,
+            module_lint_object.modules_repo.repo_path,
             module.patch_path,
             Path(module.module_dir).relative_to(module.base_dir),
             reverse=True,

nf_core/modules/lint/module_changes.py

@@ -29,7 +29,7 @@ def module_changes(module_lint_object, module):
         shutil.copytree(module.module_dir, tempdir)
         try:
             new_lines = ModulesDiffer.try_apply_patch(
-                module.module_name, module_lint_object.modules_repo.fullname, module.patch_path, tempdir, reverse=True
+                module.module_name, module_lint_object.modules_repo.repo_path, module.patch_path, tempdir, reverse=True
             )
             for file, lines in new_lines.items():
                 with open(tempdir / file, "w") as fh:

nf_core/modules/lint/module_patch.py

@@ -163,7 +163,7 @@ def patch_reversible(module_lint_object, module, patch_path):
     try:
         ModulesDiffer.try_apply_patch(
             module.module_name,
-            module_lint_object.modules_repo.fullname,
+            module_lint_object.modules_repo.repo_path,
             patch_path,
             Path(module.module_dir).relative_to(module.base_dir),
             reverse=True,

nf_core/modules/lint/module_version.py

@@ -26,7 +26,7 @@ def module_version(module_lint_object, module):
 
     # Verify that a git_sha exists in the `modules.json` file for this module
     version = module_lint_object.modules_json.get_module_version(
-        module.module_name, module_lint_object.modules_repo.fullname
+        module.module_name, module_lint_object.modules_repo.remote_url, module_lint_object.modules_repo.repo_path
     )
     if version is None:
         module.failed.append(("git_sha", "No git_sha entry in `modules.json`", modules_json_path))