Merge pull request #1999 from mirpedrol/installtrack

Track from where modules and subworkflows are installed

CHANGELOG.md

@@ -26,6 +26,7 @@
 - Bump promoted Python version from 3.7 to 3.8 ([#1971](https://github.com/nf-core/tools/pull/1971))
 - Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out`
 - Updated GitHub actions ([#1998](https://github.com/nf-core/tools/pull/1998), [#2001](https://github.com/nf-core/tools/pull/2001))
+- Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999))
 
 ### Modules
 

docs/api/make_lint_md.py

@@ -21,7 +21,6 @@ def make_docs(docs_basedir, lint_tests, md_template):
         else:
             with open(fn, "w") as fh:
                 fh.write(md_template.format(test_name))
-                print(test_name)
 
     for fn in existing_docs:
         os.remove(fn)

nf_core/components/components_create.py

@@ -99,7 +99,6 @@ def get_component_dirs(component_type, repo_type, directory, name, supername, su
         # Check whether component file already exists
         component_file = os.path.join(local_component_dir, f"{name}.nf")
         if os.path.exists(component_file) and not force_overwrite:
-            print(f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite")
             raise UserWarning(
                 f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite"
             )

nf_core/components/components_install.py

@@ -1,12 +1,7 @@
-import glob
-import json
 import logging
 import os
-import re
 
-import jinja2
 import questionary
-import rich
 
 import nf_core.modules.modules_utils
 import nf_core.utils
@@ -77,7 +72,7 @@ def get_version(component, component_type, sha, prompt, current_version, modules
         version = sha
     elif prompt:
         try:
-            version = nf_core.modules.modules_utils.prompt_component_version_sha(
+            version = prompt_component_version_sha(
                 component,
                 component_type,
                 installed_sha=current_version,
@@ -98,11 +93,11 @@ def clean_modules_json(component, component_type, modules_repo, modules_json):
     """
     for repo_url, repo_content in modules_json.modules_json["repos"].items():
         for dir, dir_components in repo_content[component_type].items():
-            for name, _ in dir_components.items():
+            for name, component_values in dir_components.items():
                 if name == component and dir == modules_repo.repo_path:
                     repo_to_remove = repo_url
                     log.info(
                         f"Removing {component_type[:-1]} '{modules_repo.repo_path}/{component}' from repo '{repo_to_remove}' from modules.json"
                     )
-                    modules_json.remove_entry(component, repo_to_remove, modules_repo.repo_path)
-                    break
+                    modules_json.remove_entry(component_type, component, repo_to_remove, modules_repo.repo_path)
+                    return component_values["installed_by"]

nf_core/modules/install.py

@@ -1,15 +1,12 @@
 import logging
 import os
 
-import questionary
-
 import nf_core.components.components_install
 import nf_core.modules.modules_utils
 import nf_core.utils
 from nf_core.modules.modules_json import ModulesJson
 
 from .modules_command import ModuleCommand
-from .modules_repo import NF_CORE_MODULES_NAME
 
 log = logging.getLogger(__name__)
 
@@ -24,11 +21,16 @@ def __init__(
         remote_url=None,
         branch=None,
         no_pull=False,
+        installed_by=False,
     ):
         super().__init__(pipeline_dir, remote_url, branch, no_pull)
         self.force = force
         self.prompt = prompt
         self.sha = sha
+        if installed_by:
+            self.installed_by = installed_by
+        else:
+            self.installed_by = self.component_type
 
     def install(self, module, silent=False):
         if self.repo_type == "modules":
@@ -71,6 +73,11 @@ def install(self, module, silent=False):
         if not nf_core.components.components_install.check_component_installed(
             self.component_type, module, current_version, module_dir, self.modules_repo, self.force, self.prompt
         ):
+            log.debug(
+                f"Module is already installed and force is not set.\nAdding the new installation source {self.installed_by} for module {module} to 'modules.json' without installing the module."
+            )
+            modules_json.load()
+            modules_json.update(self.modules_repo, module, current_version, self.installed_by)
             return False
 
         version = nf_core.components.components_install.get_version(
@@ -80,10 +87,11 @@ def install(self, module, silent=False):
             return False
 
         # Remove module if force is set
+        install_track = None
         if self.force:
             log.info(f"Removing installed version of '{self.modules_repo.repo_path}/{module}'")
             self.clear_component_dir(module, module_dir)
-            nf_core.components.components_install.clean_modules_json(
+            install_track = nf_core.components.components_install.clean_modules_json(
                 module, self.component_type, self.modules_repo, modules_json
             )
 
@@ -103,5 +111,5 @@ def install(self, module, silent=False):
 
         # Update module.json with newly installed module
         modules_json.load()
-        modules_json.update(self.modules_repo, module, version)
+        modules_json.update(self.modules_repo, module, version, self.installed_by, install_track)
         return True

nf_core/modules/modules_json.py

@@ -274,7 +274,11 @@ def determine_module_branches_and_shas(self, install_dir, remote_url, modules):
                     found_sha = True
                     break
             if found_sha:
-                repo_entry[module] = {"branch": modules_repo.branch, "git_sha": correct_commit_sha}
+                repo_entry[module] = {
+                    "branch": modules_repo.branch,
+                    "git_sha": correct_commit_sha,
+                    "installed_by": "modules",
+                }
 
         # Clean up the modules we were unable to find the sha for
         for module in sb_local:
@@ -373,25 +377,32 @@ def parse_dirs(self, dirs, missing_installation, component_type):
             component_in_file = False
             git_url = None
             for repo in missing_installation:
-                for dir_name in missing_installation[repo][component_type]:
-                    if component in missing_installation[repo][component_type][dir_name]:
-                        component_in_file = True
-                        git_url = repo
-                        break
+                if component_type in missing_installation[repo]:
+                    for dir_name in missing_installation[repo][component_type]:
+                        if component in missing_installation[repo][component_type][dir_name]:
+                            component_in_file = True
+                            git_url = repo
+                            break
             if not component_in_file:
-                # If it is not, add it to the list of missing subworkflow
+                # If it is not, add it to the list of missing components
                 untracked_dirs.append(component)
 
             else:
-                # If it does, remove the subworkflow from missing_installation
+                # If it does, remove the component from missing_installation
                 module_repo = missing_installation[git_url]
                 # Check if the entry has a git sha and branch before removing
                 components_dict = module_repo[component_type][install_dir]
                 if "git_sha" not in components_dict[component] or "branch" not in components_dict[component]:
                     self.determine_module_branches_and_shas(component, git_url, module_repo["base_path"], [component])
-                # Remove the subworkflow from subworkflows without installation
+                # Remove the component from components without installation
                 module_repo[component_type][install_dir].pop(component)
                 if len(module_repo[component_type][install_dir]) == 0:
+                    # If no modules/subworkflows with missing installation left, remove the install_dir from missing_installation
+                    missing_installation[git_url][component_type].pop(install_dir)
+                if len(module_repo[component_type]) == 0:
+                    # If no modules/subworkflows with missing installation left, remove the component_type from missing_installation
+                    missing_installation[git_url].pop(component_type)
+                if len(module_repo) == 0:
                     # If no modules/subworkflows with missing installation left, remove the git_url from missing_installation
                     missing_installation.pop(git_url)
 
@@ -469,6 +480,9 @@ def check_up_to_date(self):
 
         If a module/subworkflow is installed but the entry in 'modules.json' is missing we iterate through
         the commit log in the remote to try to determine the SHA.
+
+        Check that we have the "installed_by" value in 'modules.json', otherwise add it.
+        Assume that the modules/subworkflows were installed by an nf-core command (don't track installed by subworkflows).
         """
         try:
             self.load()
@@ -503,6 +517,16 @@ def check_up_to_date(self):
         if len(subworkflows_missing_from_modules_json) > 0:
             self.resolve_missing_from_modules_json(subworkflows_missing_from_modules_json, "subworkflows")
 
+        # If the "installed_by" value is not present for modules/subworkflows, add it.
+        for repo, repo_content in self.modules_json["repos"].items():
+            for component_type, dir_content in repo_content.items():
+                for install_dir, installed_components in dir_content.items():
+                    for component, component_features in installed_components.items():
+                        if "installed_by" not in component_features:
+                            self.modules_json["repos"][repo][component_type][install_dir][component]["installed_by"] = [
+                                component_type
+                            ]
+
         self.dump()
 
     def load(self):
@@ -521,16 +545,20 @@ def load(self):
         except FileNotFoundError:
             raise UserWarning("File 'modules.json' is missing")
 
-    def update(self, modules_repo, module_name, module_version, write_file=True):
+    def update(self, modules_repo, module_name, module_version, installed_by, installed_by_log=None, write_file=True):
         """
         Updates the 'module.json' file with new module info
 
         Args:
             modules_repo (ModulesRepo): A ModulesRepo object configured for the new module
             module_name (str): Name of new module
             module_version (str): git SHA for the new module entry
+            installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json'
             write_file (bool): whether to write the updated modules.json to a file.
         """
+        if installed_by_log is None:
+            installed_by_log = []
+
         if self.modules_json is None:
             self.load()
         repo_name = modules_repo.repo_path
@@ -543,22 +571,35 @@ def update(self, modules_repo, module_name, module_version, write_file=True):
             repo_modules_entry[module_name] = {}
         repo_modules_entry[module_name]["git_sha"] = module_version
         repo_modules_entry[module_name]["branch"] = branch
+        try:
+            if installed_by not in repo_modules_entry[module_name]["installed_by"]:
+                repo_modules_entry[module_name]["installed_by"].append(installed_by)
+        except KeyError:
+            repo_modules_entry[module_name]["installed_by"] = [installed_by]
+        finally:
+            repo_modules_entry[module_name]["installed_by"].extend(installed_by_log)
 
         # Sort the 'modules.json' repo entries
         self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"])
         if write_file:
             self.dump()
 
-    def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version, write_file=True):
+    def update_subworkflow(
+        self, modules_repo, subworkflow_name, subworkflow_version, installed_by, installed_by_log=None, write_file=True
+    ):
         """
         Updates the 'module.json' file with new subworkflow info
 
         Args:
             modules_repo (ModulesRepo): A ModulesRepo object configured for the new subworkflow
             subworkflow_name (str): Name of new subworkflow
             subworkflow_version (str): git SHA for the new subworkflow entry
+            installed_by_log (list): previous tracing of installed_by that needs to be added to 'modules.json'
             write_file (bool): whether to write the updated modules.json to a file.
         """
+        if installed_by_log is None:
+            installed_by_log = []
+
         if self.modules_json is None:
             self.load()
         repo_name = modules_repo.repo_path
@@ -574,18 +615,26 @@ def update_subworkflow(self, modules_repo, subworkflow_name, subworkflow_version
             repo_subworkflows_entry[subworkflow_name] = {}
         repo_subworkflows_entry[subworkflow_name]["git_sha"] = subworkflow_version
         repo_subworkflows_entry[subworkflow_name]["branch"] = branch
+        try:
+            if installed_by not in repo_subworkflows_entry[subworkflow_name]["installed_by"]:
+                repo_subworkflows_entry[subworkflow_name]["installed_by"].append(installed_by)
+        except KeyError:
+            repo_subworkflows_entry[subworkflow_name]["installed_by"] = [installed_by]
+        finally:
+            repo_subworkflows_entry[subworkflow_name]["installed_by"].extend(installed_by_log)
 
         # Sort the 'modules.json' repo entries
         self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"])
         if write_file:
             self.dump()
 
-    def remove_entry(self, module_name, repo_url, install_dir):
+    def remove_entry(self, component_type, name, repo_url, install_dir):
         """
         Removes an entry from the 'modules.json' file.
 
         Args:
-            module_name (str): Name of the module to be removed
+            component_type (Str): Type of component [modules, subworkflows]
+            name (str): Name of the module to be removed
             repo_url (str): URL of the repository containing the module
             install_dir (str): Name of the directory where modules are installed
         Returns:
@@ -595,15 +644,17 @@ def remove_entry(self, module_name, repo_url, install_dir):
             return False
         if repo_url in self.modules_json.get("repos", {}):
             repo_entry = self.modules_json["repos"][repo_url]
-            if module_name in repo_entry["modules"].get(install_dir, {}):
-                repo_entry["modules"][install_dir].pop(module_name)
+            if name in repo_entry[component_type].get(install_dir, {}):
+                repo_entry[component_type][install_dir].pop(name)
             else:
-                log.warning(f"Module '{install_dir}/{module_name}' is missing from 'modules.json' file.")
+                log.warning(
+                    f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file."
+                )
                 return False
-            if len(repo_entry["modules"][install_dir]) == 0:
+            if len(repo_entry[component_type][install_dir]) == 0:
                 self.modules_json["repos"].pop(repo_url)
         else:
-            log.warning(f"Module '{install_dir}/{module_name}' is missing from 'modules.json' file.")
+            log.warning(f"{component_type[:-1].title()} '{install_dir}/{name}' is missing from 'modules.json' file.")
             return False
 
         self.dump()

nf_core/modules/remove.py

@@ -50,13 +50,13 @@ def remove(self, module):
 
             if modules_json.module_present(module, self.modules_repo.remote_url, repo_path):
                 log.error(f"Found entry for '{module}' in 'modules.json'. Removing...")
-                modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path)
+                modules_json.remove_entry(self.component_type, module, self.modules_repo.remote_url, repo_path)
             return False
 
         log.info(f"Removing {module}")
 
         # Remove entry from modules.json
-        modules_json.remove_entry(module, self.modules_repo.remote_url, repo_path)
+        modules_json.remove_entry(self.component_type, module, self.modules_repo.remote_url, repo_path)
 
         # Remove the module
         return self.clear_component_dir(module, module_dir)

nf_core/modules/update.py

@@ -152,7 +152,7 @@ def update(self, module=None):
             if sha is not None:
                 version = sha
             elif self.prompt:
-                version = nf_core.modules.modules_utils.prompt_component_version_sha(
+                version = prompt_component_version_sha(
                     module, "modules", modules_repo=modules_repo, installed_sha=current_version
                 )
             else:
@@ -223,10 +223,10 @@ def update(self, module=None):
                 # Clear the module directory and move the installed files there
                 self.move_files_from_tmp_dir(module, install_tmp_dir, modules_repo.repo_path, version)
                 # Update modules.json with newly installed module
-                self.modules_json.update(modules_repo, module, version)
+                self.modules_json.update(modules_repo, module, version, self.component_type)
             else:
                 # Don't save to a file, just iteratively update the variable
-                self.modules_json.update(modules_repo, module, version, write_file=False)
+                self.modules_json.update(modules_repo, module, version, self.component_type, write_file=False)
 
         if self.save_diff_fn:
             # Write the modules.json diff to the file

nf_core/pipeline-template/modules.json

@@ -7,15 +7,18 @@
                 "nf-core": {
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
-                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905"
+                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905",
+                        "installed_by": ["modules"]
                     },
                     "fastqc": {
                         "branch": "master",
-                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905"
+                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905",
+                        "installed_by": ["modules"]
                     },
                     "multiqc": {
                         "branch": "master",
-                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905"
+                        "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905",
+                        "installed_by": ["modules"]
                     }
                 }
             }