Merge branch 'dev' into git-urls

nf_core/components/components_utils.py

@@ -1,5 +1,7 @@
 import logging
 import os
+import re
+from pathlib import Path
 
 import questionary
 import rich.prompt
@@ -113,3 +115,25 @@ def prompt_component_version_sha(component_name, component_type, modules_repo, i
  ).unsafe_ask()
  page_nbr += 1
  return git_sha
+
+
+def get_components_to_install(subworkflow_dir):
+ """
+ Parse the subworkflow test main.nf file to retrieve all imported modules and subworkflows.
+ """
+ modules = []
+ subworkflows = []
+ with open(Path(subworkflow_dir, "main.nf"), "r") as fh:
+ for line in fh:
+ regex = re.compile(
+ r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")"
+ )
+ match = regex.match(line)
+ if match and len(match.groups()) == 2:
+ name, link = match.groups()
+ if link.startswith("../../../"):
+ name_split = name.lower().split("_")
+ modules.append("/".join(name_split))
+ elif link.startswith("../"):
+ subworkflows.append(name.lower())
+ return modules, subworkflows

nf_core/components/install.py

@@ -10,7 +10,10 @@
 import nf_core.modules.modules_utils
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
-from nf_core.components.components_utils import prompt_component_version_sha
+from nf_core.components.components_utils import (
+ get_components_to_install,
+ prompt_component_version_sha,
+)
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME
 
@@ -53,7 +56,8 @@ def install(self, component, silent=False):
 
  # Verify that 'modules.json' is consistent with the installed modules and subworkflows
  modules_json = ModulesJson(self.dir)
- modules_json.check_up_to_date()
+ if not silent:
+ modules_json.check_up_to_date()
 
  # Verify SHA
  if not self.modules_repo.verify_sha(self.prompt, self.sha):
@@ -138,32 +142,11 @@ def install(self, component, silent=False):
  )
  return True
 
- def get_modules_subworkflows_to_install(self, subworkflow_dir):
- """
- Parse the subworkflow test main.nf file to retrieve all imported modules and subworkflows.
- """
- modules = []
- subworkflows = []
- with open(Path(subworkflow_dir, "main.nf"), "r") as fh:
- for line in fh:
- regex = re.compile(
- r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")"
- )
- match = regex.match(line)
- if match and len(match.groups()) == 2:
- name, link = match.groups()
- if link.startswith("../../../"):
- name_split = name.lower().split("_")
- modules.append("/".join(name_split))
- elif link.startswith("../"):
- subworkflows.append(name.lower())
- return modules, subworkflows
-
  def install_included_components(self, subworkflow_dir):
  """
  Install included modules and subworkflows
  """
- modules_to_install, subworkflows_to_install = self.get_modules_subworkflows_to_install(subworkflow_dir)
+ modules_to_install, subworkflows_to_install = get_components_to_install(subworkflow_dir)
  for s_install in subworkflows_to_install:
  original_installed = self.installed_by
  self.installed_by = Path(subworkflow_dir).parts[-1]

nf_core/components/update.py

@@ -96,7 +96,8 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
  self.check_modules_structure()
 
  # Verify that 'modules.json' is consistent with the installed modules
- self.modules_json.check_up_to_date()
+ if not silent:
+ self.modules_json.check_up_to_date()
 
  if not self.update_all and component is None:
  choices = [f"All {self.component_type}", f"Named {self.component_type[:-1]}"]
@@ -233,9 +234,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
  else:
  updated.append(component)
  recursive_update = True
- modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update(
- component, modules_repo
- )
+ modules_to_update, subworkflows_to_update = self.get_components_to_update(component, modules_repo)
  if not silent and len(modules_to_update + subworkflows_to_update) > 0:
  log.warning(
  f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be added to the same diff file.\n"
@@ -282,9 +281,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
  self.modules_json.update(self.component_type, modules_repo, component, version, self.component_type)
  updated.append(component)
  recursive_update = True
- modules_to_update, subworkflows_to_update = self.get_modules_subworkflows_to_update(
- component, modules_repo
- )
+ modules_to_update, subworkflows_to_update = self.get_components_to_update(component, modules_repo)
  if not silent and not self.update_all and len(modules_to_update + subworkflows_to_update) > 0:
  log.warning(
  f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n"
@@ -349,14 +346,14 @@ def get_single_component_info(self, component):
  # Check if there are any modules/subworkflows installed from the repo
  repo_url = self.modules_repo.remote_url
  components = self.modules_json.get_all_components(self.component_type).get(repo_url)
- choices = [component if dir == "nf-core" else f"{dir}/{component}" for dir, component in components]
- if repo_url not in self.modules_json.get_all_components(self.component_type):
+ if components is None:
  raise LookupError(f"No {self.component_type} installed from '{repo_url}'")
+ choices = [component if dir == "nf-core" else f"{dir}/{component}" for dir, component in components]
 
  if component is None:
  component = questionary.autocomplete(
  f"{self.component_type[:-1].title()} name:",
- choices=choices.sort(),
+ choices=sorted(choices),
  style=nf_core.utils.nfcore_question_style,
  ).unsafe_ask()
 
@@ -819,8 +816,13 @@ def try_apply_patch(
 
  return True
 
- def get_modules_subworkflows_to_update(self, component, modules_repo):
- """Get all modules and subworkflows linked to the updated component."""
+ def get_components_to_update(self, component, modules_repo):
+ """
+ Get all modules and subworkflows linked to the updated component.
+
+ Returns:
+ (list,list): A tuple of lists with the modules and subworkflows to update
+ """
  mods_json = self.modules_json.get_modules_json()
  modules_to_update = []
  subworkflows_to_update = []

nf_core/modules/lint/__init__.py

@@ -90,7 +90,7 @@ def __init__(
  modules_json = ModulesJson(self.dir)
  modules_json.check_up_to_date()
  all_pipeline_modules = modules_json.get_all_components(self.component_type)
- if self.modules_repo.remote_url in all_pipeline_modules:
+ if all_pipeline_modules is not None and self.modules_repo.remote_url in all_pipeline_modules:
  module_dir = Path(self.dir, "modules", "nf-core")
  self.all_remote_modules = [
  NFCoreModule(m[1], self.modules_repo.remote_url, module_dir / m[1], self.repo_type, Path(self.dir))

nf_core/modules/modules_json.py

@@ -14,6 +14,7 @@
 import nf_core.modules.modules_repo
 import nf_core.modules.modules_utils
 import nf_core.utils
+from nf_core.components.components_utils import get_components_to_install
 
 from .modules_differ import ModulesDiffer
 
@@ -323,7 +324,7 @@ def determine_branches_and_shas(self, component_type, install_dir, remote_url, c
  repo_entry[component] = {
  "branch": modules_repo.branch,
  "git_sha": correct_commit_sha,
- "installed_by": "modules",
+ "installed_by": [component_type],
  }
 
  # Clean up the modules/subworkflows we were unable to find the sha for
@@ -549,9 +550,10 @@ def check_up_to_date(self):
  if not self.has_git_url_and_modules():
  raise UserWarning
  except UserWarning:
- log.info("The 'modules.json' file is not up to date. Recreating the 'module.json' file.")
+ log.info("The 'modules.json' file is not up to date. Recreating the 'modules.json' file.")
  self.create()
 
+ # Get unsynced components
  (
  modules_missing_from_modules_json,
  subworkflows_missing_from_modules_json,
@@ -587,6 +589,16 @@ def check_up_to_date(self):
  component_type
  ]
 
+ # Recreate "installed_by" entry
+ original_pipeline_components = self.pipeline_components
+ self.pipeline_components = None
+ subworkflows_dict = self.get_all_components("subworkflows")
+ if subworkflows_dict:
+ for repo, subworkflows in subworkflows_dict.items():
+ for org, subworkflow in subworkflows:
+ self.recreate_dependencies(repo, org, subworkflow)
+ self.pipeline_components = original_pipeline_components
+
  self.dump()
 
  def load(self):
@@ -917,6 +929,8 @@ def get_all_components(self, component_type):
  if component_type in repo_entry:
  for dir, components in repo_entry[component_type].items():
  self.pipeline_components[repo] = [(dir, m) for m in components]
+ if self.pipeline_components == {}:
+ self.pipeline_components = None
 
  return self.pipeline_components
 
@@ -1106,3 +1120,28 @@ def components_with_repos():
  }
  }
  )
+
+ def recreate_dependencies(self, repo, org, subworkflow):
+ """
+ Try to recreate the installed_by entries for subworkflows.
+ Remove self installation entry from dependencies, assuming that the modules.json has been freshly created,
+ i.e., no module or subworkflow has been installed by the user in the meantime
+ """
+
+ sw_path = Path(self.subworkflows_dir, org, subworkflow)
+ dep_mods, dep_subwfs = get_components_to_install(sw_path)
+
+ for dep_mod in dep_mods:
+ installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"]
+ if installed_by == ["modules"]:
+ self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = []
+ if subworkflow not in installed_by:
+ self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"].append(subworkflow)
+
+ for dep_subwf in dep_subwfs:
+ installed_by = self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"]
+ if installed_by == ["subworkflows"]:
+ self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] = []
+ if subworkflow not in installed_by:
+ self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"].append(subworkflow)
+ self.recreate_dependencies(repo, org, dep_subwf)