From 7c5d0966c955276efe213f38713da9c705f5c184 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 13:00:10 +0100 Subject: [PATCH 1/7] allowing new line before description field --- nf_core/module-template/modules/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 05ecb6f980..9b42bb3bdf 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,7 +1,7 @@ name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords -{%- endif -%} +{% endif -%} description: write your description here keywords: - sort From 18277d666c06a57baf3be9e8707c15afed9f4a80 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 14:55:26 +0100 Subject: [PATCH 2/7] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5dd860b3e..b5c5701120 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ ### Modules -- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175)) +- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175) & [#2177](https://github.com/nf-core/tools/pull/2177)) ### Subworkflows From 53c874fc193ab73f7457d79f0239a53018393850 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 16:38:53 +0100 Subject: [PATCH 3/7] checkout to sha when getting list of available components --- nf_core/components/install.py | 6 +++--- nf_core/modules/modules_repo.py | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index f9a16f73cf..cb6c59aec3 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -169,19 +169,19 @@ def collect_and_verify_name(self, component, modules_repo): if component is None: component = questionary.autocomplete( f"{'Tool' if self.component_type == 'modules' else 'Subworkflow'} name:", - choices=sorted(modules_repo.get_avail_components(self.component_type)), + choices=sorted(modules_repo.get_avail_components(self.component_type, commit=self.sha)), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() # Check that the supplied name is an available module/subworkflow - if component and component not in modules_repo.get_avail_components(self.component_type): + if component and component not in modules_repo.get_avail_components(self.component_type, commit=self.sha): log.error( f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}." ) log.info(f"Use the command 'nf-core {self.component_type} list' to view available software") return False - if not modules_repo.component_exists(component, self.component_type): + if not modules_repo.component_exists(component, self.component_type, commit=self.sha): warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" log.warning(warn_msg) return False diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 606514e55e..5f77148867 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -302,7 +302,7 @@ def checkout(self, commit): """ self.repo.git.checkout(commit) - def component_exists(self, component_name, component_type, checkout=True): + def component_exists(self, component_name, component_type, checkout=True, commit=None): """ Check if a module/subworkflow exists in the branch of the repo @@ -312,7 +312,7 @@ def component_exists(self, component_name, component_type, checkout=True): Returns: (bool): Whether the module/subworkflow exists in this branch of the repository """ - return component_name in self.get_avail_components(component_type, checkout=checkout) + return component_name in self.get_avail_components(component_type, checkout=checkout, commit=commit) def get_component_dir(self, component_name, component_type): """ @@ -449,7 +449,7 @@ def get_commit_info(self, sha): return message, date raise LookupError(f"Commit '{sha}' not found in the '{self.remote_url}'") - def get_avail_components(self, component_type, checkout=True): + def get_avail_components(self, component_type, checkout=True, commit=None): """ Gets the names of the modules/subworkflows in the repository. They are detected by checking which directories have a 'main.nf' file @@ -459,6 +459,8 @@ def get_avail_components(self, component_type, checkout=True): """ if checkout: self.checkout_branch() + if commit is not None: + self.checkout(commit) # Get directory if component_type == "modules": directory = self.modules_dir From b2f3698631721a09e83d4d315a5b29cec5b683c2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:35:25 +0100 Subject: [PATCH 4/7] skip modules whose name has changed --- nf_core/components/update.py | 15 ++++++++++++--- tests/subworkflows/update.py | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index ef645a5a1d..e640827594 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -386,7 +386,7 @@ def get_single_component_info(self, component): ) # Check that the supplied name is an available module/subworkflow - if component and component not in self.modules_repo.get_avail_components(self.component_type): + if component and component not in self.modules_repo.get_avail_components(self.component_type, commit=self.sha): raise LookupError( f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}." f"Use the command 'nf-core {self.component_type} list remote' to view available software" @@ -879,8 +879,17 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up if m_update in updated: continue original_component_type, original_update_all = self._change_component_type("modules") - self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) - self._reset_component_type(original_component_type, original_update_all) + try: + self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) + except LookupError as e: + # If the module to be updated is not available, check if there has been a name change + if "not found in list of available" in str(e): + # Skip update, we check for name changes with manage_changes_in_linked_components in line #261 + pass + else: + raise + finally: + self._reset_component_type(original_component_type, original_update_all) def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update): """Check for linked components added or removed in the new subworkflow version""" diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 29e6cb1179..1d437f00a4 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -8,6 +8,7 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.modules.remove import ModuleRemove from nf_core.modules.update import ModuleUpdate from nf_core.subworkflows.update import SubworkflowUpdate From 39810e1d37149ead2905b7ee3b651fa9cb776266 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:47:12 +0100 Subject: [PATCH 5/7] change manage_changes order --- nf_core/components/update.py | 22 +++++++++++----------- tests/subworkflows/update.py | 4 ++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index e640827594..ab4a6eedbb 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -896,17 +896,6 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw if self.component_type == "subworkflows": subworkflow_directory = Path(self.dir, self.component_type, self.modules_repo.repo_path, component) included_modules, included_subworkflows = get_components_to_install(subworkflow_directory) - # If a new module/subworkflow is included in the subworklfow and wasn't included before - for module in included_modules: - if module not in modules_to_update: - log.info(f"Installing newly included module '{module}' for '{component}'") - install_module_object = ComponentInstall(self.dir, "modules", installed_by=component) - install_module_object.install(module, silent=True) - for subworkflow in included_subworkflows: - if subworkflow not in subworkflows_to_update: - log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'") - install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component) - install_subworkflow_object.install(subworkflow, silent=True) # If a module/subworkflow has been removed from the subworkflow for module in modules_to_update: if module not in included_modules: @@ -918,6 +907,17 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.") remove_subworkflow_object = ComponentRemove("subworkflows", self.dir) remove_subworkflow_object.remove(subworkflow, removed_by=component) + # If a new module/subworkflow is included in the subworklfow and wasn't included before + for module in included_modules: + if module not in modules_to_update: + log.info(f"Installing newly included module '{module}' for '{component}'") + install_module_object = ComponentInstall(self.dir, "modules", installed_by=component) + install_module_object.install(module, silent=True) + for subworkflow in included_subworkflows: + if subworkflow not in subworkflows_to_update: + log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'") + install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component) + install_subworkflow_object.install(subworkflow, silent=True) def _change_component_type(self, new_component_type): original_component_type = self.component_type diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 1d437f00a4..698086e186 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -322,6 +322,10 @@ def test_update_change_of_included_modules(self): # Check that tabix/tabix is there assert "tabix/tabix" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "tabix/tabix").is_dir() + # Check that ensemblevep is not there but instead we have ensemblevep/vep (due to a file re-naming) + assert "ensemblvep" not in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] + assert "ensemblvep/vep" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] + assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "ensemblvep/vep").is_dir() def cmp_component(dir1, dir2): From 297b2c8fd6398b0e1c16981616392308b2e4380a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:54:11 +0100 Subject: [PATCH 6/7] update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c5701120..3b3d15f18a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] - Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] -- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) +- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170)) ### Linting @@ -16,6 +16,8 @@ ### Subworkflows +- Fixing problem when a module included in a subworkflow had a name change from TOOL to TOOL/SUBTOOL ([#2177](https://github.com/nf-core/tools/pull/2177)) + ### General ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] From cab29c86aa70a838c18f3f5a1f05b62cf7328b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 9 Feb 2023 11:40:26 +0100 Subject: [PATCH 7/7] Update nf_core/components/update.py --- nf_core/components/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index ab4a6eedbb..5f8a2129c5 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -884,7 +884,7 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up except LookupError as e: # If the module to be updated is not available, check if there has been a name change if "not found in list of available" in str(e): - # Skip update, we check for name changes with manage_changes_in_linked_components in line #261 + # Skip update, we check for name changes with manage_changes_in_linked_components pass else: raise