From 3dc5ef5d39039d5dbd0c3263220cb32b20bd4216 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Dec 2023 14:39:33 +0100 Subject: [PATCH 01/15] run prettier only once on module.json --- nf_core/components/install.py | 2 ++ nf_core/modules/modules_json.py | 11 ++++++++--- nf_core/synced_repo.py | 7 ++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index f7a5fe6680..6385ee4092 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -127,6 +127,8 @@ def install(self, component, silent=False): self.install_included_components(component_dir) if not silent: + modules_json.load() + modules_json.dump(run_prettier=True) # Print include statement component_name = "_".join(component.upper().split("/")) log.info(f"Use the following statement to include this {self.component_type[:-1]}:") diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 9c3d1ae9b1..354a61dac9 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -6,6 +6,7 @@ import shutil import tempfile from pathlib import Path +from typing import Union import git import questionary @@ -41,7 +42,7 @@ def __init__(self, pipeline_dir): self.modules_dir = Path(self.dir, "modules") self.subworkflows_dir = Path(self.dir, "subworkflows") self.modules_json_path = Path(self.dir, "modules.json") - self.modules_json = None + self.modules_json: Union(dict, None) = None self.pipeline_modules = None self.pipeline_subworkflows = None self.pipeline_components = None @@ -1035,13 +1036,17 @@ def get_component_branch(self, component_type, component, repo_url, install_dir) ) return branch - def dump(self): + def dump(self, run_prettier: bool = False): """ Sort the modules.json, and write it to file """ # Sort the modules.json self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - dump_json_with_prettier(self.modules_json_path, self.modules_json) + if run_prettier: + dump_json_with_prettier(self.modules_json_path, self.modules_json) + else: + with open(self.modules_json_path, "w") as fh: + json.dump(self.modules_json, fh, indent=4) def resolve_missing_installation(self, missing_installation, component_type): missing_but_in_mod_json = [ diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index a2107f633c..aa996ac176 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -6,9 +6,10 @@ from typing import Dict import git -import rich.progress +from git.cmd import Git from git.exc import GitCommandError +import nf_core.modules.modules_utils from nf_core.utils import load_tools_config log = logging.getLogger(__name__) @@ -44,7 +45,7 @@ def __init__(self, progress_bar, repo_name, remote_url, operation): state="Waiting for response", ) - def update(self, op_code, cur_count, max_count=None, message=""): + def update(self, op_code, cur_count: int, max_count: int, message=""): """ Overrides git.RemoteProgress.update. Called every time there is a change in the remote operation @@ -90,7 +91,7 @@ def get_remote_branches(remote_url): (set[str]): All branches found in the remote """ try: - unparsed_branches = git.Git().ls_remote(remote_url) + unparsed_branches = Git().ls_remote(remote_url) except git.GitCommandError: raise LookupError(f"Was unable to fetch branches from '{remote_url}'") else: From f9b752e6d24f1a744f3d30bace3b2db33f70d667 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Dec 2023 14:56:34 +0100 Subject: [PATCH 02/15] fix import errors --- nf_core/synced_repo.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index aa996ac176..7e1238caac 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -9,7 +9,7 @@ from git.cmd import Git from git.exc import GitCommandError -import nf_core.modules.modules_utils +from nf_core.modules.modules_utils import repo_full_name_from_remote from nf_core.utils import load_tools_config log = logging.getLogger(__name__) @@ -45,7 +45,7 @@ def __init__(self, progress_bar, repo_name, remote_url, operation): state="Waiting for response", ) - def update(self, op_code, cur_count: int, max_count: int, message=""): + def update(self, op_code, cur_count, max_count, message=""): """ Overrides git.RemoteProgress.update. Called every time there is a change in the remote operation @@ -53,7 +53,7 @@ def update(self, op_code, cur_count: int, max_count: int, message=""): if not self.progress_bar.tasks[self.tid].started: self.progress_bar.start_task(self.tid) self.progress_bar.update( - self.tid, total=max_count, completed=cur_count, state=f"{cur_count / max_count * 100:.1f}%" + self.tid, total=max_count, completed=cur_count, state=f"{float(cur_count) / float(max_count) * 100:.1f}%" ) @@ -118,7 +118,7 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url - self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) + self.fullname = repo_full_name_from_remote(self.remote_url) self.setup_local_repo(remote_url, branch, hide_progress) From 8c526a1888f6bc118409fdbd605d50809ebbb833 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Dec 2023 15:08:29 +0100 Subject: [PATCH 03/15] fix circular import --- nf_core/synced_repo.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 7e1238caac..9ae7d632a3 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -9,7 +9,6 @@ from git.cmd import Git from git.exc import GitCommandError -from nf_core.modules.modules_utils import repo_full_name_from_remote from nf_core.utils import load_tools_config log = logging.getLogger(__name__) @@ -118,7 +117,7 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url - self.fullname = repo_full_name_from_remote(self.remote_url) + self.fullname = nf_core.modules.modules_utilsrepo_full_name_from_remote(self.remote_url) self.setup_local_repo(remote_url, branch, hide_progress) From f4a935618d05beb53e3a8534a3d0c382bf4b0e5d Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Dec 2023 15:19:07 +0100 Subject: [PATCH 04/15] disable caching to avoid weird bug --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 9f8172a3be..c9bf12643e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -47,7 +47,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - cache: "pip" + # cache: "pip" if: ${{ env.run-tests == 'true' }} outputs: python-version: ${{ matrix.python-version }} @@ -67,7 +67,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ needs.setup.outputs.python-version }} - cache: "pip" + # cache: "pip" - name: Install dependencies run: | From 9841e670a0f3cde831f457636c5365ff871110e7 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Dec 2023 15:34:14 +0100 Subject: [PATCH 05/15] Revert "disable caching to avoid weird bug" This reverts commit f4a935618d05beb53e3a8534a3d0c382bf4b0e5d. --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c9bf12643e..9f8172a3be 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -47,7 +47,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - # cache: "pip" + cache: "pip" if: ${{ env.run-tests == 'true' }} outputs: python-version: ${{ matrix.python-version }} @@ -67,7 +67,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ needs.setup.outputs.python-version }} - # cache: "pip" + cache: "pip" - name: Install dependencies run: | From 6ef6376f4bd51f435f43b5d444b32478bc4dbd08 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Dec 2023 16:46:39 +0100 Subject: [PATCH 06/15] run prettier on update once --- nf_core/components/update.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 6fc6e03544..7027c0c717 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -288,20 +288,23 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr updated.append(component) recursive_update = True modules_to_update, subworkflows_to_update = self.get_components_to_update(component) - if not silent and not self.update_all and len(modules_to_update + subworkflows_to_update) > 0: - log.warning( - f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n" - "It is advised to keep all your modules and subworkflows up to date.\n" - "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" - ) - if self.update_deps: - recursive_update = True - else: - recursive_update = questionary.confirm( - "Would you like to continue updating all modules and subworkflows?", - default=True, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() + if not silent and len(modules_to_update + subworkflows_to_update) > 0: + self.modules_json.load() + self.modules_json.dump(run_prettier=True) + if not self.update_all: + log.warning( + f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n" + "It is advised to keep all your modules and subworkflows up to date.\n" + "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n" + ) + if self.update_deps: + recursive_update = True + else: + recursive_update = questionary.confirm( + "Would you like to continue updating all modules and subworkflows?", + default=True, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() if recursive_update and len(modules_to_update + subworkflows_to_update) > 0: # Update linked components self.update_linked_components(modules_to_update, subworkflows_to_update, updated) From 095ee4af8816e5252e248359a65da2a885076822 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Dec 2023 16:47:03 +0100 Subject: [PATCH 07/15] reduce number of git checkouts, if we are already on the branch --- nf_core/synced_repo.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 9ae7d632a3..67f589d61b 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -3,7 +3,7 @@ import os import shutil from pathlib import Path -from typing import Dict +from typing import Dict, Union import git from git.cmd import Git @@ -174,8 +174,11 @@ def setup_branch(self, branch): else: self.branch = branch - # Verify that the branch exists by checking it out - self.branch_exists() + # Verify that the branch exists using git + try: + self.checkout_branch() + except git.GitCommandError: + raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'") def get_default_branch(self): """ @@ -185,15 +188,6 @@ def get_default_branch(self): _, branch = origin_head.ref.name.split("/") return branch - def branch_exists(self): - """ - Verifies that the branch exists in the repository by trying to check it out - """ - try: - self.checkout_branch() - except GitCommandError: - raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'") - def verify_branch(self): """ Verifies the active branch conforms to the correct directory structure @@ -211,7 +205,9 @@ def checkout_branch(self): """ Checks out the specified branch of the repository """ - self.repo.git.checkout(self.branch) + # only checkout if we're on a detached head or if we're not already on the branch + if self.repo.head.is_detached or self.repo.active_branch.name != self.branch: + self.repo.git.checkout(self.branch) def checkout(self, commit): """ @@ -249,7 +245,9 @@ def get_component_dir(self, component_name, component_type): elif component_type == "subworkflows": return os.path.join(self.subworkflows_dir, component_name) - def install_component(self, component_name, install_dir, commit, component_type): + def install_component( + self, component_name: Union[str, Path], install_dir: str, commit: str, component_type: str + ) -> bool: """ Install the module/subworkflow files into a pipeline at the given commit From 345afe19a8c6a36be5087ce6408b94f4d8bab222 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Dec 2023 21:31:47 +0100 Subject: [PATCH 08/15] only checkout commits if not already checked out --- nf_core/synced_repo.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 67f589d61b..728a4150fe 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -8,6 +8,7 @@ import git from git.cmd import Git from git.exc import GitCommandError +from git.repo import Repo from nf_core.utils import load_tools_config @@ -56,7 +57,7 @@ def update(self, op_code, cur_count, max_count, message=""): ) -class SyncedRepo: +class SyncedRepo(Repo): """ An object to store details about a locally cached code repository. """ @@ -117,7 +118,7 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url - self.fullname = nf_core.modules.modules_utilsrepo_full_name_from_remote(self.remote_url) + self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url) self.setup_local_repo(remote_url, branch, hide_progress) @@ -216,7 +217,9 @@ def checkout(self, commit): Args: commit (str): Git SHA of the commit """ - self.repo.git.checkout(commit) + # only checkout if we are not already on the commit + if self.repo.head.commit.hexsha != commit: + self.repo.git.checkout(commit) def component_exists(self, component_name, component_type, checkout=True, commit=None): """ From 29c87ccc11569a31a824a2a0b52cba911a56042a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 13 Dec 2023 10:34:30 +0100 Subject: [PATCH 09/15] run prettier on modules.json after updating all modules --- nf_core/components/update.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 7027c0c717..077cb2b840 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -289,8 +289,6 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr recursive_update = True modules_to_update, subworkflows_to_update = self.get_components_to_update(component) if not silent and len(modules_to_update + subworkflows_to_update) > 0: - self.modules_json.load() - self.modules_json.dump(run_prettier=True) if not self.update_all: log.warning( f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n" @@ -326,8 +324,12 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr ) elif not all_patches_successful and not silent: log.info(f"Updates complete. Please apply failed patch{plural_es(components_info)} manually.") + self.modules_json.load() + self.modules_json.dump(run_prettier=True) elif not silent: log.info("Updates complete :sparkles:") + self.modules_json.load() + self.modules_json.dump(run_prettier=True) return exit_value From 4624c64124dbcc7f9122810a962d88617a7d385a Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 13 Dec 2023 18:08:44 +0100 Subject: [PATCH 10/15] only check out as much of the repo as needed --- nf_core/modules/modules_repo.py | 2 +- nf_core/synced_repo.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 152ed7b0c0..79e2aec859 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -120,7 +120,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): ) with pbar: self.repo.remotes.origin.fetch( - progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") + depth=1, progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") ) ModulesRepo.update_local_repo_status(self.fullname, True) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 728a4150fe..d2012e2b5b 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -208,7 +208,7 @@ def checkout_branch(self): """ # only checkout if we're on a detached head or if we're not already on the branch if self.repo.head.is_detached or self.repo.active_branch.name != self.branch: - self.repo.git.checkout(self.branch) + self.repo.git.switch(self.branch) def checkout(self, commit): """ @@ -348,7 +348,17 @@ def sha_exists_on_branch(self, sha): Verifies that a given commit sha exists on the branch """ self.checkout_branch() - return sha in (commit.hexsha for commit in self.repo.iter_commits()) + + # check if sha exist, if not try shallow fetch until sha + sha_exists = False + try: + self.repo.remotes.origin.fetch(sha, depth=1) + sha_exists = True + except GitCommandError: + log.debug(f"Could not fetch commit {sha} from {self.remote_url}") + sha_exists = False + + return sha_exists def get_commit_info(self, sha): """ From 0528dbc65c522c09079651b24cb630a6050d7ecf Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 14 Dec 2023 09:18:59 +0100 Subject: [PATCH 11/15] Revert "only check out as much of the repo as needed" This reverts commit 4624c64124dbcc7f9122810a962d88617a7d385a. --- nf_core/modules/modules_repo.py | 2 +- nf_core/synced_repo.py | 14 ++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 79e2aec859..152ed7b0c0 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -120,7 +120,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): ) with pbar: self.repo.remotes.origin.fetch( - depth=1, progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") + progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling") ) ModulesRepo.update_local_repo_status(self.fullname, True) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index d2012e2b5b..728a4150fe 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -208,7 +208,7 @@ def checkout_branch(self): """ # only checkout if we're on a detached head or if we're not already on the branch if self.repo.head.is_detached or self.repo.active_branch.name != self.branch: - self.repo.git.switch(self.branch) + self.repo.git.checkout(self.branch) def checkout(self, commit): """ @@ -348,17 +348,7 @@ def sha_exists_on_branch(self, sha): Verifies that a given commit sha exists on the branch """ self.checkout_branch() - - # check if sha exist, if not try shallow fetch until sha - sha_exists = False - try: - self.repo.remotes.origin.fetch(sha, depth=1) - sha_exists = True - except GitCommandError: - log.debug(f"Could not fetch commit {sha} from {self.remote_url}") - sha_exists = False - - return sha_exists + return sha in (commit.hexsha for commit in self.repo.iter_commits()) def get_commit_info(self, sha): """ From 4d570b0ad375172139aec4bc00043eef043ff401 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 14 Dec 2023 09:35:40 +0100 Subject: [PATCH 12/15] disable pip cache to start tests --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 9f8172a3be..c9bf12643e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -47,7 +47,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - cache: "pip" + # cache: "pip" if: ${{ env.run-tests == 'true' }} outputs: python-version: ${{ matrix.python-version }} @@ -67,7 +67,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ needs.setup.outputs.python-version }} - cache: "pip" + # cache: "pip" - name: Install dependencies run: | From ecc93beea10007c4f20d1f5b27748dfd0206ba3f Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 14 Dec 2023 09:52:28 +0100 Subject: [PATCH 13/15] removed unused assignment, which cause type errors --- nf_core/download.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 2cb38061de..516ffe4235 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1381,9 +1381,6 @@ def __init__( self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache) - # expose some instance attributes - self.tags = self.repo.tags - def __repr__(self): """Called by print, creates representation of object""" return f"" @@ -1498,7 +1495,6 @@ def tidy_tags_and_branches(self): # delete unwanted tags from repository for tag in tags_to_remove: self.repo.delete_tag(tag) - self.tags = self.repo.tags # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "master") self.checkout(self.revision[0]) @@ -1535,8 +1531,6 @@ def tidy_tags_and_branches(self): if self.repo.head.is_detached: self.repo.head.reset(index=True, working_tree=True) - self.heads = self.repo.heads - # get all tags and available remote_branches completed_revisions = {revision.name for revision in self.repo.heads + self.repo.tags} From 5aaaa2bbf39d7940fa02909086d76de6cc2c00c1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 14 Dec 2023 10:02:17 +0100 Subject: [PATCH 14/15] make download tests pass again --- nf_core/download.py | 6 ++++++ nf_core/synced_repo.py | 41 ++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 516ffe4235..2cb38061de 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1381,6 +1381,9 @@ def __init__( self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache) + # expose some instance attributes + self.tags = self.repo.tags + def __repr__(self): """Called by print, creates representation of object""" return f"" @@ -1495,6 +1498,7 @@ def tidy_tags_and_branches(self): # delete unwanted tags from repository for tag in tags_to_remove: self.repo.delete_tag(tag) + self.tags = self.repo.tags # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "master") self.checkout(self.revision[0]) @@ -1531,6 +1535,8 @@ def tidy_tags_and_branches(self): if self.repo.head.is_detached: self.repo.head.reset(index=True, working_tree=True) + self.heads = self.repo.heads + # get all tags and available remote_branches completed_revisions = {revision.name for revision in self.repo.heads + self.repo.tags} diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 728a4150fe..a2107f633c 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -3,12 +3,11 @@ import os import shutil from pathlib import Path -from typing import Dict, Union +from typing import Dict import git -from git.cmd import Git +import rich.progress from git.exc import GitCommandError -from git.repo import Repo from nf_core.utils import load_tools_config @@ -45,7 +44,7 @@ def __init__(self, progress_bar, repo_name, remote_url, operation): state="Waiting for response", ) - def update(self, op_code, cur_count, max_count, message=""): + def update(self, op_code, cur_count, max_count=None, message=""): """ Overrides git.RemoteProgress.update. Called every time there is a change in the remote operation @@ -53,11 +52,11 @@ def update(self, op_code, cur_count, max_count, message=""): if not self.progress_bar.tasks[self.tid].started: self.progress_bar.start_task(self.tid) self.progress_bar.update( - self.tid, total=max_count, completed=cur_count, state=f"{float(cur_count) / float(max_count) * 100:.1f}%" + self.tid, total=max_count, completed=cur_count, state=f"{cur_count / max_count * 100:.1f}%" ) -class SyncedRepo(Repo): +class SyncedRepo: """ An object to store details about a locally cached code repository. """ @@ -91,7 +90,7 @@ def get_remote_branches(remote_url): (set[str]): All branches found in the remote """ try: - unparsed_branches = Git().ls_remote(remote_url) + unparsed_branches = git.Git().ls_remote(remote_url) except git.GitCommandError: raise LookupError(f"Was unable to fetch branches from '{remote_url}'") else: @@ -175,11 +174,8 @@ def setup_branch(self, branch): else: self.branch = branch - # Verify that the branch exists using git - try: - self.checkout_branch() - except git.GitCommandError: - raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'") + # Verify that the branch exists by checking it out + self.branch_exists() def get_default_branch(self): """ @@ -189,6 +185,15 @@ def get_default_branch(self): _, branch = origin_head.ref.name.split("/") return branch + def branch_exists(self): + """ + Verifies that the branch exists in the repository by trying to check it out + """ + try: + self.checkout_branch() + except GitCommandError: + raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'") + def verify_branch(self): """ Verifies the active branch conforms to the correct directory structure @@ -206,9 +211,7 @@ def checkout_branch(self): """ Checks out the specified branch of the repository """ - # only checkout if we're on a detached head or if we're not already on the branch - if self.repo.head.is_detached or self.repo.active_branch.name != self.branch: - self.repo.git.checkout(self.branch) + self.repo.git.checkout(self.branch) def checkout(self, commit): """ @@ -217,9 +220,7 @@ def checkout(self, commit): Args: commit (str): Git SHA of the commit """ - # only checkout if we are not already on the commit - if self.repo.head.commit.hexsha != commit: - self.repo.git.checkout(commit) + self.repo.git.checkout(commit) def component_exists(self, component_name, component_type, checkout=True, commit=None): """ @@ -248,9 +249,7 @@ def get_component_dir(self, component_name, component_type): elif component_type == "subworkflows": return os.path.join(self.subworkflows_dir, component_name) - def install_component( - self, component_name: Union[str, Path], install_dir: str, commit: str, component_type: str - ) -> bool: + def install_component(self, component_name, install_dir, commit, component_type): """ Install the module/subworkflow files into a pipeline at the given commit From 4110313c9b5dbbedd18986b1533af65c0f4d3d5a Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 14 Dec 2023 10:44:29 +0100 Subject: [PATCH 15/15] re-enable pip cache --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c9bf12643e..9f8172a3be 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -47,7 +47,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - # cache: "pip" + cache: "pip" if: ${{ env.run-tests == 'true' }} outputs: python-version: ${{ matrix.python-version }} @@ -67,7 +67,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ needs.setup.outputs.python-version }} - # cache: "pip" + cache: "pip" - name: Install dependencies run: |