From 8295f77c6d9efe65378d09c8cfdbda15fcdc1f64 Mon Sep 17 00:00:00 2001 From: Novak <116580988+NovakApis@users.noreply.github.com> Date: Tue, 28 Mar 2023 13:34:10 +0200 Subject: [PATCH 01/60] my repo test --- .github/workflows/my_pytest.yml | 50 ++++++++++++++++++ nf_core/components/components_command.py | 66 +++++++++++++----------- 2 files changed, 86 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/my_pytest.yml diff --git a/.github/workflows/my_pytest.yml b/.github/workflows/my_pytest.yml new file mode 100644 index 0000000000..176395ab0b --- /dev/null +++ b/.github/workflows/my_pytest.yml @@ -0,0 +1,50 @@ +name: Python tests +# This workflow is triggered on pushes and PRs to the repository. +# Only run if we changed a Python file +on: + push: + branches: + - feature/** + pull_request: + release: + types: [published] + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + pytest: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.8"] + + steps: + - uses: actions/checkout@v3 + name: Check out source-code repository + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + python -m pip install --upgrade pip -r requirements-dev.txt + pip install -e . + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: "latest-everything" + + - name: Test with pytest + run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core + + - uses: codecov/codecov-action@v1 + name: Upload code coverage report + with: + if: success() + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 31ab1a71fb..6ea5d8360b 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -3,6 +3,7 @@ import os import shutil from pathlib import Path +from typing import List, Tuple, Dict, Union import yaml @@ -19,7 +20,9 @@ class ComponentCommand: Base class for the 'nf-core modules' and 'nf-core subworkflows' commands """ - def __init__(self, component_type, dir, remote_url=None, branch=None, no_pull=False, hide_progress=False): + def __init__( + self, component_type, dir: str, remote_url=None, branch=None, no_pull=False, hide_progress=False + ) -> None: """ Initialise the ComponentClass object """ @@ -29,7 +32,7 @@ def __init__(self, component_type, dir, remote_url=None, branch=None, no_pull=Fa self.hide_progress = hide_progress self._configure_repo_and_paths() - def _configure_repo_and_paths(self, nf_dir_req=True): + def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None: """ Determine the repo type and set some default paths. If this is a modules repo, determine the org_path too. @@ -37,6 +40,7 @@ def _configure_repo_and_paths(self, nf_dir_req=True): Args: nf_dir_req (bool, optional): Whether this command requires being run in the nf-core modules repo or a nf-core pipeline repository. Defaults to True. """ + self.dir: Union[str, Path] try: if self.dir: self.dir, self.repo_type, self.org = get_repo_info(self.dir, use_prompt=nf_dir_req) @@ -48,12 +52,12 @@ def _configure_repo_and_paths(self, nf_dir_req=True): raise self.repo_type = None self.org = "" - self.default_modules_path = Path("modules", self.org) - self.default_tests_path = Path("tests", "modules", self.org) - self.default_subworkflows_path = Path("subworkflows", self.org) - self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org) + self.default_modules_path: Path = Path("modules", self.org) + self.default_tests_path: Path = Path("tests", "modules", self.org) + self.default_subworkflows_path: Path = Path("subworkflows", self.org) + self.default_subworkflows_tests_path: Path = Path("tests", "subworkflows", self.org) - def get_local_components(self): + def get_local_components(self) -> List[str]: """ Get the local modules/subworkflows in a pipeline """ @@ -62,7 +66,7 @@ def get_local_components(self): str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] - def get_components_clone_modules(self): + def get_components_clone_modules(self) -> List[str]: """ Get the modules/subworkflows repository available in a clone of nf-core/modules """ @@ -76,29 +80,29 @@ def get_components_clone_modules(self): if "main.nf" in files ] - def has_valid_directory(self): + def has_valid_directory(self) -> bool: """Check that we were given a pipeline or clone of nf-core/modules""" if self.repo_type == "modules": return True if self.dir is None or not os.path.exists(self.dir): log.error(f"Could not find directory: {self.dir}") return False - main_nf = os.path.join(self.dir, "main.nf") - nf_config = os.path.join(self.dir, "nextflow.config") + main_nf: str = os.path.join(self.dir, "main.nf") + nf_config: str = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): if Path(self.dir).resolve().parts[-1].startswith("nf-core"): raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") return True - def has_modules_file(self): + def has_modules_file(self) -> None: """Checks whether a module.json file has been created and creates one if it is missing""" modules_json_path = os.path.join(self.dir, "modules.json") if not os.path.exists(modules_json_path): log.info("Creating missing 'module.json' file.") ModulesJson(self.dir).create() - def clear_component_dir(self, component_name, component_dir): + def clear_component_dir(self, component_name: str, component_dir: str) -> bool: """ Removes all files in the module/subworkflow directory @@ -126,7 +130,7 @@ def clear_component_dir(self, component_name, component_dir): log.error(f"Could not remove {self.component_type[:-1]} {component_name}: {e}") return False - def components_from_repo(self, install_dir): + def components_from_repo(self, install_dir: str) -> List[str]: """ Gets the modules/subworkflows installed from a certain repository @@ -136,7 +140,7 @@ def components_from_repo(self, install_dir): Returns: [str]: The names of the modules/subworkflows """ - repo_dir = Path(self.dir, self.component_type, install_dir) + repo_dir: Path = Path(self.dir, self.component_type, install_dir) if not repo_dir.exists(): raise LookupError(f"Nothing installed from {install_dir} in pipeline") @@ -144,7 +148,9 @@ def components_from_repo(self, install_dir): str(Path(dir_path).relative_to(repo_dir)) for dir_path, _, files in os.walk(repo_dir) if "main.nf" in files ] - def install_component_files(self, component_name, component_version, modules_repo, install_dir): + def install_component_files( + self, component_name: str, component_version: str, modules_repo: ModulesRepo, install_dir: str + ) -> bool: """ Installs a module/subworkflow into the given directory @@ -159,7 +165,7 @@ def install_component_files(self, component_name, component_version, modules_rep """ return modules_repo.install_component(component_name, install_dir, component_version, self.component_type) - def load_lint_config(self): + def load_lint_config(self) -> None: """Parse a pipeline lint config file. Look for a file called either `.nf-core-lint.yml` or @@ -181,7 +187,7 @@ def load_lint_config(self): except FileNotFoundError: log.debug(f"No lint config file found: {config_fn}") - def check_modules_structure(self): + def check_modules_structure(self) -> None: """ Check that the structure of the modules directory in a pipeline is the correct one: modules/nf-core/TOOL/SUBTOOL @@ -190,11 +196,11 @@ def check_modules_structure(self): modules/nf-core/modules/TOOL/SUBTOOL """ if self.repo_type == "pipeline": - wrong_location_modules = [] + wrong_location_modules: List[Path] = [] for directory, _, files in os.walk(Path(self.dir, "modules")): if "main.nf" in files: - module_path = Path(directory).relative_to(Path(self.dir, "modules")) - parts = module_path.parts + module_path: Path = Path(directory).relative_to(Path(self.dir, "modules")) + parts: Tuple[str, ...] = module_path.parts # Check that there are modules installed directly under the 'modules' directory if parts[1] == "modules": wrong_location_modules.append(module_path) @@ -208,17 +214,17 @@ def check_modules_structure(self): ) # Move wrong modules to the right directory for module in wrong_location_modules: - modules_dir = Path("modules").resolve() - correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) - wrong_dir = Path(modules_dir, module) + modules_dir: Path = Path("modules").resolve() + correct_dir: Path = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) + wrong_dir: Path = Path(modules_dir, module) shutil.move(wrong_dir, correct_dir) log.info(f"Moved {wrong_dir} to {correct_dir}.") shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) # Regenerate modules.json file - modules_json = ModulesJson(self.dir) + modules_json: ModulesJson = ModulesJson(self.dir) modules_json.check_up_to_date() - def check_patch_paths(self, patch_path, module_name): + def check_patch_paths(self, patch_path: Path, module_name: str) -> None: """ Check that paths in patch files are updated to the new modules path """ @@ -241,7 +247,7 @@ def check_patch_paths(self, patch_path, module_name): for line in lines: fh.write(line) # Update path in modules.json if the file is in the correct format - modules_json = ModulesJson(self.dir) + modules_json: ModulesJson = ModulesJson(self.dir) modules_json.load() if modules_json.has_git_url_and_modules(): modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ @@ -249,7 +255,7 @@ def check_patch_paths(self, patch_path, module_name): ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) modules_json.dump() - def check_if_in_include_stmts(self, component_path): + def check_if_in_include_stmts(self, component_path: str) -> Dict[str, List[Dict[str, Union[int, str]]]]: """ Checks for include statements in the main.nf file of the pipeline and a list of line numbers where the component is included Args: @@ -258,9 +264,9 @@ def check_if_in_include_stmts(self, component_path): Returns: (list): A list of dictionaries, with the workflow file and the line number where the component is included """ - include_stmts = {} + include_stmts: Dict[str, List[Dict[str, Union[int, str]]]] = {} if self.repo_type == "pipeline": - workflow_files = Path(self.dir, "workflows").glob("*.nf") + workflow_files: Path = Path(self.dir, "workflows").glob("*.nf") for workflow_file in workflow_files: with open(workflow_file, "r") as fh: # Check if component path is in the file using mmap From 68513c02c6553b2094fa02b821a1fe27a5268db9 Mon Sep 17 00:00:00 2001 From: Novak <116580988+NovakApis@users.noreply.github.com> Date: Tue, 28 Mar 2023 13:36:22 +0200 Subject: [PATCH 02/60] black + isort --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 6ea5d8360b..92d5bb1489 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -3,7 +3,7 @@ import os import shutil from pathlib import Path -from typing import List, Tuple, Dict, Union +from typing import Dict, List, Tuple, Union import yaml From 203a04bd398a621086978cfb4a4dfc19a27665c1 Mon Sep 17 00:00:00 2001 From: Novak <116580988+NovakApis@users.noreply.github.com> Date: Tue, 28 Mar 2023 16:28:26 +0200 Subject: [PATCH 03/60] first test --- nf_core/components/components_command.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 92d5bb1489..01a2ce0832 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -3,7 +3,7 @@ import os import shutil from pathlib import Path -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Tuple, Union, Optional import yaml @@ -21,18 +21,24 @@ class ComponentCommand: """ def __init__( - self, component_type, dir: str, remote_url=None, branch=None, no_pull=False, hide_progress=False + self, + component_type: str, + dir: str, + remote_url: str = None, + branch: str = None, + no_pull: bool = False, + hide_progress: bool = False, ) -> None: """ Initialise the ComponentClass object """ self.component_type = component_type self.dir = dir - self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) + self.modules_repo: ModulesRepo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.hide_progress = hide_progress self._configure_repo_and_paths() - def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None: + def _configure_repo_and_paths(self, nf_dir_req: Optional[bool] = True) -> None: """ Determine the repo type and set some default paths. If this is a modules repo, determine the org_path too. @@ -41,6 +47,7 @@ def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None: nf_dir_req (bool, optional): Whether this command requires being run in the nf-core modules repo or a nf-core pipeline repository. Defaults to True. """ self.dir: Union[str, Path] + self.repo_type: Optional[str] try: if self.dir: self.dir, self.repo_type, self.org = get_repo_info(self.dir, use_prompt=nf_dir_req) @@ -61,7 +68,7 @@ def get_local_components(self) -> List[str]: """ Get the local modules/subworkflows in a pipeline """ - local_component_dir = Path(self.dir, self.component_type, "local") + local_component_dir: Path = Path(self.dir, self.component_type, "local") return [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] @@ -70,6 +77,7 @@ def get_components_clone_modules(self) -> List[str]: """ Get the modules/subworkflows repository available in a clone of nf-core/modules """ + component_base_path: Path if self.component_type == "modules": component_base_path = Path(self.dir, self.default_modules_path) elif self.component_type == "subworkflows": @@ -87,8 +95,8 @@ def has_valid_directory(self) -> bool: if self.dir is None or not os.path.exists(self.dir): log.error(f"Could not find directory: {self.dir}") return False - main_nf: str = os.path.join(self.dir, "main.nf") - nf_config: str = os.path.join(self.dir, "nextflow.config") + main_nf = os.path.join(self.dir, "main.nf") + nf_config = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): if Path(self.dir).resolve().parts[-1].startswith("nf-core"): raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") From 2b044f80f83941a3571a3dbeed9a5047cc2c0aba Mon Sep 17 00:00:00 2001 From: Novak <116580988+NovakApis@users.noreply.github.com> Date: Tue, 28 Mar 2023 16:42:23 +0200 Subject: [PATCH 04/60] python-typing/components_command.py Ready for nf-core pr --- .github/workflows/my_pytest.yml | 50 ------------------------ nf_core/components/components_command.py | 2 +- 2 files changed, 1 insertion(+), 51 deletions(-) delete mode 100644 .github/workflows/my_pytest.yml diff --git a/.github/workflows/my_pytest.yml b/.github/workflows/my_pytest.yml deleted file mode 100644 index 176395ab0b..0000000000 --- a/.github/workflows/my_pytest.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Python tests -# This workflow is triggered on pushes and PRs to the repository. -# Only run if we changed a Python file -on: - push: - branches: - - feature/** - pull_request: - release: - types: [published] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - pytest: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8"] - - steps: - - uses: actions/checkout@v3 - name: Check out source-code repository - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip -r requirements-dev.txt - pip install -e . - - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 - with: - version: "latest-everything" - - - name: Test with pytest - run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core - - - uses: codecov/codecov-action@v1 - name: Upload code coverage report - with: - if: success() - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 01a2ce0832..2b0a9f017a 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -3,7 +3,7 @@ import os import shutil from pathlib import Path -from typing import Dict, List, Tuple, Union, Optional +from typing import Dict, List, Optional, Tuple, Union import yaml From 31b474d3c7fcaf84eaee596d147ad9b128e9360d Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 28 Mar 2023 17:51:27 +0200 Subject: [PATCH 05/60] add mypy ci check --- .github/workflows/lint-code.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 869d8898d9..6c00070a2f 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -85,3 +85,19 @@ jobs: with: isortVersion: "latest" requirementsFiles: "requirements.txt requirements-dev.txt" + + mypy: + runs-on: ubuntu-latest + steps: + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Checkout + uses: actions/checkout@v3 + - name: Install mypy + run: pip install mypy + - name: Run mypy + uses: sasanquaneuf/mypy-github-action@releases/v1 + with: + checkName: 'mypy' From 47b977947719884f63a5579a7a8b5c55ae375d3d Mon Sep 17 00:00:00 2001 From: Novak <116580988+NovakApis@users.noreply.github.com> Date: Wed, 29 Mar 2023 10:41:16 +0200 Subject: [PATCH 06/60] made some variable types depend on function return value Changed components_utils->get_repo_info to return a tuple of three concrete types instead of a list of three Any types. Implemented suggestion to include self.dir as None. This was done in the __init__ argument typing --- nf_core/components/components_command.py | 5 +---- nf_core/components/components_utils.py | 7 +++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 2b0a9f017a..b7639ac204 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -23,7 +23,7 @@ class ComponentCommand: def __init__( self, component_type: str, - dir: str, + dir: Optional[Union[str, Path]], remote_url: str = None, branch: str = None, no_pull: bool = False, @@ -46,8 +46,6 @@ def _configure_repo_and_paths(self, nf_dir_req: Optional[bool] = True) -> None: Args: nf_dir_req (bool, optional): Whether this command requires being run in the nf-core modules repo or a nf-core pipeline repository. Defaults to True. """ - self.dir: Union[str, Path] - self.repo_type: Optional[str] try: if self.dir: self.dir, self.repo_type, self.org = get_repo_info(self.dir, use_prompt=nf_dir_req) @@ -77,7 +75,6 @@ def get_components_clone_modules(self) -> List[str]: """ Get the modules/subworkflows repository available in a clone of nf-core/modules """ - component_base_path: Path if self.component_type == "modules": component_base_path = Path(self.dir, self.default_modules_path) elif self.component_type == "subworkflows": diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 9a0565296e..410192181e 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -2,6 +2,7 @@ import os import re from pathlib import Path +from typing import Optional, Tuple, Union import questionary import rich.prompt @@ -11,7 +12,9 @@ log = logging.getLogger(__name__) -def get_repo_info(directory, use_prompt=True): +def get_repo_info( + directory: Union[str, Path], use_prompt=True +) -> Tuple[Union[str, Path], Optional[str], Optional[str]]: """ Determine whether this is a pipeline repository or a clone of nf-core/modules @@ -77,7 +80,7 @@ def get_repo_info(directory, use_prompt=True): raise UserWarning("Organisation path could not be established") # It was set on the command line, return what we were given - return [base_dir, repo_type, org] + return (base_dir, repo_type, org) def prompt_component_version_sha(component_name, component_type, modules_repo, installed_sha=None): From ef93b96f0d4103128835df6a3a3c88051b79cda3 Mon Sep 17 00:00:00 2001 From: Novak <116580988+NovakApis@users.noreply.github.com> Date: Wed, 29 Mar 2023 11:03:31 +0200 Subject: [PATCH 07/60] updated __init__ keyword arguments --- nf_core/components/components_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index b7639ac204..f2e2bad884 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -24,8 +24,8 @@ def __init__( self, component_type: str, dir: Optional[Union[str, Path]], - remote_url: str = None, - branch: str = None, + remote_url: Optional[str] = None, + branch: Optional[str] = None, no_pull: bool = False, hide_progress: bool = False, ) -> None: From 4afe175871e2e287f8bf71547656c4705c213d52 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Wed, 29 Mar 2023 12:08:36 +0200 Subject: [PATCH 08/60] make mypy ignore missing imports --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 2380073107..390391dfc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,3 +20,6 @@ norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".githu profile = "black" known_first_party = ["nf_core"] multi_line_output = 3 + +[mypy] +ignore_missing_imports = True From a6791538f8d9ba71a0c94aed34c9322a87bbe11f Mon Sep 17 00:00:00 2001 From: Novak <116580988+NovakApis@users.noreply.github.com> Date: Wed, 29 Mar 2023 13:14:27 +0200 Subject: [PATCH 09/60] mypy configuration edit fixed syntax error for ingroe_missing_imports and added follow_imports="skip" --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 390391dfc7..bf91b3e882 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,4 +22,5 @@ known_first_party = ["nf_core"] multi_line_output = 3 [mypy] -ignore_missing_imports = True +ignore_missing_imports = true +follow_imports = "skip" From a1119f6f1ead0bb646206f46a9e386ae8b903f62 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 10:42:09 +0200 Subject: [PATCH 10/60] use mypy without pre-built action --- .github/workflows/lint-code.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 6c00070a2f..f9955c83bf 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -98,6 +98,4 @@ jobs: - name: Install mypy run: pip install mypy - name: Run mypy - uses: sasanquaneuf/mypy-github-action@releases/v1 - with: - checkName: 'mypy' + run: mypy nf-core/ From ba848af96a6165944a9b1c3be0f9e37f8e57c8c2 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 10:46:36 +0200 Subject: [PATCH 11/60] fix typo in mypy target directory --- .github/workflows/lint-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index f9955c83bf..36ba992228 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -98,4 +98,4 @@ jobs: - name: Install mypy run: pip install mypy - name: Run mypy - run: mypy nf-core/ + run: mypy nf_core/ From 2003c5ef613d5f69e6f7eafb78639e300b28f1f3 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 10:56:00 +0200 Subject: [PATCH 12/60] install missing type annotations for mypy --- .github/workflows/lint-code.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 36ba992228..c24c55b25f 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -96,6 +96,8 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Install mypy - run: pip install mypy + run: | + pip install mypy + mypy --install-types - name: Run mypy run: mypy nf_core/ From a503c167046670511457bc3fae23db8566891cb5 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 11:08:34 +0200 Subject: [PATCH 13/60] install types non-interactive and with a target --- .github/workflows/lint-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index c24c55b25f..916417dab2 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -98,6 +98,6 @@ jobs: - name: Install mypy run: | pip install mypy - mypy --install-types + mypy --install-types --non-interactive nf_core/ - name: Run mypy run: mypy nf_core/ From 3d978de38b3588209922ee2ae25be70b7c8494f2 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 11:27:49 +0200 Subject: [PATCH 14/60] install mypy check requirements explicitly --- .github/workflows/lint-code.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 916417dab2..243ee1de1c 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -98,6 +98,7 @@ jobs: - name: Install mypy run: | pip install mypy - mypy --install-types --non-interactive nf_core/ + pip install mypy + pip install types-Markdown types-PyYAML types-jsonschema types-requests types-setuptools - name: Run mypy run: mypy nf_core/ From e3348df0342b64ab2d7b250c107e6ab91b252988 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 11:31:24 +0200 Subject: [PATCH 15/60] install tools dependencies for mypy checks --- .github/workflows/lint-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 243ee1de1c..8aca396bc4 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -97,7 +97,7 @@ jobs: uses: actions/checkout@v3 - name: Install mypy run: | - pip install mypy + pip install -r requirements.txt -r requirements-dev.txt pip install mypy pip install types-Markdown types-PyYAML types-jsonschema types-requests types-setuptools - name: Run mypy From 325f043c6419dbeb0d3e1c6c9821a756b8613ad5 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 11:36:39 +0200 Subject: [PATCH 16/60] allow redefinition in mypy checks --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index bf91b3e882..538444a7f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,3 +24,4 @@ multi_line_output = 3 [mypy] ignore_missing_imports = true follow_imports = "skip" +allow_redefinition = true From 0b25bfa5729240511f875ca5b880cbd1fe63a9f7 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 11:45:07 +0200 Subject: [PATCH 17/60] ignore redefinitions in mypy checks --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 538444a7f3..47fb6961a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,4 +24,4 @@ multi_line_output = 3 [mypy] ignore_missing_imports = true follow_imports = "skip" -allow_redefinition = true +disable_error_code = no-redef From 3c75ee859239ea7044f2dd9f0e21cfe60d23d1f4 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 11:58:10 +0200 Subject: [PATCH 18/60] use tool prefix for mypy section --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 47fb6961a4..a3898edc45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ profile = "black" known_first_party = ["nf_core"] multi_line_output = 3 -[mypy] +[tool.mypy] ignore_missing_imports = true follow_imports = "skip" disable_error_code = no-redef From 7aa26f549e744fbdaa5c7c9e5e86d39014f11364 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Mon, 3 Apr 2023 12:00:05 +0200 Subject: [PATCH 19/60] make disable error code a string --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a3898edc45..f0702742fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,4 +24,4 @@ multi_line_output = 3 [tool.mypy] ignore_missing_imports = true follow_imports = "skip" -disable_error_code = no-redef +disable_error_code = "no-redef" From e91f64e4c04699881c7dbf38b1af366d5cc8ed12 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:36:17 +0200 Subject: [PATCH 20/60] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove superfluous type hinting Co-authored-by: Júlia Mir Pedrol --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 381f050a48..108aaf3aeb 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -261,7 +261,7 @@ def check_if_in_include_stmts(self, component_path: str) -> Dict[str, List[Dict[ """ include_stmts: Dict[str, List[Dict[str, Union[int, str]]]] = {} if self.repo_type == "pipeline": - workflow_files: Path = Path(self.dir, "workflows").glob("*.nf") + workflow_files = Path(self.dir, "workflows").glob("*.nf") for workflow_file in workflow_files: with open(workflow_file, "r") as fh: # Check if component path is in the file using mmap From 3b754188f9ab113ea36ca540b98988bfc2ec3317 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:37:01 +0200 Subject: [PATCH 21/60] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove superfluous type hinting Co-authored-by: Júlia Mir Pedrol --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 108aaf3aeb..938cdc3e41 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -67,7 +67,7 @@ def get_local_components(self) -> List[str]: """ Get the local modules/subworkflows in a pipeline """ - local_component_dir: Path = Path(self.dir, self.component_type, "local") + local_component_dir = Path(self.dir, self.component_type, "local") return [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] From 594f62f3a7f8b7699491e83c54c62a2ee033c5e5 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:37:14 +0200 Subject: [PATCH 22/60] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove superfluous type hinting Co-authored-by: Júlia Mir Pedrol --- nf_core/components/components_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 938cdc3e41..3c64429c08 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -194,8 +194,8 @@ def check_modules_structure(self) -> None: wrong_location_modules: List[Path] = [] for directory, _, files in os.walk(Path(self.dir, "modules")): if "main.nf" in files: - module_path: Path = Path(directory).relative_to(Path(self.dir, "modules")) - parts: Tuple[str, ...] = module_path.parts + module_path = Path(directory).relative_to(Path(self.dir, "modules")) + parts = module_path.parts # Check that there are modules installed directly under the 'modules' directory if parts[1] == "modules": wrong_location_modules.append(module_path) From 64cef2fdf69200bd98aeec9c0da2a8a07508da7c Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:37:27 +0200 Subject: [PATCH 23/60] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove superfluous type hinting Co-authored-by: Júlia Mir Pedrol --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 3c64429c08..a75ee55094 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -216,7 +216,7 @@ def check_modules_structure(self) -> None: log.info(f"Moved {wrong_dir} to {correct_dir}.") shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) # Regenerate modules.json file - modules_json: ModulesJson = ModulesJson(self.dir) + modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() def check_patch_paths(self, patch_path: Path, module_name: str) -> None: From 929173321accbb141409bf94b60b77467fcb13da Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:37:37 +0200 Subject: [PATCH 24/60] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove superfluous type hinting Co-authored-by: Júlia Mir Pedrol --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index a75ee55094..09aa054650 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -146,7 +146,7 @@ def components_from_repo(self, install_dir: str) -> List[str]: Returns: [str]: The names of the modules/subworkflows """ - repo_dir: Path = Path(self.dir, self.component_type, install_dir) + repo_dir = Path(self.dir, self.component_type, install_dir) if not repo_dir.exists(): raise LookupError(f"Nothing installed from {install_dir} in pipeline") From c39484893a54c69924d999fba39bb51f1a8cd844 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:37:52 +0200 Subject: [PATCH 25/60] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove superfluous type hinting Co-authored-by: Júlia Mir Pedrol --- nf_core/components/components_command.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 09aa054650..c1fcc40e8a 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -209,9 +209,9 @@ def check_modules_structure(self) -> None: ) # Move wrong modules to the right directory for module in wrong_location_modules: - modules_dir: Path = Path("modules").resolve() - correct_dir: Path = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) - wrong_dir: Path = Path(modules_dir, module) + modules_dir = Path("modules").resolve() + correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) + wrong_dir = Path(modules_dir, module) shutil.move(wrong_dir, correct_dir) log.info(f"Moved {wrong_dir} to {correct_dir}.") shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) From f31c933a1279528b32062a4079f6bbe95da5bf13 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Mon, 16 Oct 2023 15:38:11 +0200 Subject: [PATCH 26/60] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove superfluous type hinting Co-authored-by: Júlia Mir Pedrol --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index c1fcc40e8a..e995897596 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -242,7 +242,7 @@ def check_patch_paths(self, patch_path: Path, module_name: str) -> None: for line in lines: fh.write(line) # Update path in modules.json if the file is in the correct format - modules_json: ModulesJson = ModulesJson(self.dir) + modules_json = ModulesJson(self.dir) modules_json.load() if modules_json.has_git_url_and_modules(): modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ From c5c8cda486b03e957d59d82670f9a092118a1f65 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 16 Oct 2023 15:42:15 +0200 Subject: [PATCH 27/60] Try adding mypy to pre-commit config --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7aeeb5bc9..07d3422b44 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,3 +11,7 @@ repos: rev: "v2.7.1" hooks: - id: prettier +- repo: https://github.com/pre-commit/mirrors-mypy + rev: '105' + hooks: + - id: mypy From 681eb5eab2080207bcb38680a74011eea87db9b4 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 16 Oct 2023 15:55:14 +0200 Subject: [PATCH 28/60] Fix invalid pre-commit config --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07d3422b44..4834c3636c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: rev: "v2.7.1" hooks: - id: prettier -- repo: https://github.com/pre-commit/mirrors-mypy + - repo: https://github.com/pre-commit/mirrors-mypy rev: '105' hooks: - id: mypy From fc1be878f14547e493cc1a18c3e1eca6bd423315 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 16 Oct 2023 16:59:48 +0200 Subject: [PATCH 29/60] Revert pre-commit change for now --- .pre-commit-config.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4834c3636c..7e1d94e332 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,4 @@ repos: rev: "v2.7.1" hooks: - id: prettier - - repo: https://github.com/pre-commit/mirrors-mypy - rev: '105' - hooks: - - id: mypy + From 315438d9fcc6feb357907367fe696f474f375bf2 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 16 Oct 2023 17:00:23 +0200 Subject: [PATCH 30/60] Try changing CI to only run mypy on changed files --- .github/workflows/lint-code.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index e25012abfe..c24c42d573 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -86,19 +86,21 @@ jobs: isortVersion: "latest" requirementsFiles: "requirements.txt requirements-dev.txt" - mypy: + static-type-check: runs-on: ubuntu-latest steps: - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Checkout - uses: actions/checkout@v3 - - name: Install mypy - run: | - pip install -r requirements.txt -r requirements-dev.txt - pip install mypy - pip install types-Markdown types-PyYAML types-jsonschema types-requests types-setuptools - - name: Run mypy - run: mypy nf_core/ + - uses: actions/checkout@v2 + - uses: actions/setup-python@v3 + with: + python-version: '3.11' + - run: pip install mypy + - name: Get Python changed files + id: changed-py-files + uses: tj-actions/changed-files@v23 + with: + files: | + *.py + **/*.py + - name: Run if any of the listed files above is changed + if: steps.changed-py-files.outputs.any_changed == 'true' + run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} --ignore-missing-imports From dcaa12415a6ffc264f4a6f6a565c04ad5c543ef1 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 16 Oct 2023 18:24:46 +0200 Subject: [PATCH 31/60] Differentiate dir arg (optional) from dir path --- nf_core/components/components_command.py | 55 ++++++++++++------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index e995897596..da71026859 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -24,7 +24,7 @@ class ComponentCommand: def __init__( self, component_type: str, - dir: Optional[Union[str, Path]], + dir_arg: Optional[Union[str, os.PathLike[str]]], remote_url: Optional[str] = None, branch: Optional[str] = None, no_pull: bool = False, @@ -34,7 +34,7 @@ def __init__( Initialise the ComponentClass object """ self.component_type = component_type - self.dir = dir + self.dir_arg = dir_arg self.modules_repo: ModulesRepo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.hide_progress = hide_progress self._configure_repo_and_paths() @@ -48,8 +48,9 @@ def _configure_repo_and_paths(self, nf_dir_req: Optional[bool] = True) -> None: nf_dir_req (bool, optional): Whether this command requires being run in the nf-core modules repo or a nf-core pipeline repository. Defaults to True. """ try: - if self.dir: - self.dir, self.repo_type, self.org = get_repo_info(self.dir, use_prompt=nf_dir_req) + if self.dir_arg: + self.dir_path: os.PathLike[str] + self.dir_path, self.repo_type, self.org = get_repo_info(self.dir_path, use_prompt=nf_dir_req) else: self.repo_type = None self.org = "" @@ -67,7 +68,7 @@ def get_local_components(self) -> List[str]: """ Get the local modules/subworkflows in a pipeline """ - local_component_dir = Path(self.dir, self.component_type, "local") + local_component_dir = Path(self.dir_path, self.component_type, "local") return [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] @@ -77,9 +78,9 @@ def get_components_clone_modules(self) -> List[str]: Get the modules/subworkflows repository available in a clone of nf-core/modules """ if self.component_type == "modules": - component_base_path = Path(self.dir, self.default_modules_path) + component_base_path = Path(self.dir_path, self.default_modules_path) elif self.component_type == "subworkflows": - component_base_path = Path(self.dir, self.default_subworkflows_path) + component_base_path = Path(self.dir_path, self.default_subworkflows_path) return [ str(Path(dir).relative_to(component_base_path)) for dir, _, files in os.walk(component_base_path) @@ -90,23 +91,23 @@ def has_valid_directory(self) -> bool: """Check that we were given a pipeline or clone of nf-core/modules""" if self.repo_type == "modules": return True - if self.dir is None or not os.path.exists(self.dir): - log.error(f"Could not find directory: {self.dir}") + if self.dir_path is None or not os.path.exists(self.dir_path): + log.error(f"Could not find directory: {self.dir_path}") return False - main_nf = os.path.join(self.dir, "main.nf") - nf_config = os.path.join(self.dir, "nextflow.config") + main_nf = os.path.join(self.dir_path, "main.nf") + nf_config = os.path.join(self.dir_path, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): - if Path(self.dir).resolve().parts[-1].startswith("nf-core"): - raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") - log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + if Path(self.dir_path).resolve().parts[-1].startswith("nf-core"): + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir_path}'") + log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir_path}'") return True def has_modules_file(self) -> None: """Checks whether a module.json file has been created and creates one if it is missing""" - modules_json_path = os.path.join(self.dir, "modules.json") + modules_json_path = os.path.join(self.dir_path, "modules.json") if not os.path.exists(modules_json_path): log.info("Creating missing 'module.json' file.") - ModulesJson(self.dir).create() + ModulesJson(self.dir_path).create() def clear_component_dir(self, component_name: str, component_dir: str) -> bool: """ @@ -121,7 +122,7 @@ def clear_component_dir(self, component_name: str, component_dir: str) -> bool: try: shutil.rmtree(component_dir) # remove all empty directories - for dir_path, dir_names, filenames in os.walk(self.dir, topdown=False): + for dir_path, dir_names, filenames in os.walk(self.dir_path, topdown=False): if not dir_names and not filenames: try: os.rmdir(dir_path) @@ -146,7 +147,7 @@ def components_from_repo(self, install_dir: str) -> List[str]: Returns: [str]: The names of the modules/subworkflows """ - repo_dir = Path(self.dir, self.component_type, install_dir) + repo_dir = Path(self.dir_path, self.component_type, install_dir) if not repo_dir.exists(): raise LookupError(f"Nothing installed from {install_dir} in pipeline") @@ -179,7 +180,7 @@ def load_lint_config(self) -> None: Add parsed config to the `self.lint_config` class attribute. """ - _, tools_config = nf_core.utils.load_tools_config(self.dir) + _, tools_config = nf_core.utils.load_tools_config(self.dir_path) self.lint_config = tools_config.get("lint", {}) def check_modules_structure(self) -> None: @@ -192,9 +193,9 @@ def check_modules_structure(self) -> None: """ if self.repo_type == "pipeline": wrong_location_modules: List[Path] = [] - for directory, _, files in os.walk(Path(self.dir, "modules")): + for directory, _, files in os.walk(Path(self.dir_path, "modules")): if "main.nf" in files: - module_path = Path(directory).relative_to(Path(self.dir, "modules")) + module_path = Path(directory).relative_to(Path(self.dir_path, "modules")) parts = module_path.parts # Check that there are modules installed directly under the 'modules' directory if parts[1] == "modules": @@ -212,11 +213,11 @@ def check_modules_structure(self) -> None: modules_dir = Path("modules").resolve() correct_dir = Path(modules_dir, self.modules_repo.repo_path, Path(*module.parts[2:])) wrong_dir = Path(modules_dir, module) - shutil.move(wrong_dir, correct_dir) + shutil.move(str(wrong_dir), str(correct_dir)) log.info(f"Moved {wrong_dir} to {correct_dir}.") - shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) + shutil.rmtree(Path(self.dir_path, "modules", self.modules_repo.repo_path, "modules")) # Regenerate modules.json file - modules_json = ModulesJson(self.dir) + modules_json = ModulesJson(self.dir_path) modules_json.check_up_to_date() def check_patch_paths(self, patch_path: Path, module_name: str) -> None: @@ -242,12 +243,12 @@ def check_patch_paths(self, patch_path: Path, module_name: str) -> None: for line in lines: fh.write(line) # Update path in modules.json if the file is in the correct format - modules_json = ModulesJson(self.dir) + modules_json = ModulesJson(self.dir_path) modules_json.load() if modules_json.has_git_url_and_modules(): modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ self.modules_repo.repo_path - ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) + ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir_path).resolve())) modules_json.dump() def check_if_in_include_stmts(self, component_path: str) -> Dict[str, List[Dict[str, Union[int, str]]]]: @@ -261,7 +262,7 @@ def check_if_in_include_stmts(self, component_path: str) -> Dict[str, List[Dict[ """ include_stmts: Dict[str, List[Dict[str, Union[int, str]]]] = {} if self.repo_type == "pipeline": - workflow_files = Path(self.dir, "workflows").glob("*.nf") + workflow_files = Path(self.dir_path, "workflows").glob("*.nf") for workflow_file in workflow_files: with open(workflow_file, "r") as fh: # Check if component path is in the file using mmap From 9db5fd5c43b477f9b80369e3dfdac781250c2b55 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Mon, 16 Oct 2023 18:25:38 +0200 Subject: [PATCH 32/60] Declare type within func --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 981895aa88..6873497ebf 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -24,7 +24,7 @@ def get_repo_info( raise UserWarning(f"Could not find directory: {directory}") # Try to find the root directory - base_dir = nf_core.utils.determine_base_dir(directory) + base_dir: Path = nf_core.utils.determine_base_dir(directory) # Figure out the repository type from the .nf-core.yml config file if we can config_fn, tools_config = nf_core.utils.load_tools_config(base_dir) From d9972b6617967355353359841949435762d40e35 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 09:42:39 +0200 Subject: [PATCH 33/60] Try installing MyPy stubs --- .github/workflows/lint-code.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index c24c42d573..6f78d8be53 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -94,6 +94,7 @@ jobs: with: python-version: '3.11' - run: pip install mypy + - run: mypy --install-types - name: Get Python changed files id: changed-py-files uses: tj-actions/changed-files@v23 From 6d8156f4b1eb96bfa1d6695d8592ba24d45540fb Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 09:44:55 +0200 Subject: [PATCH 34/60] Specify missing stubs --- .github/workflows/lint-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 6f78d8be53..eb0ab721bf 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -94,7 +94,7 @@ jobs: with: python-version: '3.11' - run: pip install mypy - - run: mypy --install-types + - run: pip install types-PyYAML - name: Get Python changed files id: changed-py-files uses: tj-actions/changed-files@v23 From 62cb1b90d0b3e73b49b20e74ab7d3b6a19b7347f Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 10:27:19 +0200 Subject: [PATCH 35/60] try passing mypy --- nf_core/components/components_command.py | 21 +++++++++++---------- nf_core/components/components_utils.py | 7 +++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index da71026859..07763bb34d 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -3,9 +3,7 @@ import os import shutil from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union - -import yaml +from typing import Dict, List, Optional, Union import nf_core.utils from nf_core.modules.modules_json import ModulesJson @@ -24,7 +22,7 @@ class ComponentCommand: def __init__( self, component_type: str, - dir_arg: Optional[Union[str, os.PathLike[str]]], + dir_arg: Optional[os.PathLike[str]], remote_url: Optional[str] = None, branch: Optional[str] = None, no_pull: bool = False, @@ -35,7 +33,7 @@ def __init__( """ self.component_type = component_type self.dir_arg = dir_arg - self.modules_repo: ModulesRepo = ModulesRepo(remote_url, branch, no_pull, hide_progress) + self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.hide_progress = hide_progress self._configure_repo_and_paths() @@ -47,9 +45,12 @@ def _configure_repo_and_paths(self, nf_dir_req: Optional[bool] = True) -> None: Args: nf_dir_req (bool, optional): Whether this command requires being run in the nf-core modules repo or a nf-core pipeline repository. Defaults to True. """ + self.dir_path: Union[str, os.PathLike[str]] + self.repo_type: Optional[str] + self.org: Union[str, os.PathLike[str]] + try: if self.dir_arg: - self.dir_path: os.PathLike[str] self.dir_path, self.repo_type, self.org = get_repo_info(self.dir_path, use_prompt=nf_dir_req) else: self.repo_type = None @@ -59,10 +60,10 @@ def _configure_repo_and_paths(self, nf_dir_req: Optional[bool] = True) -> None: raise self.repo_type = None self.org = "" - self.default_modules_path: Path = Path("modules", self.org) - self.default_tests_path: Path = Path("tests", "modules", self.org) - self.default_subworkflows_path: Path = Path("subworkflows", self.org) - self.default_subworkflows_tests_path: Path = Path("tests", "subworkflows", self.org) + self.default_modules_path = Path("modules", self.org) + self.default_tests_path = Path("tests", "modules", self.org) + self.default_subworkflows_path = Path("subworkflows", self.org) + self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org) def get_local_components(self) -> List[str]: """ diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 6873497ebf..91f8be87ff 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -12,19 +12,18 @@ log = logging.getLogger(__name__) -def get_repo_info( - directory: Union[str, Path], use_prompt=True -) -> Tuple[Union[str, Path], Optional[str], Optional[str]]: +def get_repo_info(directory: Union[str, os.PathLike[str]], use_prompt=True): """ Determine whether this is a pipeline repository or a clone of nf-core/modules """ + # Verify that the pipeline dir exists if directory is None or not Path(directory).is_dir(): raise UserWarning(f"Could not find directory: {directory}") # Try to find the root directory - base_dir: Path = nf_core.utils.determine_base_dir(directory) + base_dir: Union[str, os.PathLike[str]] = nf_core.utils.determine_base_dir(directory) # Figure out the repository type from the .nf-core.yml config file if we can config_fn, tools_config = nf_core.utils.load_tools_config(base_dir) From 95eb4b3f47cada7646c76341d5ba977b464b1dc5 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 10:36:05 +0200 Subject: [PATCH 36/60] prettier --- .github/workflows/lint-code.yml | 32 ++++++++++++++++---------------- .pre-commit-config.yaml | 1 - 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index eb0ab721bf..8a96bfe6f5 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -89,19 +89,19 @@ jobs: static-type-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v3 - with: - python-version: '3.11' - - run: pip install mypy - - run: pip install types-PyYAML - - name: Get Python changed files - id: changed-py-files - uses: tj-actions/changed-files@v23 - with: - files: | - *.py - **/*.py - - name: Run if any of the listed files above is changed - if: steps.changed-py-files.outputs.any_changed == 'true' - run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} --ignore-missing-imports + - uses: actions/checkout@v2 + - uses: actions/setup-python@v3 + with: + python-version: "3.11" + - run: pip install mypy + - run: pip install types-PyYAML + - name: Get Python changed files + id: changed-py-files + uses: tj-actions/changed-files@v23 + with: + files: | + *.py + **/*.py + - name: Run if any of the listed files above is changed + if: steps.changed-py-files.outputs.any_changed == 'true' + run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} --ignore-missing-imports diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7e1d94e332..b7aeeb5bc9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,4 +11,3 @@ repos: rev: "v2.7.1" hooks: - id: prettier - From bf60f5f2661e2baaa3bd6e0c192acd132f86a1a1 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 10:37:32 +0200 Subject: [PATCH 37/60] bump python version --- .github/workflows/lint-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 8a96bfe6f5..b8ebf0cac6 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -92,7 +92,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v3 with: - python-version: "3.11" + python-version: "3.8" - run: pip install mypy - run: pip install types-PyYAML - name: Get Python changed files From dba49b3e008bceba7582fa16cbdd8a9fd54de542 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 10:47:59 +0200 Subject: [PATCH 38/60] bugfix --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 07763bb34d..4dae05274b 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -51,7 +51,7 @@ def _configure_repo_and_paths(self, nf_dir_req: Optional[bool] = True) -> None: try: if self.dir_arg: - self.dir_path, self.repo_type, self.org = get_repo_info(self.dir_path, use_prompt=nf_dir_req) + self.dir_path, self.repo_type, self.org = get_repo_info(self.dir_arg, use_prompt=nf_dir_req) else: self.repo_type = None self.org = "" From 73e2e72ad065cddfcb7931e908f2938791e6da54 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 11:20:32 +0200 Subject: [PATCH 39/60] try reverting variable differentiation and instead adapt initial typing of variable to downstream expected function inputs --- nf_core/components/components_command.py | 56 ++++++++++++------------ 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 4dae05274b..2cc8fc72ee 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -22,7 +22,7 @@ class ComponentCommand: def __init__( self, component_type: str, - dir_arg: Optional[os.PathLike[str]], + dir: Union[str, os.PathLike[str]], remote_url: Optional[str] = None, branch: Optional[str] = None, no_pull: bool = False, @@ -32,7 +32,7 @@ def __init__( Initialise the ComponentClass object """ self.component_type = component_type - self.dir_arg = dir_arg + self.dir = dir self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.hide_progress = hide_progress self._configure_repo_and_paths() @@ -45,13 +45,10 @@ def _configure_repo_and_paths(self, nf_dir_req: Optional[bool] = True) -> None: Args: nf_dir_req (bool, optional): Whether this command requires being run in the nf-core modules repo or a nf-core pipeline repository. Defaults to True. """ - self.dir_path: Union[str, os.PathLike[str]] - self.repo_type: Optional[str] - self.org: Union[str, os.PathLike[str]] try: - if self.dir_arg: - self.dir_path, self.repo_type, self.org = get_repo_info(self.dir_arg, use_prompt=nf_dir_req) + if self.dir: + self.dir, self.repo_type, self.org = get_repo_info(self.dir, use_prompt=nf_dir_req) else: self.repo_type = None self.org = "" @@ -69,7 +66,8 @@ def get_local_components(self) -> List[str]: """ Get the local modules/subworkflows in a pipeline """ - local_component_dir = Path(self.dir_path, self.component_type, "local") + self.dir: Union[str, os.PathLike[str]] + local_component_dir = Path(self.dir, self.component_type, "local") return [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] @@ -79,9 +77,9 @@ def get_components_clone_modules(self) -> List[str]: Get the modules/subworkflows repository available in a clone of nf-core/modules """ if self.component_type == "modules": - component_base_path = Path(self.dir_path, self.default_modules_path) + component_base_path = Path(self.dir, self.default_modules_path) elif self.component_type == "subworkflows": - component_base_path = Path(self.dir_path, self.default_subworkflows_path) + component_base_path = Path(self.dir, self.default_subworkflows_path) return [ str(Path(dir).relative_to(component_base_path)) for dir, _, files in os.walk(component_base_path) @@ -92,23 +90,23 @@ def has_valid_directory(self) -> bool: """Check that we were given a pipeline or clone of nf-core/modules""" if self.repo_type == "modules": return True - if self.dir_path is None or not os.path.exists(self.dir_path): - log.error(f"Could not find directory: {self.dir_path}") + if self.dir is None or not os.path.exists(self.dir): + log.error(f"Could not find directory: {self.dir}") return False - main_nf = os.path.join(self.dir_path, "main.nf") - nf_config = os.path.join(self.dir_path, "nextflow.config") + main_nf = os.path.join(self.dir, "main.nf") + nf_config = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): - if Path(self.dir_path).resolve().parts[-1].startswith("nf-core"): - raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir_path}'") - log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir_path}'") + if Path(self.dir).resolve().parts[-1].startswith("nf-core"): + raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") + log.warning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") return True def has_modules_file(self) -> None: """Checks whether a module.json file has been created and creates one if it is missing""" - modules_json_path = os.path.join(self.dir_path, "modules.json") + modules_json_path = os.path.join(self.dir, "modules.json") if not os.path.exists(modules_json_path): log.info("Creating missing 'module.json' file.") - ModulesJson(self.dir_path).create() + ModulesJson(self.dir).create() def clear_component_dir(self, component_name: str, component_dir: str) -> bool: """ @@ -123,7 +121,7 @@ def clear_component_dir(self, component_name: str, component_dir: str) -> bool: try: shutil.rmtree(component_dir) # remove all empty directories - for dir_path, dir_names, filenames in os.walk(self.dir_path, topdown=False): + for dir_path, dir_names, filenames in os.walk(self.dir, topdown=False): if not dir_names and not filenames: try: os.rmdir(dir_path) @@ -148,7 +146,7 @@ def components_from_repo(self, install_dir: str) -> List[str]: Returns: [str]: The names of the modules/subworkflows """ - repo_dir = Path(self.dir_path, self.component_type, install_dir) + repo_dir = Path(self.dir, self.component_type, install_dir) if not repo_dir.exists(): raise LookupError(f"Nothing installed from {install_dir} in pipeline") @@ -181,7 +179,7 @@ def load_lint_config(self) -> None: Add parsed config to the `self.lint_config` class attribute. """ - _, tools_config = nf_core.utils.load_tools_config(self.dir_path) + _, tools_config = nf_core.utils.load_tools_config(self.dir) self.lint_config = tools_config.get("lint", {}) def check_modules_structure(self) -> None: @@ -194,9 +192,9 @@ def check_modules_structure(self) -> None: """ if self.repo_type == "pipeline": wrong_location_modules: List[Path] = [] - for directory, _, files in os.walk(Path(self.dir_path, "modules")): + for directory, _, files in os.walk(Path(self.dir, "modules")): if "main.nf" in files: - module_path = Path(directory).relative_to(Path(self.dir_path, "modules")) + module_path = Path(directory).relative_to(Path(self.dir, "modules")) parts = module_path.parts # Check that there are modules installed directly under the 'modules' directory if parts[1] == "modules": @@ -216,9 +214,9 @@ def check_modules_structure(self) -> None: wrong_dir = Path(modules_dir, module) shutil.move(str(wrong_dir), str(correct_dir)) log.info(f"Moved {wrong_dir} to {correct_dir}.") - shutil.rmtree(Path(self.dir_path, "modules", self.modules_repo.repo_path, "modules")) + shutil.rmtree(Path(self.dir, "modules", self.modules_repo.repo_path, "modules")) # Regenerate modules.json file - modules_json = ModulesJson(self.dir_path) + modules_json = ModulesJson(self.dir) modules_json.check_up_to_date() def check_patch_paths(self, patch_path: Path, module_name: str) -> None: @@ -244,12 +242,12 @@ def check_patch_paths(self, patch_path: Path, module_name: str) -> None: for line in lines: fh.write(line) # Update path in modules.json if the file is in the correct format - modules_json = ModulesJson(self.dir_path) + modules_json = ModulesJson(self.dir) modules_json.load() if modules_json.has_git_url_and_modules(): modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ self.modules_repo.repo_path - ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir_path).resolve())) + ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) modules_json.dump() def check_if_in_include_stmts(self, component_path: str) -> Dict[str, List[Dict[str, Union[int, str]]]]: @@ -263,7 +261,7 @@ def check_if_in_include_stmts(self, component_path: str) -> Dict[str, List[Dict[ """ include_stmts: Dict[str, List[Dict[str, Union[int, str]]]] = {} if self.repo_type == "pipeline": - workflow_files = Path(self.dir_path, "workflows").glob("*.nf") + workflow_files = Path(self.dir, "workflows").glob("*.nf") for workflow_file in workflow_files: with open(workflow_file, "r") as fh: # Check if component path is in the file using mmap From 2c234bdd762fe2b8c0a6250d216540c6a16c7e5b Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 11:31:48 +0200 Subject: [PATCH 40/60] Don't combine typing square bracket syntax with non-subscriptable abc object os.PathLike --- nf_core/components/components_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 2cc8fc72ee..b1a672d700 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -22,7 +22,7 @@ class ComponentCommand: def __init__( self, component_type: str, - dir: Union[str, os.PathLike[str]], + dir: Union[str, os.PathLike], remote_url: Optional[str] = None, branch: Optional[str] = None, no_pull: bool = False, @@ -66,7 +66,7 @@ def get_local_components(self) -> List[str]: """ Get the local modules/subworkflows in a pipeline """ - self.dir: Union[str, os.PathLike[str]] + self.dir: Union[str, os.PathLike] local_component_dir = Path(self.dir, self.component_type, "local") return [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" From 78b078c7eb1c2f7097e53fd441d06a652dc29247 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 11:39:23 +0200 Subject: [PATCH 41/60] Try further simplifying typing to prevent pytest suite from trying to interpret os.PathLike as subscriptable object --- nf_core/components/components_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index b1a672d700..5f6b98b54a 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -22,7 +22,7 @@ class ComponentCommand: def __init__( self, component_type: str, - dir: Union[str, os.PathLike], + dir: str, remote_url: Optional[str] = None, branch: Optional[str] = None, no_pull: bool = False, @@ -66,7 +66,7 @@ def get_local_components(self) -> List[str]: """ Get the local modules/subworkflows in a pipeline """ - self.dir: Union[str, os.PathLike] + self.dir: str local_component_dir = Path(self.dir, self.component_type, "local") return [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" From fbeafa4fc0ab72545e027b5687f4788e414d8928 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 11:46:21 +0200 Subject: [PATCH 42/60] extend last commit to this file --- nf_core/components/components_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 91f8be87ff..bf1d68b00b 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -def get_repo_info(directory: Union[str, os.PathLike[str]], use_prompt=True): +def get_repo_info(directory: str, use_prompt=True): """ Determine whether this is a pipeline repository or a clone of nf-core/modules @@ -23,7 +23,7 @@ def get_repo_info(directory: Union[str, os.PathLike[str]], use_prompt=True): raise UserWarning(f"Could not find directory: {directory}") # Try to find the root directory - base_dir: Union[str, os.PathLike[str]] = nf_core.utils.determine_base_dir(directory) + base_dir: str = nf_core.utils.determine_base_dir(directory) # Figure out the repository type from the .nf-core.yml config file if we can config_fn, tools_config = nf_core.utils.load_tools_config(base_dir) From b91c88e0a067091c9a611abe65d28cd9414c56f4 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 12:07:58 +0200 Subject: [PATCH 43/60] Supplement typing and remove superfluous code --- nf_core/components/components_command.py | 1 - nf_core/components/components_utils.py | 10 ++++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 5f6b98b54a..a3c1947045 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -66,7 +66,6 @@ def get_local_components(self) -> List[str]: """ Get the local modules/subworkflows in a pipeline """ - self.dir: str local_component_dir = Path(self.dir, self.component_type, "local") return [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index bf1d68b00b..30efc09b3c 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,8 +1,7 @@ import logging -import os import re from pathlib import Path -from typing import Optional, Tuple, Union +from typing import Tuple, List, Optional import questionary import rich.prompt @@ -12,7 +11,7 @@ log = logging.getLogger(__name__) -def get_repo_info(directory: str, use_prompt=True): +def get_repo_info(directory: str, use_prompt=True) -> Tuple[str, Optional[str], str]: """ Determine whether this is a pipeline repository or a clone of nf-core/modules @@ -27,7 +26,7 @@ def get_repo_info(directory: str, use_prompt=True): # Figure out the repository type from the .nf-core.yml config file if we can config_fn, tools_config = nf_core.utils.load_tools_config(base_dir) - repo_type = tools_config.get("repository_type", None) + repo_type: Optional[str] = tools_config.get("repository_type", None) # If not set, prompt the user if not repo_type and use_prompt: @@ -57,7 +56,6 @@ def get_repo_info(directory: str, use_prompt=True): raise UserWarning(f"Invalid repository type: '{repo_type}'") # Check for org if modules repo - org = None if repo_type == "pipeline": org = "" elif repo_type == "modules": @@ -128,7 +126,7 @@ def prompt_component_version_sha(component_name, component_type, modules_repo, i return git_sha -def get_components_to_install(subworkflow_dir): +def get_components_to_install(subworkflow_dir) -> Tuple[List[str], List[str]]: """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ From 6b4f9ffd4de665bafa4835a6c4273f6fbdfa6105 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 12:15:21 +0200 Subject: [PATCH 44/60] isort --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 30efc09b3c..bc29cb3a02 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,7 +1,7 @@ import logging import re from pathlib import Path -from typing import Tuple, List, Optional +from typing import List, Optional, Tuple import questionary import rich.prompt From d5c11b540e4d399fa356dfbe1082dbee262e826e Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 12:40:23 +0200 Subject: [PATCH 45/60] supplement typing, 2 errors left --- nf_core/components/components_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index bc29cb3a02..768086fd76 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -7,11 +7,12 @@ import rich.prompt import nf_core.utils +from nf_core.modules.modules_repo import ModulesRepo log = logging.getLogger(__name__) -def get_repo_info(directory: str, use_prompt=True) -> Tuple[str, Optional[str], str]: +def get_repo_info(directory: str, use_prompt: Optional[bool] = True) -> Tuple[str, Optional[str], str]: """ Determine whether this is a pipeline repository or a clone of nf-core/modules @@ -80,7 +81,9 @@ def get_repo_info(directory: str, use_prompt=True) -> Tuple[str, Optional[str], return (base_dir, repo_type, org) -def prompt_component_version_sha(component_name, component_type, modules_repo, installed_sha=None): +def prompt_component_version_sha( + component_name: str, component_type: str, modules_repo: ModulesRepo, installed_sha=None +) -> str: """ Creates an interactive questionary prompt for selecting the module/subworkflow version Args: @@ -126,7 +129,7 @@ def prompt_component_version_sha(component_name, component_type, modules_repo, i return git_sha -def get_components_to_install(subworkflow_dir) -> Tuple[List[str], List[str]]: +def get_components_to_install(subworkflow_dir: str) -> Tuple[List[str], List[str]]: """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ From 3d81425f1c2fc77083a963d66d425f676d4cd0ef Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 12:41:24 +0200 Subject: [PATCH 46/60] 1 error left --- nf_core/components/components_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 768086fd76..646926677b 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -110,7 +110,7 @@ def prompt_component_version_sha( next_page_commits = [next(all_commits, None) for _ in range(10)] next_page_commits = [commit for commit in next_page_commits if commit is not None] if all(commit is None for commit in next_page_commits): - next_page_commits = None + next_page_commits = [] choices = [] for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits): @@ -120,7 +120,7 @@ def prompt_component_version_sha( message += " (installed version)" commit_display = [(display_color, message), ("class:choice-default", "")] choices.append(questionary.Choice(title=commit_display, value=sha)) - if next_page_commits is not None: + if next_page_commits: choices += [older_commits_choice] git_sha = questionary.select( f"Select '{component_name}' commit:", choices=choices, style=nf_core.utils.nfcore_question_style From 5c88dbf5189a85738e9cff6b53acc84c810172b7 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 12:46:15 +0200 Subject: [PATCH 47/60] Try fixing typing error by clarifying list-of-dicts parsing --- nf_core/components/components_utils.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 646926677b..c1c471a83e 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -113,13 +113,16 @@ def prompt_component_version_sha( next_page_commits = [] choices = [] - for title, sha in map(lambda commit: (commit["trunc_message"], commit["git_sha"]), commits): - display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired" - message = f"{title} {sha}" - if installed_sha == sha: - message += " (installed version)" - commit_display = [(display_color, message), ("class:choice-default", "")] - choices.append(questionary.Choice(title=commit_display, value=sha)) + for commit in commits: + if commit: + title = commit["trunc_message"] + sha = commit["git_sha"] + display_color = "fg:ansiblue" if sha != installed_sha else "fg:ansired" + message = f"{title} {sha}" + if installed_sha == sha: + message += " (installed version)" + commit_display = [(display_color, message), ("class:choice-default", "")] + choices.append(questionary.Choice(title=commit_display, value=sha)) if next_page_commits: choices += [older_commits_choice] git_sha = questionary.select( From a7b26b372870ce06a6c171c095f7fc13a8fae087 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 14:29:59 +0200 Subject: [PATCH 48/60] Fix mypy complaints across entire repo --- .github/workflows/lint-code.yml | 15 ++---- nf_core/download.py | 1 - nf_core/modules/modules_repo.py | 3 +- nf_core/synced_repo.py | 3 +- nf_core/utils.py | 94 +++++++-------------------------- pyproject.toml | 1 + 6 files changed, 29 insertions(+), 88 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index b8ebf0cac6..eb88d81ef4 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -86,7 +86,7 @@ jobs: isortVersion: "latest" requirementsFiles: "requirements.txt requirements-dev.txt" - static-type-check: + mypy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -95,13 +95,6 @@ jobs: python-version: "3.8" - run: pip install mypy - run: pip install types-PyYAML - - name: Get Python changed files - id: changed-py-files - uses: tj-actions/changed-files@v23 - with: - files: | - *.py - **/*.py - - name: Run if any of the listed files above is changed - if: steps.changed-py-files.outputs.any_changed == 'true' - run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} --ignore-missing-imports + - run: pip install types-Markdown + - run: pip install types-setuptools + - run: mypy . diff --git a/nf_core/download.py b/nf_core/download.py index 9ca786b5e3..cd71a4c525 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -30,7 +30,6 @@ from nf_core.utils import ( NFCORE_CACHE_DIR, NFCORE_DIR, - SingularityCacheFilePathValidator, ) log = logging.getLogger(__name__) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 152ed7b0c0..9e37f7c74e 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -3,6 +3,7 @@ import os import shutil from pathlib import Path +from typing import Dict import git import rich @@ -34,7 +35,7 @@ class ModulesRepo(SyncedRepo): pull a remote several times in one command. """ - local_repo_statuses = {} + local_repo_statuses = Dict[str, bool] no_pull_global = False def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=False): diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 41e0853f2e..11c8272cf9 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -3,6 +3,7 @@ import os import shutil from pathlib import Path +from typing import Dict import git import rich @@ -61,7 +62,7 @@ class SyncedRepo: An object to store details about a locally cached code repository. """ - local_repo_statuses = {} + local_repo_statuses = Dict[str, bool] no_pull_global = False @staticmethod diff --git a/nf_core/utils.py b/nf_core/utils.py index 3f7d9693b1..d6c0caa9e0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1,7 +1,6 @@ """ Common utility functions for the nf-core python package. """ -import concurrent.futures import datetime import errno import hashlib @@ -52,17 +51,10 @@ ] ) -NFCORE_CACHE_DIR = os.path.join( - os.environ.get("XDG_CACHE_HOME", os.path.join(os.getenv("HOME"), ".cache")), - "nfcore", -) -NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME"), ".config")), "nfcore") - - -def fetch_remote_version(source_url): - response = requests.get(source_url, timeout=3) - remote_version = re.sub(r"[^0-9\.]", "", response.text) - return remote_version +home = os.getenv("HOME") +assert home +NFCORE_CACHE_DIR = os.path.join(os.environ.get("XDG_CACHE_HOME", os.path.join(home, ".cache")), "nfcore") +NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(home, ".config")), "nfcore") def check_if_outdated(current_version=None, remote_version=None, source_url="https://nf-co.re/tools_version"): @@ -71,7 +63,7 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt """ # Exit immediately if disabled via ENV var if os.environ.get("NFCORE_NO_VERSION_CHECK", False): - return (True, "", "") + return True # Set and clean up the current version string if current_version is None: current_version = nf_core.__version__ @@ -79,18 +71,12 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt # Build the URL to check against source_url = os.environ.get("NFCORE_VERSION_URL", source_url) source_url = f"{source_url}?v={current_version}" - # check if we have a newer version without blocking the rest of the script - is_outdated = False - if remote_version is None: # we set it manually for tests - try: - with concurrent.futures.ThreadPoolExecutor() as executor: - future = executor.submit(fetch_remote_version, source_url) - remote_version = future.result() - except Exception as e: - log.debug(f"Could not check for nf-core updates: {e}") - if remote_version is not None: - if Version(remote_version) > Version(current_version): - is_outdated = True + # Fetch and clean up the remote version + if remote_version is None: + response = requests.get(source_url, timeout=3) + remote_version = re.sub(r"[^0-9\.]", "", response.text) + # Check if we have an available update + is_outdated = Version(remote_version) > Version(current_version) return (is_outdated, current_version, remote_version) @@ -258,7 +244,7 @@ def fetch_wf_config(wf_path, cache_config=True): if cache_basedir and cache_fn: cache_path = os.path.join(cache_basedir, cache_fn) - if os.path.isfile(cache_path) and cache_config is True: + if os.path.isfile(cache_path): log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path, "r") as fh: try: @@ -274,7 +260,7 @@ def fetch_wf_config(wf_path, cache_config=True): ul = l.decode("utf-8") try: k, v = ul.split(" = ", 1) - config[k] = v.strip("'\"") + config[k] = v except ValueError: log.debug(f"Couldn't find key=value config pair:\n {ul}") @@ -510,16 +496,9 @@ def safe_get(self, url): if not self.has_init: self.lazy_init() request = self.get(url) - if request.status_code in self.return_retry: - stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors()) - try: - r = self.request_retry(url) - except Exception as e: - stderr.print_exception() - raise e - else: - return r - + if request.status_code not in self.return_ok: + self.log_content_headers(request) + raise AssertionError(f"GitHub API PR failed - got return code {request.status_code} from {url}") return request def get(self, url, **kwargs): @@ -740,7 +719,6 @@ def get_tag_date(tag_date): # Obtain common builds from Docker and Singularity images common_keys = list(all_docker.keys() & all_singularity.keys()) current_date = None - docker_image_name = docker_image["image_name"].lstrip("quay.io/") if docker_image is not None else None for k in common_keys: # Get the most recent common image date = max(all_docker[k]["date"], all_docker[k]["date"]) @@ -748,8 +726,7 @@ def get_tag_date(tag_date): docker_image = all_docker[k]["image"] singularity_image = all_singularity[k]["image"] current_date = date - docker_image_name = docker_image["image_name"].lstrip("quay.io/") - return docker_image_name, singularity_image["image_name"] + return docker_image["image_name"], singularity_image["image_name"] except TypeError: raise LookupError(f"Could not find docker or singularity container for {package}") elif response.status_code != 404: @@ -844,65 +821,34 @@ def prompt_remote_pipeline_name(wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") -def prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False): +def prompt_pipeline_release_branch(wf_releases, wf_branches): """Prompt for pipeline release / branch Args: wf_releases (array): Array of repo releases as returned by the GitHub API wf_branches (array): Array of repo branches, as returned by the GitHub API - multiple (bool): Allow selection of multiple releases & branches (for Tower) Returns: choice (str): Selected release / branch name """ - # Prompt user for release tag, tag_set will contain all available. + # Prompt user for release tag choices = [] - tag_set = [] # Releases if len(wf_releases) > 0: for tag in map(lambda release: release.get("tag_name"), wf_releases): tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] choices.append(questionary.Choice(title=tag_display, value=tag)) - tag_set.append(tag) # Branches for branch in wf_branches.keys(): branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] choices.append(questionary.Choice(title=branch_display, value=branch)) - tag_set.append(branch) if len(choices) == 0: return False - if multiple: - return ( - questionary.checkbox("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask(), - tag_set, - ) - - else: - return ( - questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask(), - tag_set, - ) - - -class SingularityCacheFilePathValidator(questionary.Validator): - """ - Validator for file path specified as --singularity-cache-index argument in nf-core download - """ - - def validate(self, value): - if len(value.text): - if os.path.isfile(value.text): - return True - else: - raise questionary.ValidationError( - message="Invalid remote cache index file", cursor_position=len(value.text) - ) - else: - return True + return questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask() def get_repo_releases_branches(pipeline, wfs): diff --git a/pyproject.toml b/pyproject.toml index f0702742fd..26390b1c30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,3 +25,4 @@ multi_line_output = 3 ignore_missing_imports = true follow_imports = "skip" disable_error_code = "no-redef" +exclude = 'docs' From ce1301800a5924a0af514b1d6f275192267d4400 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 14:32:28 +0200 Subject: [PATCH 49/60] add stubs --- .github/workflows/lint-code.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index eb88d81ef4..fbccb5e7b4 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -97,4 +97,5 @@ jobs: - run: pip install types-PyYAML - run: pip install types-Markdown - run: pip install types-setuptools + - run: pip install types-requests - run: mypy . From 12a8adf6149324aa2a779f6058bd520ef2d07eef Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 14:36:32 +0200 Subject: [PATCH 50/60] isort --- nf_core/download.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index cd71a4c525..c7905b5151 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -27,10 +27,7 @@ import nf_core.list import nf_core.utils from nf_core.synced_repo import RemoteProgressbar, SyncedRepo -from nf_core.utils import ( - NFCORE_CACHE_DIR, - NFCORE_DIR, -) +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR log = logging.getLogger(__name__) stderr = rich.console.Console( From c97ef6e077ba5f4bcc903bb42cf0a39d5e5d0417 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 14:44:32 +0200 Subject: [PATCH 51/60] try reading custom config in GHA --- .github/workflows/lint-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index fbccb5e7b4..d844b601c9 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -98,4 +98,4 @@ jobs: - run: pip install types-Markdown - run: pip install types-setuptools - run: pip install types-requests - - run: mypy . + - run: mypy --config-file pyproject.toml . From ee920c8f23e251002bb40532cc4b13c4ffc058f8 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 14:57:08 +0200 Subject: [PATCH 52/60] Revert last 4 commits --- .github/workflows/lint-code.yml | 16 ++++-- nf_core/download.py | 6 ++- nf_core/modules/modules_repo.py | 3 +- nf_core/synced_repo.py | 3 +- nf_core/utils.py | 94 ++++++++++++++++++++++++++------- pyproject.toml | 1 - 6 files changed, 92 insertions(+), 31 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index d844b601c9..b8ebf0cac6 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -86,7 +86,7 @@ jobs: isortVersion: "latest" requirementsFiles: "requirements.txt requirements-dev.txt" - mypy: + static-type-check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -95,7 +95,13 @@ jobs: python-version: "3.8" - run: pip install mypy - run: pip install types-PyYAML - - run: pip install types-Markdown - - run: pip install types-setuptools - - run: pip install types-requests - - run: mypy --config-file pyproject.toml . + - name: Get Python changed files + id: changed-py-files + uses: tj-actions/changed-files@v23 + with: + files: | + *.py + **/*.py + - name: Run if any of the listed files above is changed + if: steps.changed-py-files.outputs.any_changed == 'true' + run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} --ignore-missing-imports diff --git a/nf_core/download.py b/nf_core/download.py index c7905b5151..9ca786b5e3 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -27,7 +27,11 @@ import nf_core.list import nf_core.utils from nf_core.synced_repo import RemoteProgressbar, SyncedRepo -from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR +from nf_core.utils import ( + NFCORE_CACHE_DIR, + NFCORE_DIR, + SingularityCacheFilePathValidator, +) log = logging.getLogger(__name__) stderr = rich.console.Console( diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 9e37f7c74e..152ed7b0c0 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -3,7 +3,6 @@ import os import shutil from pathlib import Path -from typing import Dict import git import rich @@ -35,7 +34,7 @@ class ModulesRepo(SyncedRepo): pull a remote several times in one command. """ - local_repo_statuses = Dict[str, bool] + local_repo_statuses = {} no_pull_global = False def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=False): diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 11c8272cf9..41e0853f2e 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -3,7 +3,6 @@ import os import shutil from pathlib import Path -from typing import Dict import git import rich @@ -62,7 +61,7 @@ class SyncedRepo: An object to store details about a locally cached code repository. """ - local_repo_statuses = Dict[str, bool] + local_repo_statuses = {} no_pull_global = False @staticmethod diff --git a/nf_core/utils.py b/nf_core/utils.py index d6c0caa9e0..3f7d9693b1 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1,6 +1,7 @@ """ Common utility functions for the nf-core python package. """ +import concurrent.futures import datetime import errno import hashlib @@ -51,10 +52,17 @@ ] ) -home = os.getenv("HOME") -assert home -NFCORE_CACHE_DIR = os.path.join(os.environ.get("XDG_CACHE_HOME", os.path.join(home, ".cache")), "nfcore") -NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(home, ".config")), "nfcore") +NFCORE_CACHE_DIR = os.path.join( + os.environ.get("XDG_CACHE_HOME", os.path.join(os.getenv("HOME"), ".cache")), + "nfcore", +) +NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME"), ".config")), "nfcore") + + +def fetch_remote_version(source_url): + response = requests.get(source_url, timeout=3) + remote_version = re.sub(r"[^0-9\.]", "", response.text) + return remote_version def check_if_outdated(current_version=None, remote_version=None, source_url="https://nf-co.re/tools_version"): @@ -63,7 +71,7 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt """ # Exit immediately if disabled via ENV var if os.environ.get("NFCORE_NO_VERSION_CHECK", False): - return True + return (True, "", "") # Set and clean up the current version string if current_version is None: current_version = nf_core.__version__ @@ -71,12 +79,18 @@ def check_if_outdated(current_version=None, remote_version=None, source_url="htt # Build the URL to check against source_url = os.environ.get("NFCORE_VERSION_URL", source_url) source_url = f"{source_url}?v={current_version}" - # Fetch and clean up the remote version - if remote_version is None: - response = requests.get(source_url, timeout=3) - remote_version = re.sub(r"[^0-9\.]", "", response.text) - # Check if we have an available update - is_outdated = Version(remote_version) > Version(current_version) + # check if we have a newer version without blocking the rest of the script + is_outdated = False + if remote_version is None: # we set it manually for tests + try: + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(fetch_remote_version, source_url) + remote_version = future.result() + except Exception as e: + log.debug(f"Could not check for nf-core updates: {e}") + if remote_version is not None: + if Version(remote_version) > Version(current_version): + is_outdated = True return (is_outdated, current_version, remote_version) @@ -244,7 +258,7 @@ def fetch_wf_config(wf_path, cache_config=True): if cache_basedir and cache_fn: cache_path = os.path.join(cache_basedir, cache_fn) - if os.path.isfile(cache_path): + if os.path.isfile(cache_path) and cache_config is True: log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path, "r") as fh: try: @@ -260,7 +274,7 @@ def fetch_wf_config(wf_path, cache_config=True): ul = l.decode("utf-8") try: k, v = ul.split(" = ", 1) - config[k] = v + config[k] = v.strip("'\"") except ValueError: log.debug(f"Couldn't find key=value config pair:\n {ul}") @@ -496,9 +510,16 @@ def safe_get(self, url): if not self.has_init: self.lazy_init() request = self.get(url) - if request.status_code not in self.return_ok: - self.log_content_headers(request) - raise AssertionError(f"GitHub API PR failed - got return code {request.status_code} from {url}") + if request.status_code in self.return_retry: + stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors()) + try: + r = self.request_retry(url) + except Exception as e: + stderr.print_exception() + raise e + else: + return r + return request def get(self, url, **kwargs): @@ -719,6 +740,7 @@ def get_tag_date(tag_date): # Obtain common builds from Docker and Singularity images common_keys = list(all_docker.keys() & all_singularity.keys()) current_date = None + docker_image_name = docker_image["image_name"].lstrip("quay.io/") if docker_image is not None else None for k in common_keys: # Get the most recent common image date = max(all_docker[k]["date"], all_docker[k]["date"]) @@ -726,7 +748,8 @@ def get_tag_date(tag_date): docker_image = all_docker[k]["image"] singularity_image = all_singularity[k]["image"] current_date = date - return docker_image["image_name"], singularity_image["image_name"] + docker_image_name = docker_image["image_name"].lstrip("quay.io/") + return docker_image_name, singularity_image["image_name"] except TypeError: raise LookupError(f"Could not find docker or singularity container for {package}") elif response.status_code != 404: @@ -821,34 +844,65 @@ def prompt_remote_pipeline_name(wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") -def prompt_pipeline_release_branch(wf_releases, wf_branches): +def prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False): """Prompt for pipeline release / branch Args: wf_releases (array): Array of repo releases as returned by the GitHub API wf_branches (array): Array of repo branches, as returned by the GitHub API + multiple (bool): Allow selection of multiple releases & branches (for Tower) Returns: choice (str): Selected release / branch name """ - # Prompt user for release tag + # Prompt user for release tag, tag_set will contain all available. choices = [] + tag_set = [] # Releases if len(wf_releases) > 0: for tag in map(lambda release: release.get("tag_name"), wf_releases): tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] choices.append(questionary.Choice(title=tag_display, value=tag)) + tag_set.append(tag) # Branches for branch in wf_branches.keys(): branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] choices.append(questionary.Choice(title=branch_display, value=branch)) + tag_set.append(branch) if len(choices) == 0: return False - return questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask() + if multiple: + return ( + questionary.checkbox("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask(), + tag_set, + ) + + else: + return ( + questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask(), + tag_set, + ) + + +class SingularityCacheFilePathValidator(questionary.Validator): + """ + Validator for file path specified as --singularity-cache-index argument in nf-core download + """ + + def validate(self, value): + if len(value.text): + if os.path.isfile(value.text): + return True + else: + raise questionary.ValidationError( + message="Invalid remote cache index file", cursor_position=len(value.text) + ) + else: + return True def get_repo_releases_branches(pipeline, wfs): diff --git a/pyproject.toml b/pyproject.toml index 26390b1c30..f0702742fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,4 +25,3 @@ multi_line_output = 3 ignore_missing_imports = true follow_imports = "skip" disable_error_code = "no-redef" -exclude = 'docs' From 989f1b7927f10d3e15c18921dffd9a9faef23ddc Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:58:15 +0200 Subject: [PATCH 53/60] Update .github/workflows/lint-code.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "because we can" Co-authored-by: Matthias Hörtenhuber --- .github/workflows/lint-code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index b8ebf0cac6..8a96bfe6f5 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -92,7 +92,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v3 with: - python-version: "3.8" + python-version: "3.11" - run: pip install mypy - run: pip install types-PyYAML - name: Get Python changed files From 56b6a08f832bc646a347d708f198ca74b8b1ee8c Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:59:05 +0200 Subject: [PATCH 54/60] Update .github/workflows/lint-code.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .github/workflows/lint-code.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 8a96bfe6f5..d0586fdf86 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -93,8 +93,7 @@ jobs: - uses: actions/setup-python@v3 with: python-version: "3.11" - - run: pip install mypy - - run: pip install types-PyYAML + - run: pip install mypy types-PyYAML - name: Get Python changed files id: changed-py-files uses: tj-actions/changed-files@v23 From b518c1c2ce97982769d2ffdf8e01d2f8c9d7c6a9 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:59:53 +0200 Subject: [PATCH 55/60] Update nf_core/components/components_command.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no context for this arg ever being undefined Co-authored-by: Matthias Hörtenhuber --- nf_core/components/components_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index a3c1947045..fdfebde735 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -37,7 +37,7 @@ def __init__( self.hide_progress = hide_progress self._configure_repo_and_paths() - def _configure_repo_and_paths(self, nf_dir_req: Optional[bool] = True) -> None: + def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None: """ Determine the repo type and set some default paths. If this is a modules repo, determine the org_path too. From 4e71fb8c46e6ec22ec4534cf1dd80e1a385ab150 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Tue, 17 Oct 2023 16:00:36 +0200 Subject: [PATCH 56/60] Update nf_core/components/components_utils.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index c1c471a83e..ae29695c7f 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -82,7 +82,7 @@ def get_repo_info(directory: str, use_prompt: Optional[bool] = True) -> Tuple[st def prompt_component_version_sha( - component_name: str, component_type: str, modules_repo: ModulesRepo, installed_sha=None + component_name: str, component_type: str, modules_repo: ModulesRepo, installed_sha: Optional[str] =None ) -> str: """ Creates an interactive questionary prompt for selecting the module/subworkflow version From 7edf84fa278be7880e8a535591c1e3d344e57bf4 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar <89784800+kedhammar@users.noreply.github.com> Date: Tue, 17 Oct 2023 16:04:57 +0200 Subject: [PATCH 57/60] Update nf_core/components/components_utils.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change typing based on nf_core.utils.determine_base_dir() outputs. A bit messy, but let's not increase the scope further. Co-authored-by: Matthias Hörtenhuber --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index ae29695c7f..6244189839 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -23,7 +23,7 @@ def get_repo_info(directory: str, use_prompt: Optional[bool] = True) -> Tuple[st raise UserWarning(f"Could not find directory: {directory}") # Try to find the root directory - base_dir: str = nf_core.utils.determine_base_dir(directory) + base_dir: Union[str, Path] = nf_core.utils.determine_base_dir(directory) # Figure out the repository type from the .nf-core.yml config file if we can config_fn, tools_config = nf_core.utils.load_tools_config(base_dir) From ddf4fda1b2f3073ef6b0d1dc5624109e3f3086d6 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 16:11:18 +0200 Subject: [PATCH 58/60] import missing typing obj --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 6244189839..6b53e04783 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,7 +1,7 @@ import logging import re from pathlib import Path -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union import questionary import rich.prompt From e88d72138b8512d38ae3a16ebfd23ae7da3ea170 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 16:11:48 +0200 Subject: [PATCH 59/60] black --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 6b53e04783..cd446fe258 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -82,7 +82,7 @@ def get_repo_info(directory: str, use_prompt: Optional[bool] = True) -> Tuple[st def prompt_component_version_sha( - component_name: str, component_type: str, modules_repo: ModulesRepo, installed_sha: Optional[str] =None + component_name: str, component_type: str, modules_repo: ModulesRepo, installed_sha: Optional[str] = None ) -> str: """ Creates an interactive questionary prompt for selecting the module/subworkflow version From 9171fa03154bf954697ee12fc004a9737eef3367 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 17 Oct 2023 16:19:53 +0200 Subject: [PATCH 60/60] revert minor tweak to prevent breaking --- nf_core/components/components_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index cd446fe258..eec533ce60 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,7 +1,7 @@ import logging import re from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple import questionary import rich.prompt @@ -23,7 +23,7 @@ def get_repo_info(directory: str, use_prompt: Optional[bool] = True) -> Tuple[st raise UserWarning(f"Could not find directory: {directory}") # Try to find the root directory - base_dir: Union[str, Path] = nf_core.utils.determine_base_dir(directory) + base_dir: str = nf_core.utils.determine_base_dir(directory) # Figure out the repository type from the .nf-core.yml config file if we can config_fn, tools_config = nf_core.utils.load_tools_config(base_dir)