Merge branch 'dev' into modules-lint-sort

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 {
     "name": "nfcore",
     "image": "nfcore/gitpod:latest",
-    "postCreateCommand": "python -m pip install --upgrade -r ../requirements-dev.txt -e ../",
+    "postCreateCommand": "python -m pip install --upgrade -r ../requirements-dev.txt -e ../ && pre-commit install --install-hooks",
     "remoteUser": "gitpod",
 
     // Configure tool-specific properties.

.github/workflows/pytest-frozen-ubuntu-20.04.yml

@@ -0,0 +1,52 @@
+name: Python tests Ubuntu-20.04 (frozen)
+# This workflow is triggered on pushes and PRs to the repository.
+# Only run if we changed a Python file
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+  release:
+    types: [published]
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pytest:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+        name: Check out source-code repository
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+
+      - name: Install python dependencies
+        run: |
+          python -m pip install --upgrade pip -r requirements-dev.txt
+          pip install -e .
+
+      - name: Downgrade git to the Ubuntu official repository's version
+        run: |
+          sudo apt remove git git-man
+          sudo add-apt-repository --remove ppa:git-core/ppa
+          sudo apt install git
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v1
+        with:
+          version: "latest-everything"
+
+      - name: Test with pytest
+        run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core
+
+      - uses: codecov/codecov-action@v1
+        name: Upload code coverage report
+        with:
+          if: success()
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitpod.yml

@@ -1,7 +1,10 @@
 image: nfcore/gitpod:latest
 tasks:
-  - name: install current state of nf-core/tools
-    command: python -m pip install --upgrade -r requirements-dev.txt -e .
+  - name: install current state of nf-core/tools and setup pre-commit
+    command: |
+      python -m pip install -e .
+      python -m pip install -r requirements-dev.txt
+      pre-commit install --install-hooks
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack
     - codezombiech.gitignore # Language support for .gitignore files

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 22.3.0
     hooks:
       - id: black
-        language_version: python3.10
+        language_version: python3.9
   - repo: https://github.com/pycqa/isort
     rev: 5.10.1
     hooks:

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Template
 
+- Ignore files in `bin/` directory when running prettier.
 - Fix lint warnings for `samplesheet_check.nf` module
 - Add codespaces template ([#1957](https://github.com/nf-core/tools/pull/1957))
 - Check that the workflow name provided with a template doesn't contain dashes ([#1822](https://github.com/nf-core/tools/pull/1822))
@@ -20,6 +21,7 @@
 
 ### General
 
+- Use pre-commit run prettier if prettier is not available ([#1983](https://github.com/nf-core/tools/pull/1983)) and initialize pre-commit in gitpod and codespaces.
 - Refactor CLI flag `--hide-progress` to be at the top-level group, like `--verbose` ([#2016](https://github.com/nf-core/tools/pull/2016))
 - Fix error in tagging GitPod docker images during releases
 - `nf-core sync` now supports the template YAML file using `-t/--template-yaml`.
@@ -32,15 +34,18 @@
 - Add file `versions.yml` when generating `test.yml` with `nf-core modules create-test-yml` but don't check for md5sum [#1963](https://github.com/nf-core/tools/pull/1963)
 - Mock biocontainers and anaconda api calls in modules and subworkflows tests [#1967](https://github.com/nf-core/tools/pull/1967)
 - Run tests with Python 3.11 ([#1970](https://github.com/nf-core/tools/pull/1970))
+- Run test with a realistic version of git ([#2043](https://github.com/nf-core/tools/pull/2043))
 - Bump promoted Python version from 3.7 to 3.8 ([#1971](https://github.com/nf-core/tools/pull/1971))
 - Fix incorrect file deletion in `nf-core launch` when `--params_in` has the same name as `--params_out`
 - Updated GitHub actions ([#1998](https://github.com/nf-core/tools/pull/1998), [#2001](https://github.com/nf-core/tools/pull/2001))
-- Code maintenance ([#1818](https://github.com/nf-core/tools/pull/1818), [#2032](https://github.com/nf-core/tools/pull/2032))
+- Code maintenance ([#1818](https://github.com/nf-core/tools/pull/1818), [#2032](https://github.com/nf-core/tools/pull/2032), [#2073](https://github.com/nf-core/tools/pull/2073))
 - Track from where modules and subworkflows are installed ([#1999](https://github.com/nf-core/tools/pull/1999))
 - Substitute ModulesCommand and SubworkflowsCommand by ComponentsCommand ([#2000](https://github.com/nf-core/tools/pull/2000))
 - Don't print source file + line number on logging messages (except when verbose) ([#2015](https://github.com/nf-core/tools/pull/2015))
 - Extended the chat notifications to Slack ([#1829](https://github.com/nf-core/tools/pull/1829))
 - Allow other remote URLs not starting with `http` ([#2061](https://github.com/nf-core/tools/pull/2061))
+- Prevent installation with unsupported Python versions ([#2075](https://github.com/nf-core/tools/pull/2075))
+- Automatically format `test.yml` content with Prettier ([#2078](https://github.com/nf-core/tools/pull/2078))
 
 ### Modules
 

README.md

@@ -214,6 +214,22 @@ pip install --upgrade nf-core
 
 Please refer to the respective documentation for further details to manage packages, as for example [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#updating-packages) or [pip](https://packaging.python.org/en/latest/tutorials/installing-packages/#upgrading-packages).
 
+### Activate shell completions for nf-core/tools
+
+Auto-completion for the `nf-core` command is available for bash, zsh and fish. To activate it, add the following lines to the respective shell config files.
+shell | shell config file | command
+--- | --- | ---
+bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"`
+Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"`
+fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)`
+
+After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options.
+
+> **NB:** The added line will run the command `nf-core` (which will also slow down startup time of your shell). You should therefore either have the nf-core/tools installed globally.
+> You can also wrap it inside `if type nf-core > /dev/null; then ` \<YOUR EVAL CODE LINE\> `fi` for bash and zsh or `if command -v nf-core &> /dev/null eval (env _NF_CORE_COMPLETE=fish_source nf-core) end` for fish. You need to then source the config in your environment for the completions to be activated.
+
+> **NB:** If you see the error `command not found compdef` , be sure that your config file contains the line `autoload -Uz compinit && compinit` before the eval line.
+
 ## Listing pipelines
 
 The command `nf-core list` shows all available nf-core pipelines along with their latest version, when that was published and how recently the pipeline code was pulled to your local system (if at all).

nf_core/__main__.py

@@ -79,27 +79,28 @@
 
 
 def run_nf_core():
-    # Print nf-core header
-    stderr.print(f"\n[green]{' ' * 42},--.[grey39]/[green],-.", highlight=False)
-    stderr.print("[blue]          ___     __   __   __   ___     [green]/,-._.--~\\", highlight=False)
-    stderr.print(r"[blue]    |\ | |__  __ /  ` /  \ |__) |__      [yellow]   }  {", highlight=False)
-    stderr.print(r"[blue]    | \| |       \__, \__/ |  \ |___     [green]\`-._,-`-,", highlight=False)
-    stderr.print("[green]                                          `._,._,'\n", highlight=False)
-    stderr.print(
-        f"[grey39]    nf-core/tools version {nf_core.__version__} - [link=https://nf-co.re]https://nf-co.re[/]",
-        highlight=False,
-    )
-    try:
-        is_outdated, _, remote_vers = nf_core.utils.check_if_outdated()
-        if is_outdated:
-            stderr.print(
-                f"[bold bright_yellow]    There is a new version of nf-core/tools available! ({remote_vers})",
-                highlight=False,
-            )
-    except Exception as e:
-        log.debug(f"Could not check latest version: {e}")
-    stderr.print("\n")
-
+    # print nf-core header if environment variable is not set
+    if os.environ.get("_NF_CORE_COMPLETE") is None:
+        # Print nf-core header
+        stderr.print(f"\n[green]{' ' * 42},--.[grey39]/[green],-.", highlight=False)
+        stderr.print("[blue]          ___     __   __   __   ___     [green]/,-._.--~\\", highlight=False)
+        stderr.print(r"[blue]    |\ | |__  __ /  ` /  \ |__) |__      [yellow]   }  {", highlight=False)
+        stderr.print(r"[blue]    | \| |       \__, \__/ |  \ |___     [green]\`-._,-`-,", highlight=False)
+        stderr.print("[green]                                          `._,._,'\n", highlight=False)
+        stderr.print(
+            f"[grey39]    nf-core/tools version {nf_core.__version__} - [link=https://nf-co.re]https://nf-co.re[/]",
+            highlight=False,
+        )
+        try:
+            is_outdated, _, remote_vers = nf_core.utils.check_if_outdated()
+            if is_outdated:
+                stderr.print(
+                    f"[bold bright_yellow]    There is a new version of nf-core/tools available! ({remote_vers})",
+                    highlight=False,
+                )
+        except Exception as e:
+            log.debug(f"Could not check latest version: {e}")
+        stderr.print("\n")
     # Launch the click cli
     nf_core_cli(auto_envvar_prefix="NFCORE")
 

nf_core/components/components_utils.py

@@ -69,7 +69,7 @@ def get_repo_info(directory, use_prompt=True):
             ).unsafe_ask()
             log.info("To avoid this prompt in the future, add the 'org_path' key to a root '%s' file.", config_fn.name)
             if rich.prompt.Confirm.ask("[bold][blue]?[/] Would you like me to add this config now?", default=True):
-                with open(os.path.join(dir, ".nf-core.yml"), "a+") as fh:
+                with open(config_fn, "a+") as fh:
                     fh.write(f"org_path: {org}\n")
                     log.info(f"Config added to '{config_fn.name}'")
 

nf_core/components/info.py

@@ -103,13 +103,13 @@ def init_mod_name(self, component):
                     components = self.get_components_clone_modules()
                 else:
                     components = self.modules_json.get_all_components(self.component_type).get(
-                        self.modules_repo.remote_url
+                        self.modules_repo.remote_url, {}
                     )
                     components = [
                         component if directory == self.modules_repo.repo_path else f"{directory}/{component}"
                         for directory, component in components
                     ]
-                    if components is None:
+                    if not components:
                         raise UserWarning(
                             f"No {self.component_type[:-1]} installed from '{self.modules_repo.remote_url}'"
                         )

nf_core/components/remove.py

@@ -161,10 +161,8 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals
             else:
                 log.info(f"Removed files for '{component}'.")
         else:
-            installed_by = modules_json.modules_json["repos"][self.modules_repo.remote_url][self.component_type][
-                repo_path
-            ][component]["installed_by"]
-            if installed_by == self.component_type:
+            installed_by = modules_json.get_installed_by_entries(self.component_type, component)
+            if installed_by == [self.component_type]:
                 log.error(
                     f"Did not remove '{component}', because it was also manually installed. Only updated 'installed_by' entry in modules.json."
                 )

nf_core/components/update.py

@@ -9,7 +9,12 @@
 import nf_core.modules.modules_utils
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
-from nf_core.components.components_utils import prompt_component_version_sha
+from nf_core.components.components_utils import (
+    get_components_to_install,
+    prompt_component_version_sha,
+)
+from nf_core.components.install import ComponentInstall
+from nf_core.components.remove import ComponentRemove
 from nf_core.modules.modules_differ import ModulesDiffer
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
@@ -234,7 +239,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                         else:
                             updated.append(component)
                     recursive_update = True
-                    modules_to_update, subworkflows_to_update = self.get_components_to_update(component, modules_repo)
+                    modules_to_update, subworkflows_to_update = self.get_components_to_update(component)
                     if not silent and len(modules_to_update + subworkflows_to_update) > 0:
                         log.warning(
                             f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be added to the same diff file.\n"
@@ -254,6 +259,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                         self.update_linked_components(
                             modules_to_update, subworkflows_to_update, updated, check_diff_exist=False
                         )
+                        self.manage_changes_in_linked_components(component, modules_to_update, subworkflows_to_update)
 
                 elif self.show_diff:
                     ModulesDiffer.print_diff(
@@ -278,10 +284,10 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                 # Clear the component directory and move the installed files there
                 self.move_files_from_tmp_dir(component, install_tmp_dir, modules_repo.repo_path, version)
                 # Update modules.json with newly installed component
-                self.modules_json.update(self.component_type, modules_repo, component, version, self.component_type)
+                self.modules_json.update(self.component_type, modules_repo, component, version, installed_by=None)
                 updated.append(component)
                 recursive_update = True
-                modules_to_update, subworkflows_to_update = self.get_components_to_update(component, modules_repo)
+                modules_to_update, subworkflows_to_update = self.get_components_to_update(component)
                 if not silent and not self.update_all and len(modules_to_update + subworkflows_to_update) > 0:
                     log.warning(
                         f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n"
@@ -299,10 +305,16 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                 if recursive_update and len(modules_to_update + subworkflows_to_update) > 0:
                     # Update linked components
                     self.update_linked_components(modules_to_update, subworkflows_to_update, updated)
+                    self.manage_changes_in_linked_components(component, modules_to_update, subworkflows_to_update)
             else:
                 # Don't save to a file, just iteratively update the variable
                 self.modules_json.update(
-                    self.component_type, modules_repo, component, version, self.component_type, write_file=False
+                    self.component_type,
+                    modules_repo,
+                    component,
+                    version,
+                    installed_by=None,
+                    write_file=False,
                 )
 
         if self.save_diff_fn:
@@ -819,7 +831,7 @@ def try_apply_patch(
 
         return True
 
-    def get_components_to_update(self, component, modules_repo):
+    def get_components_to_update(self, component):
         """
         Get all modules and subworkflows linked to the updated component.
 
@@ -829,9 +841,7 @@ def get_components_to_update(self, component, modules_repo):
         mods_json = self.modules_json.get_modules_json()
         modules_to_update = []
         subworkflows_to_update = []
-        installed_by = mods_json["repos"][modules_repo.remote_url][self.component_type][modules_repo.repo_path][
-            component
-        ]["installed_by"]
+        installed_by = self.modules_json.get_installed_by_entries(self.component_type, component)
 
         if self.component_type == "modules":
             # All subworkflow names in the installed_by section of a module are subworkflows using this module
@@ -871,6 +881,34 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up
             self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist)
             self._reset_component_type(original_component_type, original_update_all)
 
+    def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update):
+        """Check for linked components added or removed in the new subworkflow version"""
+        if self.component_type == "subworkflows":
+            subworkflow_directory = Path(self.dir, self.component_type, self.modules_repo.repo_path, component)
+            included_modules, included_subworkflows = get_components_to_install(subworkflow_directory)
+            # If a new module/subworkflow is included in the subworklfow and wasn't included before
+            for module in included_modules:
+                if module not in modules_to_update:
+                    log.info(f"Installing newly included module '{module}' for '{component}'")
+                    install_module_object = ComponentInstall(self.dir, "modules", installed_by=component)
+                    install_module_object.install(module, silent=True)
+            for subworkflow in included_subworkflows:
+                if subworkflow not in subworkflows_to_update:
+                    log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'")
+                    install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component)
+                    install_subworkflow_object.install(subworkflow, silent=True)
+            # If a module/subworkflow has been removed from the subworkflow
+            for module in modules_to_update:
+                if module not in included_modules:
+                    log.info(f"Removing module '{module}' which is not included in '{component}' anymore.")
+                    remove_module_object = ComponentRemove("modules", self.dir)
+                    remove_module_object.remove(module, removed_by=component)
+            for subworkflow in subworkflows_to_update:
+                if subworkflow not in included_subworkflows:
+                    log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.")
+                    remove_subworkflow_object = ComponentRemove("subworkflows", self.dir)
+                    remove_subworkflow_object.remove(subworkflow, removed_by=component)
+
     def _change_component_type(self, new_component_type):
         original_component_type = self.component_type
         self.component_type = new_component_type