Merge branch 'dev' into add-openjdk-in-docker-container

nf-core · Dec 14, 2023 · aea4254 · aea4254
2 parents ee8c557 + d1994a7
commit aea4254
Show file tree

Hide file tree

Showing 12 changed files with 173 additions and 88 deletions.
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -39,16 +39,6 @@ jobs:
         id: conditions
         run: echo "run-tests=${{ github.ref == 'refs/heads/master' || (matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8') }}" >> $GITHUB_ENV
 
-      - name: Check out source-code repository
-        uses: actions/checkout@v4
-        if: ${{ env.run-tests == 'true' }}
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
-        if: ${{ env.run-tests == 'true' }}
     outputs:
       python-version: ${{ matrix.python-version }}
       runner: ${{ matrix.runner }}
@@ -97,7 +87,7 @@ jobs:
           path: |
             /usr/local/bin/nf-test
             /home/runner/.nf-test/nf-test.jar
-          key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest
+          key: ${{ runner.os }}-nftest-${{ env.date }}
 
       - name: Install nf-test
         if: steps.cache-software.outputs.cache-hit != 'true'

diff --git a/.gitpod.yml b/.gitpod.yml
@@ -6,6 +6,9 @@ tasks:
       python -m pip install -r requirements-dev.txt
       pre-commit install --install-hooks
       nextflow self-update
+  - name: unset JAVA_TOOL_OPTIONS
+    command: |
+      unset JAVA_TOOL_OPTIONS
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack
     - codezombiech.gitignore # Language support for .gitignore files

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@
 - Fancier syntax highlighting for example samplesheets in the usage.md template ([#2503](https://github.com/nf-core/tools/pull/2503))
 - Use closure for multiqc ext.args ([#2509](https://github.com/nf-core/tools/pull/2509))
 - Fix how the modules template references the conda environment file ([#2540](https://github.com/nf-core/tools/pull/2540))
+- Unset env variable JAVA_TOOL_OPTIONS in gitpod ([#2569](https://github.com/nf-core/tools/pull/2569))
 
 ### Download
 
@@ -19,13 +20,15 @@
 - Correctly pass subworkflow linting test if `COMPONENT.out.versions` is used in the script ([#2448](https://github.com/nf-core/tools/pull/2448))
 - Check for spaces in modules container URLs ([#2452](https://github.com/nf-core/tools/issues/2452))
 - Correctly ignore `timeline.enabled`, `report.enabled`, `trace.enabled`, `dag.enabled` variables when linting a pipeline. ([#2507](https://github.com/nf-core/tools/pull/2507))
+- Don't fail linting if md5sum for empty files are found in a stub test ([#2571](https://github.com/nf-core/tools/pull/2571))
 
 ### Modules
 
 - Added stub test creation to `create_test_yml` ([#2476](https://github.com/nf-core/tools/pull/2476))
 - Replace ModulePatch by ComponentPatch ([#2482](https://github.com/nf-core/tools/pull/2482))
 - Fixed `nf-core modules lint` to work with new module structure for nf-test ([#2494](https://github.com/nf-core/tools/pull/2494))
 - Add option `--migrate-pytest` to create a module with nf-test taking into account an existing module ([#2549](https://github.com/nf-core/tools/pull/2549))
+- Modules `--migrate-pytest` copies template scripts ([#2568](https://github.com/nf-core/tools/pull/2568))
 
 ### Subworkflows
 
@@ -41,6 +44,8 @@
 - Add mypy to pre-commit config for the tools repo ([#2545](https://github.com/nf-core/tools/pull/2545))
 - Use Path objects for ComponentCreate and update the structure of components templates ([#2551](https://github.com/nf-core/tools/pull/2551)).
 - GitPod base image: swap tool installation back to `conda` from `mamba` ([#2566](https://github.com/nf-core/tools/pull/2566)).
+- Sort the `installed_by` list in `modules.json` ([#2570](https://github.com/nf-core/tools/pull/2570)).
+- Unset env variable JAVA_TOOL_OPTIONS in gitpod ([#2569](https://github.com/nf-core/tools/pull/2569))
 
 # [v2.10 - Nickel Ostrich](https://github.com/nf-core/tools/releases/tag/2.10) + [2023-09-25]
 

diff --git a/nf_core/components/create.py b/nf_core/components/create.py
@@ -431,6 +431,11 @@ def _copy_old_files(self, component_old_path):
         if self.component_type == "modules":
             log.debug("Copying original environment.yml file")
             shutil.copyfile(component_old_path / "environment.yml", self.file_paths["environment.yml"])
+            if (component_old_path / "templates").is_dir():
+                log.debug("Copying original templates directory")
+                shutil.copytree(
+                    component_old_path / "templates", self.file_paths["environment.yml"].parent / "templates"
+                )
         # Create a nextflow.config file if it contains information other than publishDir
         pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir)
         nextflow_config = pytest_dir / "nextflow.config"

diff --git a/nf_core/components/install.py b/nf_core/components/install.py
@@ -127,6 +127,8 @@ def install(self, component, silent=False):
             self.install_included_components(component_dir)
 
         if not silent:
+            modules_json.load()
+            modules_json.dump(run_prettier=True)
             # Print include statement
             component_name = "_".join(component.upper().split("/"))
             log.info(f"Use the following statement to include this {self.component_type[:-1]}:")

diff --git a/nf_core/components/update.py b/nf_core/components/update.py
@@ -288,20 +288,21 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                 updated.append(component)
                 recursive_update = True
                 modules_to_update, subworkflows_to_update = self.get_components_to_update(component)
-                if not silent and not self.update_all and len(modules_to_update + subworkflows_to_update) > 0:
-                    log.warning(
-                        f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n"
-                        "It is advised to keep all your modules and subworkflows up to date.\n"
-                        "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n"
-                    )
-                    if self.update_deps:
-                        recursive_update = True
-                    else:
-                        recursive_update = questionary.confirm(
-                            "Would you like to continue updating all modules and subworkflows?",
-                            default=True,
-                            style=nf_core.utils.nfcore_question_style,
-                        ).unsafe_ask()
+                if not silent and len(modules_to_update + subworkflows_to_update) > 0:
+                    if not self.update_all:
+                        log.warning(
+                            f"All modules and subworkflows linked to the updated {self.component_type[:-1]} will be {'asked for update' if self.show_diff else 'automatically updated'}.\n"
+                            "It is advised to keep all your modules and subworkflows up to date.\n"
+                            "It is not guaranteed that a subworkflow will continue working as expected if all modules/subworkflows used in it are not up to date.\n"
+                        )
+                        if self.update_deps:
+                            recursive_update = True
+                        else:
+                            recursive_update = questionary.confirm(
+                                "Would you like to continue updating all modules and subworkflows?",
+                                default=True,
+                                style=nf_core.utils.nfcore_question_style,
+                            ).unsafe_ask()
                 if recursive_update and len(modules_to_update + subworkflows_to_update) > 0:
                     # Update linked components
                     self.update_linked_components(modules_to_update, subworkflows_to_update, updated)
@@ -323,8 +324,12 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                 )
         elif not all_patches_successful and not silent:
             log.info(f"Updates complete. Please apply failed patch{plural_es(components_info)} manually.")
+            self.modules_json.load()
+            self.modules_json.dump(run_prettier=True)
         elif not silent:
             log.info("Updates complete :sparkles:")
+            self.modules_json.load()
+            self.modules_json.dump(run_prettier=True)
 
         return exit_value
 

diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py
@@ -1,6 +1,7 @@
 """
 Lint the tests of a module in nf-core/modules
 """
+import json
 import logging
 from pathlib import Path
 
@@ -52,36 +53,64 @@ def module_tests(_, module: NFCoreComponent):
                     )
                     # Validate no empty files
                     with open(snap_file, "r") as snap_fh:
-                        snap_content = snap_fh.read()
-                        if "d41d8cd98f00b204e9800998ecf8427e" in snap_content:
+                        try:
+                            snap_content = json.load(snap_fh)
+                            for test_name in snap_content.keys():
+                                if "d41d8cd98f00b204e9800998ecf8427e" in str(snap_content[test_name]):
+                                    if "stub" not in test_name:
+                                        module.failed.append(
+                                            (
+                                                "test_snap_md5sum",
+                                                "md5sum for empty file found: d41d8cd98f00b204e9800998ecf8427e",
+                                                snap_file,
+                                            )
+                                        )
+                                    else:
+                                        module.passed.append(
+                                            (
+                                                "test_snap_md5sum",
+                                                "md5sum for empty file found, but it is a stub test",
+                                                snap_file,
+                                            )
+                                        )
+                                else:
+                                    module.passed.append(
+                                        (
+                                            "test_snap_md5sum",
+                                            "no md5sum for empty file found",
+                                            snap_file,
+                                        )
+                                    )
+                                if "7029066c27ac6f5ef18d660d5741979a" in str(snap_content[test_name]):
+                                    if "stub" not in test_name:
+                                        module.failed.append(
+                                            (
+                                                "test_snap_md5sum",
+                                                "md5sum for compressed empty file found: 7029066c27ac6f5ef18d660d5741979a",
+                                                snap_file,
+                                            )
+                                        )
+                                    else:
+                                        module.passed.append(
+                                            (
+                                                "test_snap_md5sum",
+                                                "md5sum for compressed empty file found, but it is a stub test",
+                                                snap_file,
+                                            )
+                                        )
+                                else:
+                                    module.passed.append(
+                                        (
+                                            "test_snap_md5sum",
+                                            "no md5sum for compressed empty file found",
+                                            snap_file,
+                                        )
+                                    )
+                        except json.decoder.JSONDecodeError as e:
                             module.failed.append(
                                 (
-                                    "test_snap_md5sum",
-                                    "md5sum for empty file found: d41d8cd98f00b204e9800998ecf8427e",
-                                    snap_file,
-                                )
-                            )
-                        else:
-                            module.passed.append(
-                                (
-                                    "test_snap_md5sum",
-                                    "no md5sum for empty file found",
-                                    snap_file,
-                                )
-                            )
-                        if "7029066c27ac6f5ef18d660d5741979a" in snap_content:
-                            module.failed.append(
-                                (
-                                    "test_snap_md5sum",
-                                    "md5sum for compressed empty file found: 7029066c27ac6f5ef18d660d5741979a",
-                                    snap_file,
-                                )
-                            )
-                        else:
-                            module.passed.append(
-                                (
-                                    "test_snap_md5sum",
-                                    "no md5sum for compressed empty file found",
+                                    "test_snapshot_exists",
+                                    f"snapshot file `main.nf.test.snap` can't be read: {e}",
                                     snap_file,
                                 )
                             )

diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py
@@ -6,6 +6,7 @@
 import shutil
 import tempfile
 from pathlib import Path
+from typing import Union
 
 import git
 import questionary
@@ -41,7 +42,7 @@ def __init__(self, pipeline_dir):
         self.modules_dir = Path(self.dir, "modules")
         self.subworkflows_dir = Path(self.dir, "subworkflows")
         self.modules_json_path = Path(self.dir, "modules.json")
-        self.modules_json = None
+        self.modules_json: Union(dict, None) = None
         self.pipeline_modules = None
         self.pipeline_subworkflows = None
         self.pipeline_components = None
@@ -678,7 +679,7 @@ def update(
             repo_component_entry[component_name]["installed_by"] = [installed_by]
         finally:
             new_installed_by = repo_component_entry[component_name]["installed_by"] + list(installed_by_log)
-            repo_component_entry[component_name]["installed_by"] = [*set(new_installed_by)]
+            repo_component_entry[component_name]["installed_by"] = sorted([*set(new_installed_by)])
 
         # Sort the 'modules.json' repo entries
         self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"])
@@ -1035,13 +1036,17 @@ def get_component_branch(self, component_type, component, repo_url, install_dir)
             )
         return branch
 
-    def dump(self):
+    def dump(self, run_prettier: bool = False):
         """
         Sort the modules.json, and write it to file
         """
         # Sort the modules.json
         self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"])
-        dump_json_with_prettier(self.modules_json_path, self.modules_json)
+        if run_prettier:
+            dump_json_with_prettier(self.modules_json_path, self.modules_json)
+        else:
+            with open(self.modules_json_path, "w") as fh:
+                json.dump(self.modules_json, fh, indent=4)
 
     def resolve_missing_installation(self, missing_installation, component_type):
         missing_but_in_mod_json = [

diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml
@@ -4,7 +4,9 @@ tasks:
     command: |
       pre-commit install --install-hooks
       nextflow self-update
-
+  - name: unset JAVA_TOOL_OPTIONS
+    command: |
+      unset JAVA_TOOL_OPTIONS
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack
     - codezombiech.gitignore # Language support for .gitignore files