Merge pull request #3167 from mirpedrol/try-catch

Template: remove try-catch blocks from nextflow.config

.github/actions/create-lint-wf/action.yml

@@ -53,6 +53,12 @@ runs:
       run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
       working-directory: create-lint-wf
 
+    # Uncomment includeConfig statement
+    - name: uncomment include config
+      shell: bash
+      run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+      working-directory: create-lint-wf
+
     # Replace zenodo.XXXXXX to pass readme linting
     - name: replace zenodo.XXXXXX
       shell: bash

.github/workflows/create-lint-wf.yml

@@ -78,6 +78,11 @@ jobs:
         run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
         working-directory: create-lint-wf
 
+      # Uncomment includeConfig statement
+      - name: uncomment include config
+        run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+        working-directory: create-lint-wf
+
       # Run the other nf-core commands
       - name: nf-core pipelines list
         run: nf-core --log-file log.txt pipelines list

.github/workflows/create-test-lint-wf-template.yml

@@ -137,6 +137,11 @@ jobs:
         run: find my-prefix-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
         working-directory: create-test-lint-wf
 
+      # Uncomment includeConfig statement
+      - name: uncomment include config
+        run: find my-prefix-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+        working-directory: create-test-lint-wf
+
       # Replace zenodo.XXXXXX to pass readme linting
       - name: replace zenodo.XXXXXX
         run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;

CHANGELOG.md

@@ -41,6 +41,7 @@
 - Remove if/else block to include `igenomes.config` ([#3168](https://github.com/nf-core/tools/pull/3168))
 - Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([#3037](https://github.com/nf-core/tools/pull/3037))
 - Fixed release announcement hashtags for Mastodon ([#3099](https://github.com/nf-core/tools/pull/3176))
+- Remove try/catch blocks from `nextflow.config` ([#3167](https://github.com/nf-core/tools/pull/3167))
 
 ### Linting
 

docs/api/_src/pipeline_lint_tests/included_configs.md

@@ -0,0 +1,5 @@
+# included_configs
+
+    ```{eval-rst}
+    .. automethod:: nf_core.pipelines.lint.PipelineLint.included_configs
+    ```

docs/api/_src/pipeline_lint_tests/index.md

@@ -7,6 +7,7 @@
     - [base_config](./base_config/)
     - [files_exist](./files_exist/)
     - [files_unchanged](./files_unchanged/)
+    - [included_configs](./included_configs/)
     - [merge_markers](./merge_markers/)
     - [modules_config](./modules_config/)
     - [modules_json](./modules_json/)
@@ -16,6 +17,7 @@
     - [nfcore_yml](./nfcore_yml/)
     - [pipeline_name_conventions](./pipeline_name_conventions/)
     - [pipeline_todos](./pipeline_todos/)
+    - [plugin_includes](./plugin_includes/)
     - [readme](./readme/)
     - [schema_description](./schema_description/)
     - [schema_lint](./schema_lint/)

nf_core/pipeline-template/nextflow.config

@@ -186,18 +186,11 @@ profiles {
 
 {% if nf_core_configs -%}
 // Load nf-core custom profiles from different Institutions
-try {
-    includeConfig "${params.custom_config_base}/nfcore_custom.config"
-} catch (Exception e) {
-    System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")
-}
+includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null"
 
 // Load {{ name }} custom profiles from different institutions.
-try {
-    includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config"
-} catch (Exception e) {
-    System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config")
-}
+// TODO nf-core: Optionally, you can add a pipeline-specific nf-core config at https://github.com/nf-core/configs
+// includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/{{ short_name }}.config" : "/dev/null"
 {% endif -%}
 
 // Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile

nf_core/pipelines/create/template_features.yml

@@ -120,6 +120,7 @@ nf_core_configs:
       - "custom_config"
       - "params.custom_config_version"
       - "params.custom_config_base"
+    included_configs: False
   nfcore_pipelines: False
   custom_pipelines: True
 is_nfcore:

nf_core/pipelines/lint/__init__.py

@@ -37,6 +37,7 @@
 from .configs import base_config, modules_config
 from .files_exist import files_exist
 from .files_unchanged import files_unchanged
+from .included_configs import included_configs
 from .merge_markers import merge_markers
 from .modules_json import modules_json
 from .modules_structure import modules_structure
@@ -101,6 +102,7 @@ class PipelineLint(nf_core.utils.Pipeline):
     system_exit = system_exit
     template_strings = template_strings
     version_consistency = version_consistency
+    included_configs = included_configs
 
     def __init__(
         self, wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False
@@ -152,7 +154,7 @@ def _get_all_lint_tests(release_mode):
             "base_config",
             "modules_config",
             "nfcore_yml",
-        ] + (["version_consistency"] if release_mode else [])
+        ] + (["version_consistency", "included_configs"] if release_mode else [])
 
     def _load(self) -> bool:
         """Load information about the pipeline into the PipelineLint object"""

nf_core/pipelines/lint/included_configs.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+
+def included_configs(self):
+    """Check that the pipeline nextflow.config includes the pipeline custom configs.
+
+    If the include line is uncommented, the test passes.
+    If the include line is commented, the test fails.
+    If the include line is missing, the test warns.
+
+    Can be skipped by adding the following to the .nf-core.yml file:
+    lint:
+        included_configs: False
+    """
+    passed = []
+    failed = []
+    warned = []
+
+    config_file = Path(self.wf_path / "nextflow.config")
+
+    with open(config_file) as fh:
+        config = fh.read()
+        if (
+            f"// includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? \"${{params.custom_config_base}}/pipeline/{self.pipeline_name}.config\""
+            in config
+        ):
+            failed.append("Pipeline config does not include custom configs. Please uncomment the includeConfig line.")
+        elif (
+            f"includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? \"${{params.custom_config_base}}/pipeline/{self.pipeline_name}.config\""
+            in config
+        ):
+            passed.append("Pipeline config includes custom configs.")
+        else:
+            warned.append("Pipeline config does not include custom configs. Please add the includeConfig line.")
+
+    return {"passed": passed, "failed": failed, "warned": warned}

nf_core/pipelines/lint/nextflow_config.py

@@ -346,26 +346,40 @@ def nextflow_config(self) -> Dict[str, List[str]]:
             failed.append(f"Config `params.custom_config_base` is not set to `{custom_config_base}`")
 
         # Check that lines for loading custom profiles exist
-        lines = [
+        old_lines = [
             r"// Load nf-core custom profiles from different Institutions",
             r"try {",
             r'includeConfig "${params.custom_config_base}/nfcore_custom.config"',
             r"} catch (Exception e) {",
             r'System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")',
             r"}",
         ]
+        lines = [
+            r"// Load nf-core custom profiles from different Institutions",
+            r'''includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null"''',
+        ]
         path = Path(self.wf_path, "nextflow.config")
         i = 0
         with open(path) as f:
             for line in f:
-                if lines[i] in line:
+                if old_lines[i] in line:
+                    i += 1
+                    if i == len(old_lines):
+                        break
+                elif lines[i] in line:
                     i += 1
                     if i == len(lines):
                         break
                 else:
                     i = 0
         if i == len(lines):
             passed.append("Lines for loading custom profiles found")
+        elif i == len(old_lines):
+            failed.append(
+                "Old lines for loading custom profiles found. File should contain: ```groovy\n{}".format(
+                    "\n".join(lines)
+                )
+            )
         else:
             lines[2] = f"\t{lines[2]}"
             lines[4] = f"\t{lines[4]}"

tests/utils.py

@@ -113,7 +113,7 @@ def create_tmp_pipeline(no_git: bool = False) -> Tuple[Path, Path, str, Path]:
     tmp_dir = Path(tempfile.TemporaryDirectory().name)
     root_repo_dir = Path(__file__).resolve().parent.parent
     template_dir = root_repo_dir / "nf_core" / "pipeline-template"
-    pipeline_name = "mypipeline"
+    pipeline_name = "testpipeline"
     pipeline_dir = tmp_dir / pipeline_name
     pipeline_dir.mkdir(parents=True)
 
@@ -123,7 +123,7 @@ def create_tmp_pipeline(no_git: bool = False) -> Tuple[Path, Path, str, Path]:
         org_path="nf-core",
         lint=None,
         template=NFCoreTemplateConfig(
-            name="mypipeline",
+            name="testpipeline",
             author="me",
             description="it is mine",
             org="nf-core",