feat: add slurm script templates to package

and use them with importlib

src/ccbr_tools/pipeline/nextflow.py

@@ -1,19 +1,29 @@
-from ..pkg_util import repo_base, msg_box
+"""
+Module: nextflow
+
+This module provides functions for running Nextflow workflows in local and HPC environments.
+
+Functions:
+- run(nextfile_path=None, nextflow_args=None, mode="local", pipeline_name=None, debug=False, hpc_options={})
+    Run a Nextflow workflow.
+"""
+
+from ..pkg_util import repo_base, msg_box, use_template
 from ..shell import shell_run
 from .util import get_hpcname
 
 
-def run_nextflow(
+def run(
     nextfile_path=None,
-    merge_config=None,
-    threads=None,
     nextflow_args=None,
     mode="local",
+    pipeline_name=None,
+    debug=False,
     hpc_options={
-        "biowulf": {"profile": "biowulf", "slurm": "assets/slurm_header_biowulf.sh"},
+        "biowulf": {"profile": "biowulf", "slurm": "slurm_nxf_biowulf.sh"},
         "fnlcr": {
             "profile": "frce",
-            "slurm": "assets/slurm_header_frce.sh",
+            "slurm": "slurm_nxf_frce.sh",
         },
     },
 ):
@@ -22,8 +32,6 @@ def run_nextflow(
 
     Args:
         nextfile_path (str, optional): Path to the Nextflow file. Defaults to None.
-        merge_config (str, optional): Merge configuration. Defaults to None.
-        threads (int, optional): Number of threads. Defaults to None.
         nextflow_args (list, optional): Additional Nextflow arguments. Defaults to None.
         mode (str, optional): Execution mode. Defaults to "local".
         hpc_options (dict, optional): HPC options. Defaults to {"biowulf": {"profile": "biowulf", "slurm": "assets/slurm_header_biowulf.sh"}, "fnlcr": {"profile": "frce", "slurm": "assets/slurm_header_frce.sh"}}.
@@ -70,6 +78,12 @@ def run_nextflow(
 
     if mode == "slurm":
         slurm_filename = "submit_slurm.sh"
+        use_template(
+            hpc_options[hpc]["slurm"],
+            output_filepath="submit_slurm.sh",
+            PIPELINE=pipeline_name if pipeline_name else "CCBR_nxf",
+            RUN_COMMAND=nextflow_command,
+        )
         with open(slurm_filename, "w") as sbatch_file:
             with open(repo_base(hpc_options[hpc]["slurm"]), "r") as template:
                 sbatch_file.writelines(template.readlines())
@@ -82,5 +96,6 @@ def run_nextflow(
         run_command = nextflow_command
     else:
         raise ValueError(f"mode {mode} not recognized")
+
     # Run Nextflow
     shell_run(run_command, capture_output=False)

src/ccbr_tools/pkg_util.py

@@ -1,4 +1,6 @@
-""" Miscellaneous utility functions for the package """
+"""
+Miscellaneous utility functions for the package
+"""
 
 from cffconvert.cli.create_citation import create_citation
 from cffconvert.cli.validate_or_write_output import validate_or_write_output
@@ -9,6 +11,8 @@
 from time import localtime, strftime
 import tomllib
 
+from . import templates
+
 
 class CustomClickGroup(click.Group):
     def format_epilog(self, ctx, formatter):
@@ -149,3 +153,42 @@ def msg_box(splash, errmsg=None):
     msg(("-" * (len(splash) + 4)))
     if errmsg:
         click.echo("\n" + errmsg, err=True)
+
+
+def read_template(template_name):
+    """
+    Read a template file
+
+    Args:
+        template_name (str): Name of the template file
+
+    Returns:
+        template (str): Contents of the template file
+    """
+
+    template_files = importlib.resources.files(templates)
+    template_path = template_files / template_name
+    with open(template_path, "rt") as template_file:
+        return template_file.read()
+
+
+def use_template(template_name, output_filepath=None, **kwargs):
+    """
+    Uses a template, formats variables, and write it to a file.
+    Args:
+        template_name (str): The name of the template to use.
+        output_filepath (str, optional): The filepath to save the output file. If not provided, it will be written to `template_name` in the current working directory.
+        **kwargs: Keyword arguments to fill in the template variables.
+    Returns:
+        None
+    Raises:
+        FileNotFoundError: If the template file is not found.
+        IOError: If there is an error writing the output file.
+    Examples:
+        use_template("slurm_nxf_biowulf.sh", output_filepath="submit_slurm.sh", PIPELINE="CCBR_nxf", RUN_COMMAND="nextflow run main.nf -stub")
+    """
+    template_str = read_template(template_name)
+    if not output_filepath:
+        output_filepath = template_name
+    with open(output_filepath, "wt") as outfile:
+        outfile.write(template_str.format(**kwargs))

src/ccbr_tools/templates/slurm_nxf_biowulf.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=1g
+#SBATCH --time=1-00:00:00
+#SBATCH --parsable
+#SBATCH -J "{PIPELINE}"
+#SBATCH --mail-type=BEGIN,END,FAIL
+#SBATCH --output "log/slurm_%j.log"
+#SBATCH --output "log/slurm_%j.log"
+
+module load ccbrpipeliner nextflow
+NXF_SINGULARITY_CACHEDIR=/data/CCBR_Pipeliner/SIFS
+
+{RUN_COMMAND}

src/ccbr_tools/templates/slurm_nxf_frce.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=1g
+#SBATCH --time=1-00:00:00
+#SBATCH --parsable
+#SBATCH -J "{PIPELINE}"
+#SBATCH --mail-type=BEGIN,END,FAIL
+#SBATCH --output "log/slurm_%j.log"
+#SBATCH --output "log/slurm_%j.log"
+
+module load nextflow
+NXF_SINGULARITY_CACHEDIR=/mnt/projects/CCBR-Pipelines/SIFs
+# TODO add shared tools to the path
+
+{RUN_COMMAND}

src/ccbr_tools/templates/slurm_smk_biowulf.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=1g
+#SBATCH --time=1-00:00:00
+#SBATCH --parsable
+#SBATCH -J "{PIPELINE}"
+#SBATCH --mail-type=BEGIN,END,FAIL
+#SBATCH --output "log/slurm_%j.log"
+#SBATCH --output "log/slurm_%j.log"
+
+module load ccbrpipeliner snakemake/7 singularity
+
+{RUN_COMMAND}

src/ccbr_tools/templates/slurm_smk_frce.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=1g
+#SBATCH --time=1-00:00:00
+#SBATCH --parsable
+#SBATCH -J "{PIPELINE}"
+#SBATCH --mail-type=BEGIN,END,FAIL
+#SBATCH --output "log/slurm_%j.log"
+#SBATCH --output "log/slurm_%j.log"
+
+module load ccbrpipeliner snakemake/7 singularity
+# TODO add shared Tools to the path
+
+{RUN_COMMAND}

tests/test_pkg_util.py

@@ -0,0 +1,11 @@
+from ccbr_tools.pkg_util import read_template
+
+
+def test_read_template():
+    template_str = read_template("slurm_nxf_biowulf.sh")
+    assert all(
+        [
+            template_str.startswith("#!/usr/bin/env bash"),
+            template_str.endswith("{RUN_COMMAND}"),
+        ]
+    )