fix: Update scheduler script

BALSAMIC/assets/scripts/immediate_submit.py

@@ -0,0 +1,78 @@
+"""Script to submit jobs to a cluster."""
+import shutil
+from typing import Any, Dict, List, Optional
+
+import click
+from snakemake import utils
+
+from BALSAMIC.commands.options import (
+    OPTION_BENCHMARK,
+    OPTION_CLUSTER_ACCOUNT,
+    OPTION_CLUSTER_MAIL,
+    OPTION_CLUSTER_MAIL_TYPE,
+    OPTION_CLUSTER_PROFILE,
+    OPTION_CLUSTER_QOS,
+)
+from BALSAMIC.constants.cluster import QOS, ClusterProfile
+from BALSAMIC.models.scheduler import Scheduler
+
+
+@click.command()
+@click.argument("case_id", nargs=1, required=True, type=click.STRING)
+@click.argument("dependencies", nargs=-1, type=click.STRING)
+@click.argument("job_script", nargs=1, type=click.Path(exists=True, resolve_path=True))
+@OPTION_CLUSTER_ACCOUNT
+@OPTION_BENCHMARK
+@OPTION_CLUSTER_MAIL_TYPE
+@OPTION_CLUSTER_MAIL
+@OPTION_CLUSTER_PROFILE
+@OPTION_CLUSTER_QOS
+@click.option(
+    "--log-dir",
+    type=click.Path(exists=True, resolve_path=True),
+    required=True,
+    help="Logging directory path",
+)
+@click.option(
+    "--script-dir",
+    type=click.Path(exists=True, resolve_path=True),
+    required=True,
+    help="Script directory path",
+)
+def immediate_submit(
+    account: str,
+    case_id: str,
+    job_script: str,
+    log_dir: str,
+    profile: ClusterProfile,
+    script_dir: str,
+    benchmark: Optional[bool] = False,
+    dependencies: Optional[List[str]] = None,
+    mail_type: Optional[str] = None,
+    mail_user: Optional[str] = None,
+    qos: Optional[QOS] = QOS.LOW,
+) -> None:
+    """
+    Submits jobs to the cluster. Each job is submitted sequentially, and their respective job IDs are collected
+    from the output. These job IDs are then forwarded as dependencies to the subsequent jobs.
+    """
+    job_script: str = shutil.copy2(src=job_script, dst=script_dir)
+    job_properties: Dict[str, Any] = utils.read_job_properties(job_script)
+    scheduler: Scheduler = Scheduler(
+        account=account,
+        benchmark=benchmark,
+        case_id=case_id,
+        dependencies=dependencies,
+        job_properties=job_properties,
+        job_script=job_script,
+        log_dir=log_dir,
+        mail_type=mail_type,
+        mail_user=mail_user,
+        profile=profile,
+        qos=qos,
+    )
+    scheduler.submit_job()
+
+
+if __name__ == "__main__":
+    immediate_submit()

BALSAMIC/commands/init/base.py

@@ -5,42 +5,42 @@
 import sys
 from datetime import datetime
 from pathlib import Path
-from typing import Union, List, Optional
+from typing import List, Optional, Union
 
 import click
 
 from BALSAMIC.commands.options import (
-    OPTION_RUN_MODE,
-    OPTION_CLUSTER_PROFILE,
-    OPTION_CLUSTER_QOS,
+    OPTION_CACHE_VERSION,
     OPTION_CLUSTER_ACCOUNT,
-    OPTION_SNAKEMAKE_OPT,
-    OPTION_GENOME_VERSION,
-    OPTION_FORCE_ALL,
-    OPTION_RUN_ANALYSIS,
+    OPTION_CLUSTER_CONFIG,
     OPTION_CLUSTER_MAIL,
     OPTION_CLUSTER_MAIL_TYPE,
+    OPTION_CLUSTER_PROFILE,
+    OPTION_CLUSTER_QOS,
+    OPTION_COSMIC_KEY,
+    OPTION_FORCE_ALL,
+    OPTION_GENOME_VERSION,
+    OPTION_OUT_DIR,
     OPTION_QUIET,
-    OPTION_CACHE_VERSION,
+    OPTION_RUN_ANALYSIS,
+    OPTION_RUN_MODE,
     OPTION_SNAKEFILE,
-    OPTION_OUT_DIR,
-    OPTION_CLUSTER_CONFIG,
-    OPTION_COSMIC_KEY,
+    OPTION_SNAKEMAKE_OPT,
 )
 from BALSAMIC.constants.analysis import BIOINFO_TOOL_ENV, RunMode
-from BALSAMIC.constants.cache import GenomeVersion, REFERENCE_FILES
+from BALSAMIC.constants.cache import REFERENCE_FILES, GenomeVersion
 from BALSAMIC.constants.cluster import (
-    ClusterMailType,
     QOS,
-    ClusterProfile,
     ClusterConfigType,
+    ClusterMailType,
+    ClusterProfile,
 )
-from BALSAMIC.models.cache import CacheConfig, ReferencesHg, ReferencesCanFam
+from BALSAMIC.models.cache import CacheConfig, ReferencesCanFam, ReferencesHg
 from BALSAMIC.models.snakemake import SnakemakeExecutable
 from BALSAMIC.utils.analysis import get_cache_singularity_bind_paths
 from BALSAMIC.utils.cache import get_containers
-from BALSAMIC.utils.cli import get_snakefile, get_config_path
-from BALSAMIC.utils.io import write_json, generate_workflow_graph
+from BALSAMIC.utils.cli import get_config_path, get_snakefile
+from BALSAMIC.utils.io import generate_workflow_graph, write_json
 
 LOG = logging.getLogger(__name__)
 
@@ -160,7 +160,6 @@ def initialize(
         profile=profile,
         qos=qos,
         quiet=quiet,
-        result_dir=references_dir,
         run_analysis=run_analysis,
         run_mode=run_mode,
         script_dir=script_dir,

BALSAMIC/commands/run/analysis.py

@@ -37,12 +37,8 @@
 from BALSAMIC.models.config import ConfigModel
 from BALSAMIC.models.snakemake import SnakemakeExecutable
 from BALSAMIC.utils.analysis import get_singularity_bind_paths
-from BALSAMIC.utils.cli import (
-    createDir,
-    get_config_path,
-    get_snakefile,
-    job_id_dump_to_yaml,
-)
+from BALSAMIC.utils.cli import createDir, get_config_path, get_snakefile
+from BALSAMIC.utils.io import write_sacct_to_yaml
 
 LOG = logging.getLogger(__name__)
 
@@ -92,9 +88,7 @@ def analysis(
         run_mode: RunMode = RunMode.LOCAL
 
     if run_mode == RunMode.CLUSTER and not account:
-        LOG.info(
-            "slurm-account, qsub-account, or account is required for slurm run mode"
-        )
+        LOG.info("An account is required for cluster run mode")
         raise click.Abort()
 
     sample_config_path: Path = Path(sample_config).absolute()
@@ -152,7 +146,6 @@ def analysis(
         profile=profile,
         qos=qos,
         quiet=quiet,
-        result_dir=result_path.as_posix(),
         run_analysis=run_analysis,
         run_mode=run_mode,
         script_dir=script_path.as_posix(),
@@ -167,8 +160,10 @@ def analysis(
     )
 
     if run_analysis and run_mode == "cluster":
-        jobid_dump = Path(log_path, f"{case_name}.sacct")
-        jobid_yaml = Path(result_path, f"{profile}_jobids.yaml")
-        job_id_dump_to_yaml(
-            job_id_dump=jobid_dump, job_id_yaml=jobid_yaml, case_name=case_name
+        sacct_file_path: Path = Path(log_path, f"{case_name}.sacct")
+        yaml_file_path: Path = Path(result_path, f"{profile}_jobids.yaml")
+        write_sacct_to_yaml(
+            case_id=case_name,
+            sacct_file_path=sacct_file_path,
+            yaml_file_path=yaml_file_path,
         )

BALSAMIC/constants/cluster.py

@@ -16,7 +16,6 @@ class ClusterProfile(StrEnum):
     """Profile to submit jobs to the cluster."""
 
     SLURM: str = "slurm"
-    QSUB: str = "qsub"
 
 
 CLUSTER_PROFILES: List[ClusterProfile] = [profile for profile in ClusterProfile]

BALSAMIC/constants/paths.py

@@ -9,7 +9,7 @@
 ASSETS_DIR: Path = Path(BALSAMIC_DIR, "assets")
 SCRIPT_DIR: Path = Path(ASSETS_DIR, "scripts")
 REFSEQ_SCRIPT_PATH: Path = Path(SCRIPT_DIR, "refseq_sql.awk")
-SCHEDULER_PATH: Path = Path(BALSAMIC_DIR, "utils", "scheduler.py")
+IMMEDIATE_SUBMIT_PATH: Path = Path(SCRIPT_DIR, "immediate_submit.py")
 
 # Sentieon specific constants
 SENTIEON_MODELS_DIR: Path = Path(BALSAMIC_DIR, "assets", "sentieon_models")

BALSAMIC/constants/workflow_params.py

@@ -115,7 +115,7 @@
     },
 }
 
-SLEEP_BEFORE_START = 300
+SLEEP_BEFORE_START = 600
 
 WORKFLOW_PARAMS = {
     "common": {