import re
import stat
import subprocess
+from pathlib import Path
from textwrap import dedent, indent
-from typing import Any
+from typing import Any, Sequence
from . import __version__
from .format import format_with_config
@@ -241,8 +242,7 @@ Source code for easy_slurm.jobs
[docs]def submit_job(
job_dir: str,
*,
- src: str = "",
- assets: str = "",
+ src: Sequence[str] = (),
on_run: str = "",
on_run_resume: str = "",
setup: str = "",
@@ -253,23 +253,20 @@ Source code for easy_slurm.jobs
submit: bool = True,
interactive: bool = False,
resubmit_limit: int = 64,
+ config: dict[str, Any] = {},
) -> str:
"""Submits job.
- Creates job directory with frozen assets and submits job to slurm.
+ Creates job directory with frozen src and submits job to slurm.
Args:
job_dir (str):
Path to directory to keep all job files including
- ``src.tar``, ``assets.tar``, and auto-generated ``job.sh``.
- src (str):
- Path to directory containing only source code.
+ ``src.tar`` and auto-generated ``job.sh``.
+ src (list[str]):
+ Path to directories containing only source code.
These will be archived in ``$JOB_DIR/src.tar`` and
- extracted during job run into ``$SLURM_TMPDIR/src``.
- assets (str):
- Path to directory containing additional assets.
- These will be archived in ``$JOB_DIR/assets.tar`` and
- extracted during job run into ``$SLURM_TMPDIR/assets``.
+ extracted during job run into ``$SLURM_TMPDIR``.
on_run (str):
Bash code executed in "on_run" stage, but only for new jobs
that are running for the first time.
@@ -307,13 +304,18 @@ Source code for easy_slurm.jobs
Maximum number of times to auto-submit a job for "resume".
(Not entirely unlike submitting a resume for a job.)
Default is 64 resubmissions.
+ config (dict[str, Any]):
+ A dictionary of configuration values to use for formatting.
Returns:
Path to the newly created job directory.
"""
job_name = sbatch_options.get("job-name", "untitled")
- job_dir = _expand_path(format_with_config(job_dir, {"job_name": job_name}))
- create_job_dir(job_dir, src, assets)
+ job_name = format_with_config(job_name, config)
+ job_dir = _expand_path(
+ format_with_config(job_dir, {**config, "job_name": job_name})
+ )
+ create_job_dir(job_dir, src)
_write_script(
filename=f"{job_dir}/job.sh",
@@ -396,21 +398,13 @@ Source code for easy_slurm.jobs
)
-[docs]def create_job_dir(
- job_dir: str,
- src: str,
- assets: str,
-):
+[docs]def create_job_dir(job_dir: str, src: Sequence[str]):
"""Creates job directory and freezes all necessary files."""
job_dir = _expand_path(job_dir)
- src = _expand_path(src)
- assets = _expand_path(assets)
+ src = [_expand_path(x) for x in src]
os.makedirs(job_dir, exist_ok=True)
- if src != "":
- _create_tar_dir(src, f"{job_dir}/src.tar.gz", "src")
- if assets != "":
- _create_tar_dir(assets, f"{job_dir}/assets.tar.gz", "assets")
+ _create_tar_dir(src, f"{job_dir}/src.tar.gz")
with open(f"{job_dir}/status", "w") as f:
print("status=new", file=f)
@@ -445,9 +439,18 @@ Source code for easy_slurm.jobs
return "" if path == "" else os.path.abspath(os.path.expandvars(path))
-def _create_tar_dir(src, dst, root_name):
- transform = rf"s/^\./{root_name}/"
- cmd = ["tar", "czf", dst, "-C", src, ".", "--transform", transform]
+def _create_tar_dir(src, dst, root_name=None):
+ if not src:
+ src_args = ["-T", "/dev/null"]
+ else:
+ src_args = [
+ arg
+ for srcdir in src
+ for arg in ["-C", Path(srcdir).parent, Path(srcdir).name]
+ ]
+ cmd = ["tar", "czf", dst, *src_args]
+ if root_name is not None:
+ cmd.extend(["--transform", rf"s/^/{root_name}\//"])
subprocess.run(cmd, check=True)
diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt
index 60e5611..5ba7d1c 100644
--- a/_sources/index.rst.txt
+++ b/_sources/index.rst.txt
@@ -10,7 +10,7 @@ Easy Slurm allows you to easily manage and submit robust jobs to Slurm using Pyt
Features
--------
-- **Freezes** source code and assets by copying to separate ``$JOB_DIR``.
+- **Freezes** source code by copying to separate ``$JOB_DIR``.
- **Auto-submits** another job if current job times out.
- **Exposes hooks** for custom bash code: ``setup``/``setup_resume``, ``on_run``/``on_run_resume``, and ``teardown``.
- |Format job names|_ using parameters from config files.
@@ -40,8 +40,7 @@ To submit a job, simply fill in the various parameters shown in the example belo
easy_slurm.submit_job(
job_dir="$HOME/jobs/{date}-{job_name}",
- src="./src",
- assets="./assets",
+ src=["./src", "./assets"],
setup="""
virtualenv "$SLURM_TMPDIR/env"
source "$SLURM_TMPDIR/env/bin/activate"
@@ -51,8 +50,8 @@ To submit a job, simply fill in the various parameters shown in the example belo
# Runs only on subsequent runs. Call setup and do anything else needed.
setup
""",
- on_run="python main.py",
- on_run_resume="python main.py --resume",
+ on_run="cd src && python main.py",
+ on_run_resume="cd src && python main.py --resume",
teardown="""
# Do any cleanup tasks here.
""",
@@ -65,7 +64,7 @@ To submit a job, simply fill in the various parameters shown in the example belo
resubmit_limit=64, # Automatic resubmission limit.
)
-All job files will be kept in the ``job_dir`` directory. Provide directory paths to ``src`` and ``assets`` -- these will be archived and copied to the ``job_dir`` directory. Also provide Bash code in the hooks, which will be run in the following order:
+All job files will be kept in the ``job_dir`` directory. Provide directory paths to ``src`` -- these will be archived and copied to the ``job_dir`` directory. Also provide Bash code in the hooks, which will be run in the following order:
.. list-table:: Hooks order
:widths: 50 50
@@ -95,8 +94,7 @@ Jobs can also be fully configured using YAML files. See `examples/simple_yaml`_.
.. code-block:: yaml
job_dir: "$HOME/jobs/{date}-{job_name}"
- src: "./src"
- assets: "./assets"
+ src: ["./src", "./assets"]
setup: |
virtualenv "$SLURM_TMPDIR/env"
source "$SLURM_TMPDIR/env/bin/activate"
@@ -104,8 +102,8 @@ Jobs can also be fully configured using YAML files. See `examples/simple_yaml`_.
setup_resume: |
# Runs only on subsequent runs. Call setup and do anything else needed.
setup
- on_run: "python main.py"
- on_run_resume: "python main.py --resume"
+ on_run: "cd src && python main.py"
+ on_run_resume: "cd src && python main.py --resume"
teardown: |
# Do any cleanup tasks here.
sbatch_options:
diff --git a/easy_slurm/format.html b/easy_slurm/format.html
index 49d8df4..91adcef 100644
--- a/easy_slurm/format.html
+++ b/easy_slurm/format.html
@@ -266,7 +266,7 @@ easy_slurm.format
-
-easy_slurm.format.format_with_config(template: str, config: dict[str, Any], silent: bool = False, _now: Optional[datetime.datetime] = None) str[source]#
+easy_slurm.format.format_with_config(template: str, config: dict[str, Any], silent: bool = False, _now: Optional[datetime.datetime] = None) str[source]#
Formats template using given config.
The template syntax is very similar to Python string templates.
One useful addition is that nested config keys can be accessed
diff --git a/easy_slurm/jobs.html b/easy_slurm/jobs.html
index 134eab3..6c2ac5c 100644
--- a/easy_slurm/jobs.html
+++ b/easy_slurm/jobs.html
@@ -267,65 +267,63 @@
easy_slurm.jobs
-
-easy_slurm.jobs.create_job_dir(job_dir: str, src: str, assets: str)[source]#
+easy_slurm.jobs.create_job_dir(job_dir: str, src: Sequence[str])[source]#
Creates job directory and freezes all necessary files.
-
-easy_slurm.jobs.create_job_interactive_script_source(sbatch_options: dict[str, Any], job_dir: str, job_path: str, cleanup_seconds: int) str[source]#
+easy_slurm.jobs.create_job_interactive_script_source(sbatch_options: dict[str, Any], job_dir: str, job_path: str, cleanup_seconds: int) str[source]#
Returns source for interactive job script.
-
-easy_slurm.jobs.create_job_script_source(sbatch_options: dict[str, Any], on_run: str, on_run_resume: str, setup: str, setup_resume: str, teardown: str, job_dir: str, cleanup_seconds: int, resubmit_limit: int) str[source]#
+easy_slurm.jobs.create_job_script_source(sbatch_options: dict[str, Any], on_run: str, on_run_resume: str, setup: str, setup_resume: str, teardown: str, job_dir: str, cleanup_seconds: int, resubmit_limit: int) str[source]#
Returns source for job script.
-
-easy_slurm.jobs.submit_job(job_dir: str, *, src: str = '', assets: str = '', on_run: str = '', on_run_resume: str = '', setup: str = '', setup_resume: str = '', teardown: str = '', sbatch_options: dict[str, Any] = {}, cleanup_seconds: int = 120, submit: bool = True, interactive: bool = False, resubmit_limit: int = 64) str[source]#
+easy_slurm.jobs.submit_job(job_dir: str, *, src: Sequence[str] = (), on_run: str = '', on_run_resume: str = '', setup: str = '', setup_resume: str = '', teardown: str = '', sbatch_options: dict[str, Any] = {}, cleanup_seconds: int = 120, submit: bool = True, interactive: bool = False, resubmit_limit: int = 64, config: dict[str, Any] = {}) str[source]#
Submits job.
-Creates job directory with frozen assets and submits job to slurm.
+Creates job directory with frozen src and submits job to slurm.
- Parameters
-job_dir (str) – Path to directory to keep all job files including
-src.tar, assets.tar, and auto-generated job.sh.
src (str) – Path to directory containing only source code.
+
job_dir (str) – Path to directory to keep all job files including
+src.tar and auto-generated job.sh.
src (list[str]) – Path to directories containing only source code.
These will be archived in $JOB_DIR/src.tar and
-extracted during job run into $SLURM_TMPDIR/src.
assets (str) – Path to directory containing additional assets.
-These will be archived in $JOB_DIR/assets.tar and
-extracted during job run into $SLURM_TMPDIR/assets.
on_run (str) – Bash code executed in “on_run” stage, but only for new jobs
+extracted during job run into $SLURM_TMPDIR.
on_run (str) – Bash code executed in “on_run” stage, but only for new jobs
that are running for the first time.
Must be a single command only.
Optionally, the command may gracefully handle interrupts.
on_run_resume (str) – Bash code executed in “on_run” stage, but only for jobs that
+
on_run_resume (str) – Bash code executed in “on_run” stage, but only for jobs that
are resuming from previous incomplete runs.
Must be a single command only.
Optionally, the command may gracefully handle interrupts.
setup (str) – Bash code executed in “setup” stage, but only for new jobs
+
setup (str) – Bash code executed in “setup” stage, but only for new jobs
that are running for the first time.
setup_resume (str) – Bash code executed in “setup” stage, but only for jobs that
+
setup_resume (str) – Bash code executed in “setup” stage, but only for jobs that
are resuming from previous incomplete runs.
To reuse the code from setup, simply set this to
"setup", which calls the code inside the setup
function.
teardown (str) – Bash code executed in “teardown” stage.
sbatch_options (dict[str, Any]) – Dictionary of options to pass to sbatch.
cleanup_seconds (int) – Interrupts a job n seconds before timeout to run cleanup
+
teardown (str) – Bash code executed in “teardown” stage.
sbatch_options (dict[str, Any]) – Dictionary of options to pass to sbatch.
cleanup_seconds (int) – Interrupts a job n seconds before timeout to run cleanup
tasks (teardown, auto-schedule new job).
Default is 120 seconds.
submit (bool) – Submit created job to scheduler. Set this to False if
+
submit (bool) – Submit created job to scheduler. Set this to False if
you are manually submitting the created $JOB_DIR later.
Default is True.
interactive (bool) – Run as a blocking interactive job. Default is False.
resubmit_limit (int) – Maximum number of times to auto-submit a job for “resume”.
+
interactive (bool) – Run as a blocking interactive job. Default is False.
resubmit_limit (int) – Maximum number of times to auto-submit a job for “resume”.
(Not entirely unlike submitting a resume for a job.)
Default is 64 resubmissions.
config (dict[str, Any]) – A dictionary of configuration values to use for formatting.
- Returns
@@ -336,7 +334,7 @@ easy_slurm.jobs
-
-easy_slurm.jobs.submit_job_dir(job_dir: str, interactive: bool)[source]#
+easy_slurm.jobs.submit_job_dir(job_dir: str, interactive: bool)[source]#
Submits a $JOB_DIR created by easy_slurm to slurm.
Note that submit_job already does this for the user,
except when it is called with submit=False.
diff --git a/index.html b/index.html
index c2d6fb9..7f01ac5 100644
--- a/index.html
+++ b/index.html
@@ -353,7 +353,7 @@ Easy Slurm
Features#
-Freezes source code and assets by copying to separate $JOB_DIR.
Freezes source code by copying to separate $JOB_DIR.
Auto-submits another job if current job times out.
Exposes hooks for custom bash code: setup/setup_resume, on_run/on_run_resume, and teardown.
Format job names using parameters from config files.
Usage
easy_slurm.submit_job(
job_dir="$HOME/jobs/{date}-{job_name}",
- src="./src",
- assets="./assets",
+ src=["./src", "./assets"],
setup="""
virtualenv "$SLURM_TMPDIR/env"
source "$SLURM_TMPDIR/env/bin/activate"
@@ -384,8 +383,8 @@ Usage
# Runs only on subsequent runs. Call setup and do anything else needed.
setup
""",
- on_run="python main.py",
- on_run_resume="python main.py --resume",
+ on_run="cd src && python main.py",
+ on_run_resume="cd src && python main.py --resume",
teardown="""
# Do any cleanup tasks here.
""",
@@ -399,7 +398,7 @@ Usage
)
-All job files will be kept in the job_dir directory. Provide directory paths to src and assets – these will be archived and copied to the job_dir directory. Also provide Bash code in the hooks, which will be run in the following order:
+All job files will be kept in the job_dir directory. Provide directory paths to src – these will be archived and copied to the job_dir directory. Also provide Bash code in the hooks, which will be run in the following order:
Hooks order#
@@ -428,8 +427,7 @@ Usage
YAML#
Jobs can also be fully configured using YAML files. See examples/simple_yaml.