Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

No job tag in out_fpath for single-node jobs #21

Merged
merged 4 commits into from
Sep 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 30 additions & 10 deletions gaps/cli/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,10 @@ def __init__(self, ctx, config_file, command_config):
self.exec_kwargs = None
self.logging_options = None
self.exclude_from_status = None
self._include_tag_in_out_fpath = (
self.command_config.is_split_spatially
and self.config.get("execution_control", {}).get("nodes", 1) > 1
)

@property
def project_dir(self):
Expand Down Expand Up @@ -228,16 +232,9 @@ def kickoff_jobs(self):
jobs = sorted(product(*lists_to_run))
num_jobs_submit = len(jobs)
self._warn_about_excessive_au_usage(num_jobs_submit)
n_zfill = len(str(max(0, num_jobs_submit - 1)))
extra_exec_args = {}
for param in EXTRA_EXEC_PARAMS:
if param in self.exec_kwargs:
extra_exec_args[param] = self.exec_kwargs.pop(param)
extra_exec_args = self._extract_extra_exec_args_for_command()
for node_index, values in enumerate(jobs):
if num_jobs_submit > 1:
tag = f"{TAG}{str(node_index).zfill(n_zfill)}"
else:
tag = ""
tag = _tag(node_index, num_jobs_submit)
self.ctx.obj["NAME"] = job_name = f"{self.job_name}{tag}"
node_specific_config = deepcopy(self.config)
node_specific_config.pop("execution_control", None)
Expand All @@ -249,7 +246,7 @@ def kickoff_jobs(self):
"project_dir": self.project_dir.as_posix(),
"job_name": job_name,
"out_dir": self.project_dir.as_posix(),
"out_fpath": (self.project_dir / job_name).as_posix(),
"out_fpath": self._suggested_stem(job_name).as_posix(),
"run_method": getattr(
self.command_config, "run_method", None
),
Expand Down Expand Up @@ -278,6 +275,21 @@ def kickoff_jobs(self):

return self

def _suggested_stem(self, job_name_with_tag):
"""Determine suggested filepath with filename stem."""
if self._include_tag_in_out_fpath:
return self.project_dir / job_name_with_tag
return self.project_dir / self.job_name

def _extract_extra_exec_args_for_command(self):
"""Dictionary of function args from the exec block."""
extra_exec_args = {}
for param in EXTRA_EXEC_PARAMS:
if param not in self.exec_kwargs:
continue
extra_exec_args[param] = self.exec_kwargs.pop(param)
return extra_exec_args

def _keys_and_lists_to_run(self):
"""Compile run lists based on `command_config.split_keys` input."""
keys_to_run = []
Expand Down Expand Up @@ -422,6 +434,14 @@ def _project_points_last(key):
return key


def _tag(node_index, num_jobs):
"""Determine node tag based on total number of jobs."""
n_zfill = len(str(max(0, num_jobs - 1)))
if num_jobs > 1:
return f"{TAG}{str(node_index).zfill(n_zfill)}"
return ""


def as_script_str(input_):
"""Convert input to how it would appear in a python script.

Expand Down
2 changes: 1 addition & 1 deletion gaps/version.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""GAPs Version Number. """

__version__ = "0.4.4"
__version__ = "0.4.5"
Loading
Loading