Skip to content

Commit

Permalink
code working ( together with autosubmitconfig parser update ) pytest …
Browse files Browse the repository at this point in the history
…missing
  • Loading branch information
dbeltrankyl committed Nov 25, 2024
1 parent b2c19e4 commit 1cfa0c0
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 52 deletions.
3 changes: 2 additions & 1 deletion autosubmit/autosubmit.py
Original file line number Diff line number Diff line change
Expand Up @@ -2062,7 +2062,8 @@ def prepare_run(expid, notransitive=False, start_time=None, start_after=None,
job.platform_name = hpcarch
# noinspection PyTypeChecker
try:
job.platform = submitter.platforms[job.platform_name.upper()]
job.platform_name = as_conf.experiment_data["PLATFORMS"].get(job.platform_name.upper(), hpcarch)
job.platform = submitter.platforms[job.platform_name.upper()]
except Exception as e:
raise AutosubmitCritical(
"hpcarch={0} not found in the platforms configuration file".format(job.platform_name),
Expand Down
2 changes: 1 addition & 1 deletion autosubmit/job/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -940,7 +940,7 @@ def add_edge_info(self, parent, special_conditions):
if special_conditions["STATUS"] not in self.edge_info:
self.edge_info[special_conditions["STATUS"]] = {}

self.edge_info[special_conditions["STATUS"]][parent.name] = (parent, special_conditions.get("FROM_STEP", 0), special_conditions.get("OPTIONAL", False))
self.edge_info[special_conditions["STATUS"]][parent.name] = (parent, special_conditions.get("FROM_STEP", 0), special_conditions.get("ANY_FINAL_STATUS_IS_VALID", False))

def delete_parent(self, parent):
"""
Expand Down
23 changes: 12 additions & 11 deletions autosubmit/job/job_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -987,7 +987,7 @@ def _apply_jobs_edge_info(self, job, dependencies):
special_conditions = dict()
special_conditions["STATUS"] = filters_to_apply_by_section[key].pop("STATUS", None)
special_conditions["FROM_STEP"] = filters_to_apply_by_section[key].pop("FROM_STEP", None)
special_conditions["OPTIONAL"] = filters_to_apply_by_section[key].pop("OPTIONAL", False)
special_conditions["ANY_FINAL_STATUS_IS_VALID"] = filters_to_apply_by_section[key].pop("ANY_FINAL_STATUS_IS_VALID", False)
for parent in list_of_parents:
self.add_special_conditions(job, special_conditions, filters_to_apply_by_section[key],
parent)
Expand Down Expand Up @@ -1214,7 +1214,7 @@ def _calculate_filter_dependencies(self, filters_to_apply, dic_jobs, job, depend
def get_filters_to_apply(self, job, dependency):
filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships))
filters_to_apply.pop("STATUS", None)
filters_to_apply.pop("OPTIONAL", None)
filters_to_apply.pop("ANY_FINAL_STATUS_IS_VALID", None)
# Don't do perform special filter if only "FROM_STEP" is applied
if "FROM_STEP" in filters_to_apply:
if filters_to_apply.get("CHUNKS_TO","none") == "none" and filters_to_apply.get("MEMBERS_TO","none") == "none" and filters_to_apply.get("DATES_TO","none") == "none" and filters_to_apply.get("SPLITS_TO","none") == "none":
Expand Down Expand Up @@ -2638,7 +2638,7 @@ def check_special_status(self) -> List[Job]:
continue
if target_status in ["RUNNING", "FAILED"]:
self._check_checkpoint(job)
relations_unsatisfated, _ = self._count_parents_status(job, target_status)
relations_unsatisfated, _ = self._check_relationship_is_ready(job, target_status)
if not relations_unsatisfated:
jobs_to_check.append(job)
return jobs_to_check
Expand All @@ -2654,15 +2654,18 @@ def _check_checkpoint(job: Job) -> None:
job.get_checkpoint_files()

@staticmethod
def _count_parents_status(job: Job, target_status_key: str) -> Tuple[List[Job], List[Job]]:
def _check_relationship_is_ready(job: Job, target_status_key: str) -> Tuple[List[Job], List[Job]]:
"""
Count the number of relations satisfated and relations unsatisfated for a job.
Check the correctness of the relationship between a job and its parents.
:param job: The job to check.
:param target_status: The target status to compare against.
Default: For a relation to be satisfated, the parent must be in COMPLETED status.
Special: For a relation to be satisfated, the parent must be equal or superior to target status.
:param job: The job to check. job.edge_info: contains a tuple with (parent, checkpoint_number, optional)
:param target_status_key: The target status to compare against.
:return: A tuple containing two lists:
- non_completed_parents_current: Non-completed parents.
- completed_parents: Completed parents.
- relation_unsatisfated: Parent jobs that doesn't satisfate the relationship.
- relation_satisfated: Parent jobs that satisfate the relationship.
"""
relation_satisfated = []
relation_unsatisfated = []
Expand Down Expand Up @@ -2690,8 +2693,6 @@ def _count_parents_status(job: Job, target_status_key: str) -> Tuple[List[Job],

return relation_unsatisfated, relation_satisfated

#def _check_relation_is_still_valid(self, job, parent):

def update_log_status(self, job, as_conf):
"""
Updates the log err and log out.
Expand Down
63 changes: 24 additions & 39 deletions autosubmit/monitor/monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from log.log import Log, AutosubmitCritical
from autosubmitconfigparser.config.yamlparser import YAMLParserFactory

from typing import Dict, List
from typing import Dict, List, Optional, Tuple

GENERAL_STATS_OPTION_MAX_LENGTH = 1000

Expand Down Expand Up @@ -278,46 +278,31 @@ def create_tree_list(self, expid, joblist, packages, groups, hide_groups=False):
Log.debug('Graph definition finalized')
return graph

def _check_final_status(self, job, child):
# order of self._table
# the dictionary is composed by:
def _check_final_status(self, job: Job, child: Job) -> Tuple[Optional[str], Optional[str], bool]:
"""
Check the final status of a job in relation to its child.
:param job: The job to check.
:type job: Job
:param child: The child job to check against.
:type child: Job
:return: A tuple containing:
- color: The color associated with the status.
- label: The label associated with the status.
- dashed: Whether the relation line should be dashed or straight
:rtype: Tuple[Optional[str], Optional[str], bool]
"""
color = None
label = None
optional = False
dashed = False
if len(child.edge_info) > 0:
if job.name in child.edge_info.get("FAILED", {}):
color = self._table.get(Status.FAILED, None)
label = None if child.edge_info["FAILED"].get(job.name, None) is None else child.edge_info["FAILED"][job.name][1]
optional = None if child.edge_info["FAILED"].get(job.name, None) is None else child.edge_info["FAILED"][job.name][2]
elif job.name in child.edge_info.get("RUNNING", {}):
color = self._table.get(Status.RUNNING, None)
label = child.edge_info["RUNNING"].get(job.name, 0)[1]
optional = True # Always optional if it's running

elif job.name in child.edge_info.get("QUEUING", {}):
color = self._table.get(Status.QUEUING, None)
elif job.name in child.edge_info.get("HELD", {}):
color = self._table.get(Status.HELD, None)
elif job.name in child.edge_info.get("DELAYED", {}):
color = self._table.get(Status.DELAYED, None)
elif job.name in child.edge_info.get("UNKNOWN", {}):
color = self._table.get(Status.UNKNOWN, None)
elif job.name in child.edge_info.get("SUSPENDED", {}):
color = self._table.get(Status.SUSPENDED, None)
elif job.name in child.edge_info.get("SKIPPED", {}):
color = self._table.get(Status.SKIPPED, None)
elif job.name in child.edge_info.get("WAITING", {}):
color = self._table.get(Status.WAITING, None)
elif job.name in child.edge_info.get("READY", {}):
color = self._table.get(Status.READY, None)
elif job.name in child.edge_info.get("SUBMITTED", {}):
color = self._table.get(Status.SUBMITTED, None)
else:
return None, None, optional
if label and label == 0:
label = None
return color, label, optional
else:
return None, None, optional
for section, data in child.edge_info.items():
parent_info = data.get(job.name, None)
if parent_info:
color = self._table[Status.KEY_TO_VALUE[section]]
label = parent_info[1]
dashed = parent_info[2]
return color, label, dashed

def _add_children(self, job, exp, node_job, groups, hide_groups):
if job in self.nodes_plotted:
Expand Down

0 comments on commit 1cfa0c0

Please sign in to comment.