Issue #3320 Slurm autoscaler effective job start date parsing

workflows/pipe-common/pipeline/hpc/autoscaler.py

@@ -368,6 +368,7 @@ def scale_down(self, child_host):
         Logger.info('Disabling additional worker %s...' % child_host)
         self.grid_engine.disable_host(child_host)
         jobs = self.grid_engine.get_jobs()
+        # todo: Ignore canceled/completed job here and etc.
         disabled_host_jobs = [job for job in jobs if child_host in job.hosts]
         if disabled_host_jobs:
             Logger.warn('Disabled additional worker %s has %s associated jobs. Scaling down is interrupted.'

workflows/pipe-common/pipeline/hpc/engine/slurm.py

@@ -107,13 +107,18 @@ def _parse_jobs(self, scontrol_jobs_output):
             # this job to `num_node` nodes and provide portion of resources
             # TODO maybe there is another way to achieve that?
             for node_idx in range(num_node):
+                job_state = GridEngineJobState.from_letter_code(job_dict.get("JobState"))
+                if job_state == GridEngineJobState.PENDING:
+                    # In certain cases pending job's start date can be estimated start date.
+                    # It confuses autoscaler and therefore should be ignored.
+                    job_dict["StartTime"] = "Unknown"
                 jobs.append(
                     GridEngineJob(
                         id=job_dict.get("JobId") + "_" + str(node_idx),
                         root_id=job_dict.get("JobId"),
                         name=job_dict.get("JobName"),
                         user=self._parse_user(job_dict.get("UserId")),
-                        state=GridEngineJobState.from_letter_code(job_dict.get("JobState")),
+                        state=job_state,
                         datetime=self._parse_date(
                             job_dict.get("StartTime") if job_dict.get("StartTime") != "Unknown" else job_dict.get("SubmitTime")),
                         hosts=self._parse_nodelist(job_dict.get("NodeList")),