Skip to content

Commit

Permalink
Show duration for which SLURM job has been running
Browse files Browse the repository at this point in the history
  • Loading branch information
mhrtmnn committed Nov 28, 2020
1 parent ff4a8e9 commit b7c0a20
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion toolflow/scala/src/main/scala/tapasco/slurm/Slurm.scala
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import tapasco.filemgmt._
import tapasco.task.ResourceConsumer
import tapasco.util.{Publisher, Template}

import scala.concurrent.duration.Duration
import scala.collection.JavaConverters._
import scala.sys.ShutdownHookThread
import scala.sys.process._
Expand Down Expand Up @@ -375,9 +376,12 @@ final object Slurm extends Publisher {
/** Wait until the given SLURM job is not listed as RUNNING anymore in `sacct` output. */
def waitFor(id: Int): SlurmStatus = {
val hook = ShutdownHookThread(Slurm.cancel(id))
val start = System.currentTimeMillis()
var status: SlurmStatus = Slurm.Running()
while (status == Running()) { // can be cancelled by SIGINT
logger.info("SLURM job #%d is still running, sleeping for %d secs ...".format(id, slurmDelay / 1000))
val dur = Duration(System.currentTimeMillis() - start, "millis")
logger.info("SLURM job #%d is running since %dh %02dm %02ds"
.format(id, dur.toHours, dur.toMinutes % 60, dur.toSeconds % 60))
Thread.sleep(slurmDelay)
status = getSlurmStatus(id)
}
Expand Down

0 comments on commit b7c0a20

Please sign in to comment.