feat(manager): increase termination period of run-batch pods (#572)

reanahub · Mar 1, 2024 · f05096a · f05096a
1 parent b013d49
commit f05096a
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py
@@ -208,3 +208,11 @@ def _env_vars_dict_to_k8s_list(env_vars):
     RunStatus.pending,
 ]
 """Alive workflow statuses."""
+
+REANA_RUNTIME_BATCH_TERMINATION_GRACE_PERIOD = int(
+    os.getenv("REANA_RUNTIME_BATCH_TERMINATION_GRACE_PERIOD", "120")
+)
+"""Grace period before terminating the job controller and workflow engine pod.
+
+The job controller needs to clean up all the running jobs before the end of the grace period.
+"""
diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py
@@ -63,6 +63,7 @@
     JOB_CONTROLLER_CONTAINER_PORT,
     JOB_CONTROLLER_ENV_VARS,
     JOB_CONTROLLER_SHUTDOWN_ENDPOINT,
+    REANA_RUNTIME_BATCH_TERMINATION_GRACE_PERIOD,
     REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT,
     REANA_KUBERNETES_JOBS_MEMORY_LIMIT,
     REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT,
@@ -662,6 +663,7 @@ def _create_job_spec(
             containers=containers,
             node_selector=REANA_RUNTIME_BATCH_KUBERNETES_NODE_LABEL,
             init_containers=[],
+            termination_grace_period_seconds=REANA_RUNTIME_BATCH_TERMINATION_GRACE_PERIOD,
         )
         spec.template.spec.service_account_name = (
             REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME