From dcb8b0019e442e32b68344a1ef5489f3a4a3c0c6 Mon Sep 17 00:00:00 2001
From: Nehal J Wani <nehaljw.kkd1@gmail.com>
Date: Fri, 17 Nov 2017 22:01:03 +0530
Subject: [PATCH] Add option to apply caps only on undead pods

A pod can be in one of five phases: running, pending, succeeded, failed
or unknown. The pods which are not in the running/pending phase do not
consume any CPU/Memory resources, they just need to be GC'ed. If, in the
scenario when a cap has been specified on a k8s cloud (single namespace)
and let's say 40% of the count are in the 'failed' phase, and 60% are
'runnning/pending', newer pods will not be spawned in that namespace,
since the plugin will count all those pods for instance/pod cap and even
though it could have spawned 40% more pods, it won't and jobs will
starve. This patch adds an option to calculate the cap for pods only in
the 'running' or 'pending' phase, on a cloud level and on a pod template
level.
---
 .../plugins/kubernetes/KubernetesCloud.java   | 28 +++++++++++++++++++
 .../plugins/kubernetes/PodTemplate.java       | 11 ++++++++
 .../kubernetes/KubernetesCloud/config.jelly   |  4 +++
 .../kubernetes/PodTemplate/config.jelly       |  4 +++
 4 files changed, 47 insertions(+)

diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java
index aa15bbbc20..8bf3d7ed03 100644
--- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java
+++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java
@@ -18,6 +18,7 @@
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
@@ -91,6 +92,8 @@ public class KubernetesCloud extends Cloud {
 
     private boolean skipTlsVerify;
 
+    private boolean capOnlyOnUnDeadPods;
+
     private String namespace;
     private String jenkinsUrl;
     @CheckForNull
@@ -220,6 +223,15 @@ public String getJenkinsUrl() {
         return jenkinsUrl;
     }
 
+    @DataBoundSetter
+    public void setCapOnlyOnUnDeadPods(boolean capOnlyOnUnDeadPods) {
+        this.capOnlyOnUnDeadPods = capOnlyOnUnDeadPods;
+    }
+
+    public boolean isCapOnlyOnUnDeadPods() {
+        return capOnlyOnUnDeadPods;
+    }
+
     /**
      * Returns Jenkins URL to be used by agents launched by this cloud. Always ends with a trailing slash.
      *
@@ -424,6 +436,22 @@ private boolean addProvisionedSlave(@Nonnull PodTemplate template, @CheckForNull
         PodList namedList = client.pods().inNamespace(templateNamespace).withLabels(labelsMap).list();
         List<Pod> namedListItems = namedList.getItems();
 
+        if (this.isCapOnlyOnUnDeadPods()) {
+            slaveListItems = slaveListItems.stream()
+                                           .filter(x -> x.getStatus()
+                                                         .getPhase().toLowerCase()
+                                                                    .matches("(running|pending)"))
+                                           .collect(Collectors.toList());
+        }
+
+        if (template.isCapOnlyOnUnDeadPods()) {
+            namedListItems = namedListItems.stream()
+                                           .filter(x -> x.getStatus()
+                                                         .getPhase().toLowerCase()
+                                                                    .matches("(running|pending)"))
+                                           .collect(Collectors.toList());
+        }
+
         if (slaveListItems != null && containerCap <= slaveListItems.size()) {
             LOGGER.log(Level.INFO,
                     "Total container cap of {0} reached, not provisioning: {1} running or errored in namespace {2}",
diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java
index a71082c224..098b72d6a5 100644
--- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java
+++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java
@@ -56,6 +56,8 @@ public class PodTemplate extends AbstractDescribableImpl<PodTemplate> implements
 
     private boolean privileged;
 
+    private boolean capOnlyOnUnDeadPods;
+
     private boolean alwaysPullImage;
 
     private String command;
@@ -394,6 +396,15 @@ public boolean isAlwaysPullImage() {
         return getFirstContainer().map(ContainerTemplate::isAlwaysPullImage).orElse(false);
     }
 
+    @DataBoundSetter
+    public void setCapOnlyOnUnDeadPods(boolean capOnlyOnUnDeadPods) {
+        this.capOnlyOnUnDeadPods = capOnlyOnUnDeadPods;
+    }
+
+    public boolean isCapOnlyOnUnDeadPods() {
+        return capOnlyOnUnDeadPods;
+    }
+
     public List<TemplateEnvVar> getEnvVars() {
         if (envVars == null) {
             return Collections.emptyList();
diff --git a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly
index 31aaa7aa3a..e570576668 100644
--- a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly
+++ b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly
@@ -47,6 +47,10 @@
         <f:textbox default="10"/>
     </f:entry>
 
+    <f:entry field="capOnlyOnUnDeadPods" title="${%Apply cap only on alive pods}">
+        <f:checkbox/>
+    </f:entry>
+
     <f:entry title="${%Max connections to Kubernetes API}" field="maxRequestsPerHostStr">
         <f:textbox default="32"/>
     </f:entry>
diff --git a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/PodTemplate/config.jelly b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/PodTemplate/config.jelly
index bd9b50e8fc..cc87e4f9c0 100644
--- a/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/PodTemplate/config.jelly
+++ b/src/main/resources/org/csanchez/jenkins/plugins/kubernetes/PodTemplate/config.jelly
@@ -48,6 +48,10 @@
     <f:textbox/>
   </f:entry>
 
+  <f:entry field="capOnlyOnUnDeadPods" title="${%Apply cap only on alive pods}">
+    <f:checkbox/>
+  </f:entry>
+
   <f:entry field="idleMinutesStr" title="${%Time in minutes to retain slave when idle}">
     <f:textbox/>
   </f:entry>