Add option to apply caps only on alive pods

A pod can be in one of five phases: running, pending, succeeded, failed
or unknown. The pods which are not in the running/pending phase do not
consume any CPU/Memory resources, they just need to be GC'ed. If, in the
scenario when a cap has been specified on a k8s cloud (single namespace)
and let's say 40% of the count are in the 'failed' phase, and 60% are
'runnning/pending', newer pods will not be spawned in that namespace,
since the plugin will count all those pods for instance/pod cap and even
though it could have spawned 40% more pods, it won't and jobs will
starve. This patch adds an option to calculate the cap for pods only in
the 'running' or 'pending' phase, on a cloud level and on a pod template
level, so that the cap applies to only those pods which are alive or
about to be.

Change-Id: Id77e837aa9a42742618cd3c543e7b99f9d38a10a

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java

@@ -18,6 +18,7 @@
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
@@ -91,6 +92,8 @@ public class KubernetesCloud extends Cloud {
 
     private boolean skipTlsVerify;
 
+    private boolean capOnlyOnAlivePods;
+
     private String namespace;
     private String jenkinsUrl;
     @CheckForNull
@@ -220,6 +223,15 @@ public String getJenkinsUrl() {
         return jenkinsUrl;
     }
 
+    @DataBoundSetter
+    public void setCapOnlyOnAlivePods(boolean capOnlyOnAlivePods) {
+        this.capOnlyOnAlivePods = capOnlyOnAlivePods;
+    }
+
+    public boolean isCapOnlyOnAlivePods() {
+        return capOnlyOnAlivePods;
+    }
+
     /**
      * Returns Jenkins URL to be used by agents launched by this cloud. Always ends with a trailing slash.
      *
@@ -424,6 +436,22 @@ private boolean addProvisionedSlave(@Nonnull PodTemplate template, @CheckForNull
         PodList namedList = client.pods().inNamespace(templateNamespace).withLabels(labelsMap).list();
         List<Pod> namedListItems = namedList.getItems();
 
+        if (this.isCapOnlyOnAlivePods()) {
+            slaveListItems = slaveListItems.stream()
+                                           .filter(x -> x.getStatus()
+                                                         .getPhase().toLowerCase()
+                                                                    .matches("(running|pending)"))
+                                           .collect(Collectors.toList());
+        }
+
+        if (template.isCapOnlyOnAlivePods()) {
+            namedListItems = namedListItems.stream()
+                                           .filter(x -> x.getStatus()
+                                                         .getPhase().toLowerCase()
+                                                                    .matches("(running|pending)"))
+                                           .collect(Collectors.toList());
+        }
+
         if (slaveListItems != null && containerCap <= slaveListItems.size()) {
             LOGGER.log(Level.INFO,
                     "Total container cap of {0} reached, not provisioning: {1} running or errored in namespace {2}",

src/main/java/org/csanchez/jenkins/plugins/kubernetes/PodTemplate.java

@@ -56,6 +56,8 @@ public class PodTemplate extends AbstractDescribableImpl<PodTemplate> implements
 
     private boolean privileged;
 
+    private boolean capOnlyOnAlivePods;
+
     private boolean alwaysPullImage;
 
     private String command;
@@ -394,6 +396,15 @@ public boolean isAlwaysPullImage() {
         return getFirstContainer().map(ContainerTemplate::isAlwaysPullImage).orElse(false);
     }
 
+    @DataBoundSetter
+    public void setCapOnlyOnAlivePods(boolean capOnlyOnAlivePods) {
+        this.capOnlyOnAlivePods = capOnlyOnAlivePods;
+    }
+
+    public boolean isCapOnlyOnAlivePods() {
+        return capOnlyOnAlivePods;
+    }
+
     public List<TemplateEnvVar> getEnvVars() {
         if (envVars == null) {
             return Collections.emptyList();

src/main/resources/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud/config.jelly

@@ -47,6 +47,10 @@
         <f:textbox default="10"/>
     </f:entry>
 
+    <f:entry field="capOnlyOnAlivePods" title="${%Apply cap only on alive pods}">
+        <f:checkbox/>
+    </f:entry>
+
     <f:entry title="${%Max connections to Kubernetes API}" field="maxRequestsPerHostStr">
         <f:textbox default="32"/>
     </f:entry>

src/main/resources/org/csanchez/jenkins/plugins/kubernetes/PodTemplate/config.jelly

@@ -48,6 +48,10 @@
     <f:textbox/>
   </f:entry>
 
+  <f:entry field="capOnlyOnAlivePods" title="${%Apply cap only on alive pods}">
+    <f:checkbox/>
+  </f:entry>
+
   <f:entry field="idleMinutesStr" title="${%Time in minutes to retain slave when idle}">
     <f:textbox/>
   </f:entry>