[JENKINS-45910] Delete pods in the cloud namespace when pod namespace is not defined

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java

@@ -35,6 +35,8 @@
 import org.csanchez.jenkins.plugins.kubernetes.pipeline.PodTemplateStepExecution;
 import org.csanchez.jenkins.plugins.kubernetes.volumes.PodVolume;
 import org.jenkinsci.plugins.durabletask.executors.OnceRetentionStrategy;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
 import org.kohsuke.stapler.DataBoundConstructor;
 import org.kohsuke.stapler.DataBoundSetter;
 import org.kohsuke.stapler.QueryParameter;
@@ -113,7 +115,7 @@ public class KubernetesCloud extends Cloud {
             .getProperty(PodTemplateStepExecution.class.getName() + ".defaultImage", "jenkinsci/jnlp-slave:alpine");
 
     /** label for all pods started by the plugin */
-    private static final Map<String, String> POD_LABEL = ImmutableMap.of("jenkins", "slave");
+    public static final Map<String, String> DEFAULT_POD_LABELS = ImmutableMap.of("jenkins", "slave");
 
     private static final String JNLPMAC_REF = "\\$\\{computer.jnlpmac\\}";
     private static final String NAME_REF = "\\$\\{computer.name\\}";
@@ -324,8 +326,7 @@ public KubernetesClient connect() throws UnrecoverableKeyException, NoSuchAlgori
                 new String[] { getDisplayName(), serverUrl });
         client = new KubernetesFactoryAdapter(serverUrl, namespace, serverCertificate, credentialsId, skipTlsVerify,
                 connectTimeout, readTimeout, maxRequestsPerHost).createClient();
-        LOGGER.log(Level.FINE, "Connected to Kubernetes {0} URL {1}" + serverUrl,
-                new String[] { getDisplayName(), serverUrl });
+        LOGGER.log(Level.FINE, "Connected to Kubernetes {0} URL {1}", new String[] { getDisplayName(), serverUrl });
         return client;
     }
 
@@ -497,7 +498,7 @@ private Pod getPodTemplate(KubernetesSlave slave, PodTemplate template) {
 
     private Map<String, String> getLabelsMap(Set<LabelAtom> labelSet) {
         ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String> builder();
-        builder.putAll(POD_LABEL);
+        builder.putAll(DEFAULT_POD_LABELS);
         if (!labelSet.isEmpty()) {
             for (LabelAtom label: labelSet) {
                 builder.put(getIdForLabel(label), "true");
@@ -809,7 +810,7 @@ private boolean addProvisionedSlave(@Nonnull PodTemplate template, @CheckForNull
             templateNamespace = client.getNamespace();
         }
 
-        PodList slaveList = client.pods().inNamespace(templateNamespace).withLabels(POD_LABEL).list();
+        PodList slaveList = client.pods().inNamespace(templateNamespace).withLabels(DEFAULT_POD_LABELS).list();
         List<Pod> slaveListItems = slaveList.getItems();
 
         Map<String, String> labelsMap = getLabelsMap(template.getLabelSet());

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java

@@ -1,6 +1,10 @@
 package org.csanchez.jenkins.plugins.kubernetes;
 
 import java.io.IOException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateEncodingException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -12,6 +16,7 @@
 import org.kohsuke.stapler.DataBoundConstructor;
 
 import hudson.Extension;
+import hudson.Util;
 import hudson.model.Computer;
 import hudson.model.Descriptor;
 import hudson.model.Label;
@@ -22,10 +27,8 @@
 import hudson.slaves.JNLPLauncher;
 import hudson.slaves.OfflineCause;
 import hudson.slaves.RetentionStrategy;
-import io.fabric8.kubernetes.api.model.DoneablePod;
-import io.fabric8.kubernetes.api.model.Pod;
 import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.dsl.PodResource;
+import io.fabric8.kubernetes.client.KubernetesClientException;
 import jenkins.model.Jenkins;
 
 /**
@@ -80,9 +83,8 @@ public KubernetesSlave(PodTemplate template, String nodeDescription, String clou
                 rs,
                 template.getNodeProperties());
 
-        // this.pod = pod;
         this.cloudName = cloudName;
-        this.namespace = template.getNamespace();
+        this.namespace = Util.fixEmpty(template.getNamespace());
     }
 
     public String getCloudName() {
@@ -131,35 +133,60 @@ protected void _terminate(TaskListener listener) throws IOException, Interrupted
             String msg = String.format("Cloud name is not set for agent, can't terminate: %s", name);
             LOGGER.log(Level.SEVERE, msg);
             listener.fatalError(msg);
+            computer.disconnect(OfflineCause.create(new Localizable(HOLDER, "offline")));
+            return;
+        }
+
+        Cloud cloud = getCloud();
+        if (cloud == null) {
+            String msg = String.format("Agent cloud no longer exists: %s", getCloudName());
+            LOGGER.log(Level.WARNING, msg);
+            listener.fatalError(msg);
+            computer.disconnect(OfflineCause.create(new Localizable(HOLDER, "offline")));
+            return;
+        }
+        if (!(cloud instanceof KubernetesCloud)) {
+            String msg = String.format("Agent cloud is not a KubernetesCloud, something is very wrong: %s",
+                    getCloudName());
+            LOGGER.log(Level.SEVERE, msg);
+            listener.fatalError(msg);
+            computer.disconnect(OfflineCause.create(new Localizable(HOLDER, "offline")));
+            return;
+        }
+        KubernetesClient client;
+        try {
+            client = ((KubernetesCloud) cloud).connect();
+        } catch (UnrecoverableKeyException | CertificateEncodingException | NoSuchAlgorithmException
+                | KeyStoreException e) {
+            String msg = String.format("Failed to connect to cloud %s", getCloudName());
+            listener.fatalError(msg);
+            computer.disconnect(OfflineCause.create(new Localizable(HOLDER, "offline")));
             return;
         }
 
+        String actualNamespace = getNamespace() == null ? client.getNamespace() : getNamespace();
         try {
-            Cloud cloud = getCloud();
-            if (cloud == null) {
-                String msg = String.format("Slave cloud no longer exists: %s", getCloudName());
+            Boolean deleted = client.pods().inNamespace(actualNamespace).withName(name).delete();
+            if (!Boolean.TRUE.equals(deleted)) {
+                String msg = String.format("Failed to delete pod for agent %s/%s: not found", actualNamespace, name);
                 LOGGER.log(Level.WARNING, msg);
-                listener.fatalError(msg);
-                return;
-            }
-            if (!(cloud instanceof KubernetesCloud)) {
-                String msg = String.format("Slave cloud is not a KubernetesCloud, something is very wrong: %s",
-                        getCloudName());
-                LOGGER.log(Level.SEVERE, msg);
-                listener.fatalError(msg);
+                listener.error(msg);
                 return;
             }
-            KubernetesClient client = ((KubernetesCloud) cloud).connect();
-            PodResource<Pod, DoneablePod> pods = client.pods().inNamespace(namespace).withName(name);
-            pods.delete();
-            String msg = String.format("Terminated Kubernetes instance for slave %s", name);
-            LOGGER.log(Level.INFO, msg);
-            listener.getLogger().println(msg);
+        } catch (KubernetesClientException e) {
+            String msg = String.format("Failed to delete pod for agent %s/%s: %s", actualNamespace, name,
+                    e.getMessage());
+            LOGGER.log(Level.WARNING, msg, e);
+            listener.error(msg);
             computer.disconnect(OfflineCause.create(new Localizable(HOLDER, "offline")));
-            LOGGER.log(Level.INFO, "Disconnected computer {0}", name);
-        } catch (Exception e) {
-            LOGGER.log(Level.SEVERE, "Failed to terminate pod for slave " + name, e);
+            return;
         }
+
+        String msg = String.format("Terminated Kubernetes instance for agent %s/%s", actualNamespace, name);
+        LOGGER.log(Level.INFO, msg);
+        listener.getLogger().println(msg);
+        computer.disconnect(OfflineCause.create(new Localizable(HOLDER, "offline")));
+        LOGGER.log(Level.INFO, "Disconnected computer {0}", name);
     }
 
     @Override

src/main/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/PodTemplateStepExecution.java

@@ -15,6 +15,7 @@
 import hudson.AbortException;
 import hudson.model.Run;
 import hudson.slaves.Cloud;
+import io.fabric8.kubernetes.client.KubernetesClient;
 import jenkins.model.Jenkins;
 
 public class PodTemplateStepExecution extends AbstractStepExecutionImpl {
@@ -124,14 +125,25 @@ private PodTemplateCallback(PodTemplate podTemplate) {
         protected void finished(StepContext context) throws Exception {
             Cloud cloud = Jenkins.getInstance().getCloud(step.getCloud());
             if (cloud == null) {
-                LOGGER.log(Level.FINE, "Cloud {0} no longer exists, cannot delete pod template {1}",
+                LOGGER.log(Level.WARNING, "Cloud {0} no longer exists, cannot delete pod template {1}",
                         new Object[] { step.getCloud(), podTemplate.getName() });
                 return;
             }
             if (cloud instanceof KubernetesCloud) {
+                LOGGER.log(Level.INFO, "Removing pod template and deleting pod {1} from cloud {0}",
+                        new Object[] { cloud.name, podTemplate.getName() });
                 KubernetesCloud kubernetesCloud = (KubernetesCloud) cloud;
                 kubernetesCloud.removeTemplate(podTemplate);
-                kubernetesCloud.connect().pods().withName(podTemplate.getName()).delete();
+                KubernetesClient client = kubernetesCloud.connect();
+                Boolean deleted = client.pods().withName(podTemplate.getName()).delete();
+                if (!Boolean.TRUE.equals(deleted)) {
+                    LOGGER.log(Level.WARNING, "Failed to delete pod for agent {0}/{1}: not found",
+                            new String[] { client.getNamespace(), podTemplate.getName() });
+                    return;
+                }
+            } else {
+                LOGGER.log(Level.WARNING, "Cloud is not a KubernetesCloud: {0} {1}",
+                        new String[] { cloud.name, cloud.getClass().getName() });
             }
         }
     }

src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesTestUtil.java

@@ -24,6 +24,7 @@
 package org.csanchez.jenkins.plugins.kubernetes;
 
 import static io.fabric8.kubernetes.client.Config.*;
+import static java.util.logging.Level.*;
 import static org.hamcrest.Matchers.*;
 import static org.junit.Assume.*;
 
@@ -33,22 +34,35 @@
 import java.security.NoSuchAlgorithmException;
 import java.security.UnrecoverableKeyException;
 import java.security.cert.CertificateEncodingException;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
-
-import io.fabric8.kubernetes.client.KubernetesClientException;
-import org.junit.Assume;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import io.fabric8.kubernetes.api.model.Cluster;
 import io.fabric8.kubernetes.api.model.Config;
 import io.fabric8.kubernetes.api.model.NamedCluster;
 import io.fabric8.kubernetes.api.model.NamedContext;
 import io.fabric8.kubernetes.api.model.NamespaceBuilder;
+import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.api.model.PodList;
 import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClientException;
+import io.fabric8.kubernetes.client.Watch;
+import io.fabric8.kubernetes.client.Watcher;
+import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable;
 import io.fabric8.kubernetes.client.internal.KubeConfigUtils;
 import io.fabric8.kubernetes.client.utils.Utils;
 
 public class KubernetesTestUtil {
 
+    private static final Logger LOGGER = Logger.getLogger(KubernetesTestUtil.class.getName());
+
     public static final String TESTING_NAMESPACE = "kubernetes-plugin-test";
     public static final String KUBERNETES_CONTEXT = System.getProperty("kubernetes.context", "minikube");
 
@@ -81,9 +95,73 @@ public static KubernetesCloud setupCloud() throws UnrecoverableKeyException, Cer
             // Run in our own testing namespace
             client.namespaces().createOrReplace(
                     new NamespaceBuilder().withNewMetadata().withName(TESTING_NAMESPACE).endMetadata().build());
-        } catch (KubernetesClientException e){
+        } catch (KubernetesClientException e) {
             assumeNoException("Kubernetes cluster is not accessible", e);
         }
         return cloud;
     }
+
+    /**
+     * Delete pods with matching labels
+     * 
+     * @param client
+     * @param labels
+     * @param wait
+     *            wait some time for pods to finish
+     * @return whether any pod was deleted
+     * @throws Exception
+     */
+    public static boolean deletePods(KubernetesClient client, Map<String, String> labels, boolean wait)
+            throws Exception {
+
+        if (client != null) {
+
+            // wait for 30 seconds for all pods to be terminated
+            if (wait) {
+                LOGGER.log(INFO, "Waiting for pods to terminate");
+                ForkJoinPool forkJoinPool = new ForkJoinPool(1);
+                try {
+                    forkJoinPool.submit(() -> IntStream.range(1, 1_000_000).anyMatch(i -> {
+                        try {
+                            FilterWatchListDeletable<Pod, PodList, Boolean, Watch, Watcher<Pod>> pods = client.pods()
+                                    .withLabels(labels);
+                            LOGGER.log(INFO, "Still waiting for pods to terminate: {0}", print(pods));
+                            boolean allTerminated = pods.list().getItems().isEmpty();
+                            if (allTerminated) {
+                                LOGGER.log(INFO, "All pods are terminated: {0}", print(pods));
+                            } else {
+                                LOGGER.log(INFO, "Still waiting for pods to terminate: {0}", print(pods));
+                                Thread.sleep(5000);
+                            }
+                            return allTerminated;
+                        } catch (InterruptedException e) {
+                            LOGGER.log(INFO, "Waiting for pods to terminate - interrupted");
+                            return true;
+                        }
+                    })).get(60, TimeUnit.SECONDS);
+                } catch (TimeoutException e) {
+                    LOGGER.log(INFO, "Waiting for pods to terminate - timed out");
+                    // job not done in interval
+                }
+            }
+
+            FilterWatchListDeletable<Pod, PodList, Boolean, Watch, Watcher<Pod>> pods = client.pods()
+                    .withLabels(labels);
+            if (!pods.list().getItems().isEmpty()) {
+                LOGGER.log(WARNING, "Deleting leftover pods: {0}", print(pods));
+                if (Boolean.TRUE.equals(pods.delete())) {
+                    return true;
+                }
+
+            }
+        }
+        return false;
+    }
+
+    private static List<String> print(FilterWatchListDeletable<Pod, PodList, Boolean, Watch, Watcher<Pod>> pods) {
+        return pods.list().getItems().stream()
+                .map(pod -> String.format("%s (%s)", pod.getMetadata().getName(), pod.getStatus().getPhase()))
+                .collect(Collectors.toList());
+    }
+
 }

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java

@@ -67,14 +67,12 @@ public class ContainerExecDecoratorTest {
     @BeforeClass
     public static void configureCloud() throws Exception {
         client = setupCloud().connect();
-        deletePods();
+        deletePods(client, labels, false);
     }
 
     @AfterClass
-    public static void deletePods() throws Exception {
-        if (client != null) {
-            client.pods().withLabel("class", labels.get("class")).delete();
-        }
+    public static void after() throws Exception {
+        deletePods(client, labels, false);
     }
 
     @Before

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java

@@ -43,6 +43,7 @@
 import org.jenkinsci.plugins.workflow.job.WorkflowJob;
 import org.jenkinsci.plugins.workflow.job.WorkflowRun;
 import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -104,6 +105,13 @@ public static void configureCloud() throws Exception {
     public void configureTemplates() throws Exception {
         cloud.getTemplates().clear();
         cloud.addTemplate(buildBusyboxTemplate("busybox"));
+        deletePods(cloud.connect(), Collections.emptyMap(), false);
+    }
+
+    @After
+    public void cleanup() throws Exception {
+        assertFalse("There are pods leftover after test execution, see previous logs",
+                deletePods(cloud.connect(), KubernetesCloud.DEFAULT_POD_LABELS, true));
     }
 
     /**
@@ -246,6 +254,9 @@ public void runWithOverriddenNamespace() throws Exception {
     }
 
     @Test
+    /**
+     * Step namespace should have priority over anything else.
+     */
     public void runWithOverriddenNamespace2() throws Exception {
         String overriddenNamespace = "kubernetes-plugin-overridden-namespace";
         KubernetesClient client = cloud.connect();

src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/runWithOverriddenNamespace2.groovy

@@ -1,3 +1,4 @@
+// Step namespace should have priority over anything else.
 podTemplate(cloud: 'kubernetes-plugin-test', namespace: 'testns2', label: 'mypod', volumes: [emptyDirVolume(mountPath: '/my-mount')], containers: [
         containerTemplate(name: 'jnlp', image: 'jenkinsci/jnlp-slave:2.62-alpine', args: '${computer.jnlpmac} ${computer.name}')
 ]) {