diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java index 0d2580c43a..e46ab0970c 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java @@ -35,6 +35,8 @@ import org.csanchez.jenkins.plugins.kubernetes.pipeline.PodTemplateStepExecution; import org.csanchez.jenkins.plugins.kubernetes.volumes.PodVolume; import org.jenkinsci.plugins.durabletask.executors.OnceRetentionStrategy; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; @@ -113,7 +115,7 @@ public class KubernetesCloud extends Cloud { .getProperty(PodTemplateStepExecution.class.getName() + ".defaultImage", "jenkinsci/jnlp-slave:alpine"); /** label for all pods started by the plugin */ - private static final Map POD_LABEL = ImmutableMap.of("jenkins", "slave"); + public static final Map DEFAULT_POD_LABELS = ImmutableMap.of("jenkins", "slave"); private static final String JNLPMAC_REF = "\\$\\{computer.jnlpmac\\}"; private static final String NAME_REF = "\\$\\{computer.name\\}"; @@ -324,8 +326,7 @@ public KubernetesClient connect() throws UnrecoverableKeyException, NoSuchAlgori new String[] { getDisplayName(), serverUrl }); client = new KubernetesFactoryAdapter(serverUrl, namespace, serverCertificate, credentialsId, skipTlsVerify, connectTimeout, readTimeout, maxRequestsPerHost).createClient(); - LOGGER.log(Level.FINE, "Connected to Kubernetes {0} URL {1}" + serverUrl, - new String[] { getDisplayName(), serverUrl }); + LOGGER.log(Level.FINE, "Connected to Kubernetes {0} URL {1}", new String[] { getDisplayName(), serverUrl }); return client; } @@ -497,7 +498,7 @@ private Pod getPodTemplate(KubernetesSlave slave, PodTemplate template) { private Map getLabelsMap(Set labelSet) { ImmutableMap.Builder builder = ImmutableMap. builder(); - builder.putAll(POD_LABEL); + builder.putAll(DEFAULT_POD_LABELS); if (!labelSet.isEmpty()) { for (LabelAtom label: labelSet) { builder.put(getIdForLabel(label), "true"); @@ -809,7 +810,7 @@ private boolean addProvisionedSlave(@Nonnull PodTemplate template, @CheckForNull templateNamespace = client.getNamespace(); } - PodList slaveList = client.pods().inNamespace(templateNamespace).withLabels(POD_LABEL).list(); + PodList slaveList = client.pods().inNamespace(templateNamespace).withLabels(DEFAULT_POD_LABELS).list(); List slaveListItems = slaveList.getItems(); Map labelsMap = getLabelsMap(template.getLabelSet()); diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java index b46edb6db2..ba7274a116 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesSlave.java @@ -166,8 +166,8 @@ protected void _terminate(TaskListener listener) throws IOException, Interrupted String actualNamespace = getNamespace() == null ? client.getNamespace() : getNamespace(); try { - Boolean delete = client.pods().inNamespace(actualNamespace).withName(name).delete(); - if (delete == null) { + Boolean deleted = client.pods().inNamespace(actualNamespace).withName(name).delete(); + if (!Boolean.TRUE.equals(deleted)) { String msg = String.format("Failed to delete pod for agent %s/%s: not found", actualNamespace, name); LOGGER.log(Level.WARNING, msg); listener.error(msg); diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/PodTemplateStepExecution.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/PodTemplateStepExecution.java index c974e0c755..8d2301f694 100755 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/PodTemplateStepExecution.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/PodTemplateStepExecution.java @@ -15,6 +15,7 @@ import hudson.AbortException; import hudson.model.Run; import hudson.slaves.Cloud; +import io.fabric8.kubernetes.client.KubernetesClient; import jenkins.model.Jenkins; public class PodTemplateStepExecution extends AbstractStepExecutionImpl { @@ -124,14 +125,25 @@ private PodTemplateCallback(PodTemplate podTemplate) { protected void finished(StepContext context) throws Exception { Cloud cloud = Jenkins.getInstance().getCloud(step.getCloud()); if (cloud == null) { - LOGGER.log(Level.FINE, "Cloud {0} no longer exists, cannot delete pod template {1}", + LOGGER.log(Level.WARNING, "Cloud {0} no longer exists, cannot delete pod template {1}", new Object[] { step.getCloud(), podTemplate.getName() }); return; } if (cloud instanceof KubernetesCloud) { + LOGGER.log(Level.INFO, "Removing pod template and deleting pod {1} from cloud {0}", + new Object[] { cloud.name, podTemplate.getName() }); KubernetesCloud kubernetesCloud = (KubernetesCloud) cloud; kubernetesCloud.removeTemplate(podTemplate); - kubernetesCloud.connect().pods().withName(podTemplate.getName()).delete(); + KubernetesClient client = kubernetesCloud.connect(); + Boolean deleted = client.pods().withName(podTemplate.getName()).delete(); + if (!Boolean.TRUE.equals(deleted)) { + LOGGER.log(Level.WARNING, "Failed to delete pod for agent {0}/{1}: not found", + new String[] { client.getNamespace(), podTemplate.getName() }); + return; + } + } else { + LOGGER.log(Level.WARNING, "Cloud is not a KubernetesCloud: {0} {1}", + new String[] { cloud.name, cloud.getClass().getName() }); } } } diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesTestUtil.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesTestUtil.java index aeed9b1459..7c458895fb 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesTestUtil.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesTestUtil.java @@ -24,6 +24,7 @@ package org.csanchez.jenkins.plugins.kubernetes; import static io.fabric8.kubernetes.client.Config.*; +import static java.util.logging.Level.*; import static org.hamcrest.Matchers.*; import static org.junit.Assume.*; @@ -33,22 +34,35 @@ import java.security.NoSuchAlgorithmException; import java.security.UnrecoverableKeyException; import java.security.cert.CertificateEncodingException; +import java.util.List; +import java.util.Map; import java.util.Optional; - -import io.fabric8.kubernetes.client.KubernetesClientException; -import org.junit.Assume; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import io.fabric8.kubernetes.api.model.Cluster; import io.fabric8.kubernetes.api.model.Config; import io.fabric8.kubernetes.api.model.NamedCluster; import io.fabric8.kubernetes.api.model.NamedContext; import io.fabric8.kubernetes.api.model.NamespaceBuilder; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodList; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.dsl.FilterWatchListDeletable; import io.fabric8.kubernetes.client.internal.KubeConfigUtils; import io.fabric8.kubernetes.client.utils.Utils; public class KubernetesTestUtil { + private static final Logger LOGGER = Logger.getLogger(KubernetesTestUtil.class.getName()); + public static final String TESTING_NAMESPACE = "kubernetes-plugin-test"; public static final String KUBERNETES_CONTEXT = System.getProperty("kubernetes.context", "minikube"); @@ -81,9 +95,73 @@ public static KubernetesCloud setupCloud() throws UnrecoverableKeyException, Cer // Run in our own testing namespace client.namespaces().createOrReplace( new NamespaceBuilder().withNewMetadata().withName(TESTING_NAMESPACE).endMetadata().build()); - } catch (KubernetesClientException e){ + } catch (KubernetesClientException e) { assumeNoException("Kubernetes cluster is not accessible", e); } return cloud; } + + /** + * Delete pods with matching labels + * + * @param client + * @param labels + * @param wait + * wait some time for pods to finish + * @return whether any pod was deleted + * @throws Exception + */ + public static boolean deletePods(KubernetesClient client, Map labels, boolean wait) + throws Exception { + + if (client != null) { + + // wait for 30 seconds for all pods to be terminated + if (wait) { + LOGGER.log(INFO, "Waiting for pods to terminate"); + ForkJoinPool forkJoinPool = new ForkJoinPool(1); + try { + forkJoinPool.submit(() -> IntStream.range(1, 1_000_000).anyMatch(i -> { + try { + FilterWatchListDeletable> pods = client.pods() + .withLabels(labels); + LOGGER.log(INFO, "Still waiting for pods to terminate: {0}", print(pods)); + boolean allTerminated = pods.list().getItems().isEmpty(); + if (allTerminated) { + LOGGER.log(INFO, "All pods are terminated: {0}", print(pods)); + } else { + LOGGER.log(INFO, "Still waiting for pods to terminate: {0}", print(pods)); + Thread.sleep(5000); + } + return allTerminated; + } catch (InterruptedException e) { + LOGGER.log(INFO, "Waiting for pods to terminate - interrupted"); + return true; + } + })).get(60, TimeUnit.SECONDS); + } catch (TimeoutException e) { + LOGGER.log(INFO, "Waiting for pods to terminate - timed out"); + // job not done in interval + } + } + + FilterWatchListDeletable> pods = client.pods() + .withLabels(labels); + if (!pods.list().getItems().isEmpty()) { + LOGGER.log(WARNING, "Deleting leftover pods: {0}", print(pods)); + if (Boolean.TRUE.equals(pods.delete())) { + return true; + } + + } + } + return false; + } + + private static List print(FilterWatchListDeletable> pods) { + return pods.list().getItems().stream() + .map(pod -> String.format("%s (%s)", pod.getMetadata().getName(), pod.getStatus().getPhase())) + .collect(Collectors.toList()); + } + } diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java index a320c7e196..0fd1f5fbf7 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/ContainerExecDecoratorTest.java @@ -67,14 +67,12 @@ public class ContainerExecDecoratorTest { @BeforeClass public static void configureCloud() throws Exception { client = setupCloud().connect(); - deletePods(); + deletePods(client, labels, false); } @AfterClass - public static void deletePods() throws Exception { - if (client != null) { - client.pods().withLabel("class", labels.get("class")).delete(); - } + public static void after() throws Exception { + deletePods(client, labels, false); } @Before diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java index 8e62cbbba6..c3f18a59dd 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java @@ -43,6 +43,7 @@ import org.jenkinsci.plugins.workflow.job.WorkflowJob; import org.jenkinsci.plugins.workflow.job.WorkflowRun; import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -104,6 +105,13 @@ public static void configureCloud() throws Exception { public void configureTemplates() throws Exception { cloud.getTemplates().clear(); cloud.addTemplate(buildBusyboxTemplate("busybox")); + deletePods(cloud.connect(), Collections.emptyMap(), false); + } + + @After + public void cleanup() throws Exception { + assertFalse("There are pods leftover after test execution, see previous logs", + deletePods(cloud.connect(), KubernetesCloud.DEFAULT_POD_LABELS, true)); } /**