Check that kube-dns is healthy before running tests (#1260)

script/e2e-test.sh

@@ -89,19 +89,42 @@ for dep in ${deployments[@]}; do
   echo "Deployment ${dep} ready"
 done
 
+# Wait for DNS to be ready
+k8s_wait_for_deployment kube-system coredns
+
 # Wait for Kubeapps Jobs
 k8s_wait_for_job_completed kubeapps apprepositories.kubeapps.com/repo-name=stable
 echo "Job apprepositories.kubeapps.com/repo-name=stable ready"
 
 echo "All deployments ready. PODs:"
 kubectl get pods -n kubeapps -o wide
 
+# Wait for all the endpoints to be ready
+kubectl get ep --namespace=kubeapps
+svcs=(
+  kubeapps-ci
+  kubeapps-ci-internal-chartsvc
+  kubeapps-ci-internal-tiller-proxy
+  kubeapps-ci-internal-dashboard
+)
+for svc in ${svcs[@]}; do
+  k8s_wait_for_endpoint kubeapps ${svc} 2
+  echo "Endpoints for ${svc} available"
+done
+
 # Run helm tests
 set +e
 
-helm test ${HELM_CLIENT_TLS_FLAGS} kubeapps-ci
+helm test ${HELM_CLIENT_TLS_FLAGS} kubeapps-ci --cleanup
 code=$?
 
+if [[ "$code" != 0 ]]; then
+  echo "Helm test failed, retrying..."
+  # Avoid temporary issues, retry
+  helm test ${HELM_CLIENT_TLS_FLAGS} kubeapps-ci
+  code=$?
+fi
+
 set -e
 
 if [[ "$code" != 0 ]]; then

script/libtest.sh

@@ -30,6 +30,37 @@ k8s_wait_for_deployment() {
     return $res
 }
 
+## k8s specific Helper functions
+k8s_wait_for_endpoint() {
+    namespace=${1:?}
+    svc=${2:?}
+    number_of_endpoints=${3:?}
+
+    local -i retryTimeSeconds=${TEST_MAX_WAIT_SEC:?}
+    local -i retryTimeStepSeconds=5
+
+    echo "Waiting for the endpoints of ${svc} to be at least ${number_of_endpoints}"
+
+    while [ "$retryTimeSeconds" -gt 0 ]; do
+        # Avoid to exit the function if the job is not completed yet
+        set +e
+        kubectl get ep -n kubeapps kubeapps-ci -o jsonpath="{.subsets[0].addresses[$(expr $number_of_endpoints - 1)]}" > /dev/null
+        res=$?
+        set -e
+        # There is a job that finished
+        if [[ "$res" -eq "0" ]]; then
+            echo "Endpoint ready"
+            return 0
+        fi
+        # It did not finished so we reduce the remaining time and wait for next retry cycle
+        echo "Waiting for endpoing ${svc} to be completed, will retry in $retryTimeStepSeconds seconds ... "
+        retryTimeSeconds=retryTimeSeconds-$retryTimeStepSeconds
+        sleep $retryTimeStepSeconds
+    done
+
+    return $res
+}
+
 k8s_ensure_image() {
     namespace=${1:?}
     deployment=${2:?}