wait CorePodsList components to be Ready only if --wait=all

pkg/minikube/bootstrapper/bsutil/kverify/kverify.go

@@ -37,16 +37,18 @@ const (
 	NodeReadyKey = "node_ready"
 	// KubeletKey is the name used in the flags for waiting for the kubelet status to be ready
 	KubeletKey = "kubelet"
+	// OperationalKey is the name used for waiting for pods in CorePodsList to be Ready
+	OperationalKey = "operational"
 )
 
 //  vars related to the --wait flag
 var (
 	// DefaultComponents is map of the the default components to wait for
 	DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
 	// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
-	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false}
+	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false, OperationalKey: false}
 	// AllComponents is map for waiting for all components.
-	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, KubeletKey: true}
+	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, NodeReadyKey: true, KubeletKey: true, OperationalKey: true}
 	// DefaultWaitList is list of all default components to wait for. only names to be used for start flags.
 	DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
 	// AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags.
@@ -60,8 +62,8 @@ var (
 		"kube-proxy",
 		"kube-scheduler",
 	}
-	// SystemPodsList is a list of essential pods for running kurnetes to wait for them to be Ready
-	SystemPodsList = []string{
+	// CorePodsList is a list of essential pods for running kurnetes to wait for them to be operational ("Ready")
+	CorePodsList = []string{
 		"kube-dns", // coredns
 		"etcd",
 		"kube-apiserver",

pkg/minikube/bootstrapper/bsutil/kverify/pod_ready.go

@@ -31,6 +31,27 @@ import (
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
+// WaitOperational calls WaitForPodReadyByLabel for each pod in labels list and returns any errors occurred.
+func WaitOperational(cs *kubernetes.Clientset, labels []string, timeout time.Duration) error {
+	klog.Info("waiting for kube-system core pods %s to be Ready ...", labels)
+	pStart := time.Now()
+	defer func() {
+		klog.Infof("duration metric: took %s for waiting for kube-system core pods to be Ready ...", time.Since(pStart))
+	}()
+
+	var errs []string
+	for _, label := range labels {
+		if err := WaitForPodReadyByLabel(cs, label, "kube-system", timeout); err != nil {
+			errs = append(errs, fmt.Sprintf("%q: %q", label, err.Error()))
+		}
+	}
+	if errs != nil {
+		return fmt.Errorf(strings.Join(errs, ", "))
+	}
+
+	return nil
+}
+
 // WaitForPodReadyByLabel waits for pod with label ([key:]val) in a namespace to be in Ready condition.
 // If namespace is not provided, it defaults to "kube-system".
 // If label key is not provided, it will try with "component" and "k8s-app".

pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go

@@ -36,25 +36,40 @@ import (
 	"k8s.io/minikube/pkg/util/retry"
 )
 
-// WaitForSystemPods verifies essential pods for running kurnetes are Ready
+// WaitForSystemPods verifies essential pods for running kurnetes is running
 func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg config.ClusterConfig, cr command.Runner, client *kubernetes.Clientset, start time.Time, timeout time.Duration) error {
-	klog.Info("waiting for kube-system pods to be Ready ...")
+	klog.Info("waiting for kube-system pods to appear ...")
 	pStart := time.Now()
-	defer func() {
-		klog.Infof("duration metric: took %s for waiting for kube-system pods to be Ready ...", time.Since(pStart))
-	}()
 
-	if time.Since(start) > minLogCheckTime {
-		announceProblems(r, bs, cfg, cr)
-		time.Sleep(kconst.APICallRetryInterval * 5)
-	}
+	podList := func() error {
+		if time.Since(start) > minLogCheckTime {
+			announceProblems(r, bs, cfg, cr)
+			time.Sleep(kconst.APICallRetryInterval * 5)
+		}
 
-	for _, label := range SystemPodsList {
-		if err := WaitForPodReadyByLabel(client, label, "kube-system", timeout); err != nil {
+		// Wait for any system pod, as waiting for apiserver may block until etcd
+		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+		if err != nil {
+			klog.Warningf("pod list returned error: %v", err)
 			return err
 		}
+
+		klog.Infof("%d kube-system pods found", len(pods.Items))
+		for _, pod := range pods.Items {
+			klog.Infof(podStatusMsg(pod))
+		}
+
+		if len(pods.Items) < 2 {
+			return fmt.Errorf("only %d pod(s) have shown up", len(pods.Items))
+		}
+
+		return nil
 	}
 
+	if err := retry.Local(podList, timeout); err != nil {
+		return fmt.Errorf("apiserver never returned a pod list")
+	}
+	klog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
 	return nil
 }
 

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -477,8 +477,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 
 	if n.ControlPlane {
 		if cfg.VerifyComponents[kverify.APIServerWaitKey] {
-			if err := kverify.WaitForPodReadyByLabel(client, "component: kube-apiserver", "kube-system", timeout); err != nil {
-				return errors.Wrapf(err, "waiting for API server pod to be Ready")
+			if err := kverify.WaitForAPIServerProcess(cr, k, cfg, k.c, start, timeout); err != nil {
+				return errors.Wrap(err, "wait for apiserver proc")
+			}
+
+			if err := kverify.WaitForHealthyAPIServer(cr, k, cfg, k.c, client, start, hostname, port, timeout); err != nil {
+				return errors.Wrap(err, "wait for healthy API server")
 			}
 		}
 
@@ -499,6 +503,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 				return errors.Wrap(err, "waiting for apps_running")
 			}
 		}
+
+		if cfg.VerifyComponents[kverify.OperationalKey] {
+			if err := kverify.WaitOperational(client, kverify.CorePodsList, timeout); err != nil {
+				return errors.Wrap(err, "waiting for operational status")
+			}
+		}
 	}
 	if cfg.VerifyComponents[kverify.KubeletKey] {
 		if err := kverify.WaitForService(k.c, "kubelet", timeout); err != nil {