wip: add new wait components "node_ready" and "no_pressure"

cmd/performance/mkcmp/cmd/mkcmp.go

@@ -35,7 +35,11 @@ var rootCmd = &cobra.Command{
 		return validateArgs(args)
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return perf.CompareMinikubeStart(context.Background(), os.Stdout, args)
+		binaries, err := retrieveBinaries(args)
+		if err != nil {
+			return err
+		}
+		return perf.CompareMinikubeStart(context.Background(), os.Stdout, binaries)
 	},
 }
 
@@ -46,6 +50,18 @@ func validateArgs(args []string) error {
 	return nil
 }
 
+func retrieveBinaries(args []string) ([]*perf.Binary, error) {
+	binaries := []*perf.Binary{}
+	for _, a := range args {
+		binary, err := perf.NewBinary(a)
+		if err != nil {
+			return nil, err
+		}
+		binaries = append(binaries, binary)
+	}
+	return binaries, nil
+}
+
 // Execute runs the mkcmp command
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {

hack/jenkins/windows_integration_test_docker.ps1

@@ -0,0 +1,34 @@
+# Copyright 2019 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p out
+gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
+gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
+
+./out/minikube-windows-amd64.exe delete --all
+
+out/e2e-windows-amd64.exe -minikube-start-args="--driver=docker" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
+$env:result=$lastexitcode
+# If the last exit code was 0->success, x>0->error
+If($env:result -eq 0){$env:status="success"}
+Else {$env:status="failure"}
+
+# $env:SHORT_COMMIT=$env:COMMIT.substring(0, 7)
+# to be used later to implement https://github.com/kubernetes/minikube/issues/6593
+$env:target_url="https://storage.googleapis.com/minikube-builds/logs/$env:MINIKUBE_LOCATION/Docker_Windows.txt"
+$json = "{`"state`": `"$env:status`", `"description`": `"Jenkins`", `"target_url`": `"$env:target_url`", `"context`": `"Docker_Windows`"}"
+Invoke-WebRequest -Uri "https://api.github.com/repos/kubernetes/minikube/statuses/$env:COMMIT`?access_token=$env:access_token" -Body $json -ContentType "application/json" -Method Post -usebasicparsing
+
+Exit $env:result

hack/jenkins/windows_integration_test_hyperv.ps1

@@ -17,9 +17,9 @@ gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-am
 gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
 gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
 
-./out/minikube-windows-amd64.exe delete
+./out/minikube-windows-amd64.exe delete --all
 
-out/e2e-windows-amd64.exe -minikube-start-args="--driver=hyperv --hyperv-virtual-switch=primary-virtual-switch" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
+out/e2e-windows-amd64.exe -minikube-start-args="--driver=hyperv" -binary=out/minikube-windows-amd64.exe -test.v -test.timeout=65m
 $env:result=$lastexitcode
 # If the last exit code was 0->success, x>0->error
 If($env:result -eq 0){$env:status="success"}

pkg/addons/addons.go

@@ -332,7 +332,7 @@ func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]boo
 
 	var awg sync.WaitGroup
 
-	out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")})
+	defer out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")})
 	for _, a := range toEnableList {
 		awg.Add(1)
 		go func(name string) {

pkg/minikube/bootstrapper/bsutil/kverify/default_sa.go

@@ -18,36 +18,37 @@ limitations under the License.
 package kverify
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/minikube/pkg/util/retry"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
 // WaitForDefaultSA waits for the default service account to be created.
 func WaitForDefaultSA(cs *kubernetes.Clientset, timeout time.Duration) error {
 	glog.Info("waiting for default service account to be created ...")
 	start := time.Now()
-	saReady := func() error {
+	saReady := func() (bool, error) {
 		// equivalent to manual check of 'kubectl --context profile get serviceaccount default'
 		sas, err := cs.CoreV1().ServiceAccounts("default").List(meta.ListOptions{})
 		if err != nil {
 			glog.Infof("temproary error waiting for default SA: %v", err)
-			return err
+			return false, nil
 		}
 		for _, sa := range sas.Items {
 			if sa.Name == "default" {
 				glog.Infof("found service account: %q", sa.Name)
-				return nil
+				return true, nil
 			}
 		}
-		return fmt.Errorf("couldn't find default service account")
+		return false, nil
 	}
-	if err := retry.Expo(saReady, 500*time.Millisecond, timeout); err != nil {
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, timeout, saReady); err != nil {
 		return errors.Wrapf(err, "waited %s for SA", time.Since(start))
 	}
 

pkg/minikube/bootstrapper/bsutil/kverify/kverify.go

@@ -31,22 +31,27 @@ const (
 	SystemPodsWaitKey = "system_pods"
 	// DefaultSAWaitKey is the name used in the flags for default service account
 	DefaultSAWaitKey = "default_sa"
-	// AppsRunning is the name used in the flags for waiting for k8s-apps to be running
-	AppsRunning = "apps_running"
+	// AppsRunningKey is the name used in the flags for waiting for k8s-apps to be running
+	AppsRunningKey = "apps_running"
+	// NodePressureKey is the name used in the flags detecting node coditions such as
+	// disk, memory and PID pressure or network not ready.
+	NodePressureKey = "no_pressure"
+	// NodeReadyKey is the name used in the flags for waiting for the node status to be ready
+	NodeReadyKey = "node_ready"
 )
 
 //  vars related to the --wait flag
 var (
 	// DefaultComponents is map of the the default components to wait for
 	DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true}
 	// NoWaitComponents is map of componets to wait for if specified 'none' or 'false'
-	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunning: false}
+	NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodePressureKey: false, NodeReadyKey: false}
 	// AllComponents is map for waiting for all components.
-	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunning: true}
+	AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, NodePressureKey: true, NodeReadyKey: true}
 	// DefaultWaitList is list of all default components to wait for. only names to be used for start flags.
 	DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey}
 	// AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags.
-	AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunning}
+	AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunningKey, NodePressureKey, NodeReadyKey}
 	// AppsRunningList running list are valid k8s-app components to wait for them to be running
 	AppsRunningList = []string{
 		"kube-dns", // coredns

pkg/minikube/bootstrapper/bsutil/kverify/node_health.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running kubernetes cluster is healthy
+package kverify
+
+import (
+	"fmt"
+	"runtime"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	v1 "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// NodeConditions verfies that node is not under disk, memory, pid or network pressure.
+func NodeConditions(cs *kubernetes.Clientset, drver string) error {
+	glog.Info("verifying NodePressure condition ...")
+	start := time.Now()
+	defer func() {
+		glog.Infof("duration metric: took %s to wait for NodePressure...", time.Since(start))
+	}()
+
+	ns, err := cs.CoreV1().Nodes().List(meta.ListOptions{})
+	if err != nil {
+		return errors.Wrap(err, "list nodes")
+	}
+
+	for _, n := range ns.Items {
+		glog.Infof("node storage ephemeral capacity is %s", n.Status.Capacity.StorageEphemeral())
+		glog.Infof("node cpu capacity is %s", n.Status.Capacity.Cpu().AsDec())
+		for _, c := range n.Status.Conditions {
+			if c.Type == v1.NodeDiskPressure && c.Status != v1.ConditionTrue {
+				out.Ln("")
+				out.ErrT(out.FailureType, "node {{.name}} has unwanted condition {{.condition_type}} : Reason {{.reason}} Message: {{.message}}", out.V{"name": n.Name, "condition_type": c.Type, "reason": c.Reason, "message": c.Message})
+				out.WarningT("The node on {{.name}} has ran out of disk space. please consider allocating more disk using or pruning un-used images", out.V{"name": n.Name})
+				if driver.IsKIC(drver) && runtime.GOOS != "linux" {
+					out.T(out.Stopped, "Please increase Docker Desktop's disk image size.")
+					if runtime.GOOS == "darwin" {
+						out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"})
+					}
+					if runtime.GOOS == "windows" {
+						out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-windows/"})
+					}
+				} else { // VM-drivers
+					out.T(out.Stopped, "You can specify a larger disk for your cluster using `minikube start --disk` ")
+				}
+				out.Ln("") // if there is error message, lets make an empty space for better visilibtly
+				return fmt.Errorf("node %q has unwanted condition %q : Reason %q Message: %q ", n.Name, c.Type, c.Reason, c.Message)
+			}
+
+			if c.Type == v1.NodeMemoryPressure && c.Status == v1.ConditionTrue {
+				out.Ln("")
+				out.ErrT(out.FailureType, "node {{.name}} has unwanted condition {{.condition_type}} : Reason {{.reason}} Message: {{.message}}", out.V{"name": n.Name, "condition_type": c.Type, "reason": c.Reason, "message": c.Message})
+				out.WarningT("The node on {{.name}} has ran of memory.", out.V{"name": n.Name})
+				if driver.IsKIC(drver) && runtime.GOOS != "linux" {
+					out.T(out.Stopped, "Please increase Docker Desktop's memory.")
+					if runtime.GOOS == "darwin" {
+						out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"})
+					}
+					if runtime.GOOS == "windows" {
+						out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-windows/"})
+					}
+				} else {
+					out.T(out.Stopped, "You can specify a larger memory size for your cluster using `minikube start --memory` ")
+				}
+				out.Ln("") // if there is error message, lets make an empty space for better visilibtly
+				return fmt.Errorf("node %q has unwanted condition %q : Reason %q Message: %q ", n.Name, c.Type, c.Reason, c.Message)
+			}
+
+			if c.Type == v1.NodePIDPressure && c.Status == v1.ConditionTrue {
+				out.Ln("")
+				out.ErrT(out.FailureType, "node {{.name}} has unwanted condition {{.condition_type}} : Reason {{.reason}} Message: {{.message}}", out.V{"name": n.Name, "condition_type": c.Type, "reason": c.Reason, "message": c.Message})
+				out.WarningT("The node has ran out of available PIDs.", out.V{"name": n.Name})
+				out.Ln("")
+				return fmt.Errorf("node %q has unwanted condition %q : Reason %q Message: %q ", n.Name, c.Type, c.Reason, c.Message)
+			}
+
+			if c.Type == v1.NodeNetworkUnavailable && c.Status == v1.ConditionTrue {
+				out.Ln("")
+				out.ErrT(out.FailureType, "node {{.name}} has unwanted condition {{.condition_type}} : Reason {{.reason}} Message: {{.message}}", out.V{"name": n.Name, "condition_type": c.Type, "reason": c.Reason, "message": c.Message})
+				out.WarningT("The node networking is not configured correctly.", out.V{"name": n.Name})
+				out.Ln("")
+				return fmt.Errorf("node %q has unwanted condition %q : Reason %q Message: %q ", n.Name, c.Type, c.Reason, c.Message)
+			}
+		}
+	}
+
+	return nil
+}
+
+// WaitForNodeReady waits for a node to be ready
+func WaitForNodeReady(cs *kubernetes.Clientset, timeout time.Duration) error {
+	glog.Info("waiting for node to be ready ...")
+	start := time.Now()
+	defer func() {
+		glog.Infof("duration metric: took %s to wait for WaitForNodeReady...", time.Since(start))
+	}()
+	checkReady := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("wait for node to be ready timed out")
+		}
+		ns, err := cs.CoreV1().Nodes().List(meta.ListOptions{})
+		if err != nil {
+			glog.Infof("error listing nodes will retry: %v", err)
+			return false, nil
+		}
+
+		for _, n := range ns.Items {
+			for _, c := range n.Status.Conditions {
+				if c.Type == v1.NodeReady && c.Status != v1.ConditionTrue {
+					glog.Infof("node %q has unwanted condition %q : Reason %q Message: %q. will try. ", n.Name, c.Type, c.Reason, c.Message)
+					return false, nil
+				}
+			}
+		}
+		return true, nil
+	}
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil {
+		return errors.Wrapf(err, "wait node ready")
+	}
+
+	return nil
+}