Merge pull request #6452 from tstromberg/none-paused

Fix none driver bugs with "pause"

cmd/minikube/cmd/delete.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/cluster"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
@@ -192,7 +193,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	}
 
 	if err == nil && driver.BareMetal(cc.VMDriver) {
-		if err := uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
+		if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
 			deletionError, ok := err.(DeletionError)
 			if ok {
 				delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err))
@@ -273,12 +274,34 @@ func profileDeletionErr(profileName string, additionalInfo string) error {
 	return fmt.Errorf("error deleting profile \"%s\": %s", profileName, additionalInfo)
 }
 
-func uninstallKubernetes(api libmachine.API, kc pkg_config.KubernetesConfig, bsName string) error {
+func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string) error {
 	out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": kc.KubernetesVersion, "bootstrapper_name": bsName})
 	clusterBootstrapper, err := getClusterBootstrapper(api, bsName)
 	if err != nil {
 		return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal}
-	} else if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
+	}
+
+	host, err := cluster.CheckIfHostExistsAndLoad(api, profile)
+	if err != nil {
+		exit.WithError("Error getting host", err)
+	}
+	r, err := machine.CommandRunner(host)
+	if err != nil {
+		exit.WithError("Failed to get command runner", err)
+	}
+
+	cr, err := cruntime.New(cruntime.Config{Type: kc.ContainerRuntime, Runner: r})
+	if err != nil {
+		exit.WithError("Failed runtime", err)
+	}
+
+	// Unpause the cluster if necessary to avoid hung kubeadm
+	_, err = cluster.Unpause(cr, r, nil)
+	if err != nil {
+		glog.Errorf("unpause failed: %v", err)
+	}
+
+	if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
 		return DeletionError{Err: fmt.Errorf("failed to delete cluster: %v", err), Errtype: Fatal}
 	}
 	return nil

cmd/minikube/cmd/status.go

@@ -30,7 +30,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -138,30 +138,27 @@ func status(api libmachine.API, name string) (*Status, error) {
 	}
 
 	hs, err := cluster.GetHostStatus(api, name)
+	glog.Infof("%s host status = %q (err=%v)", name, hs, err)
 	if err != nil {
 		return st, errors.Wrap(err, "host")
 	}
 
-	// Nonexistent it is!
+	// We have no record of this host. Return nonexistent struct
 	if hs == state.None.String() {
 		return st, nil
 	}
 	st.Host = hs
+
+	// If it's not running, quickly bail out rather than delivering conflicting messages
 	if st.Host != state.Running.String() {
+		glog.Infof("host is not running, skipping remaining checks")
+		st.APIServer = st.Host
+		st.Kubelet = st.Host
+		st.Kubeconfig = st.Host
 		return st, nil
 	}
 
-	bs, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
-	if err != nil {
-		return st, errors.Wrap(err, "bootstrapper")
-	}
-
-	st.Kubelet, err = bs.GetKubeletStatus()
-	if err != nil {
-		glog.Warningf("kubelet err: %v", err)
-		st.Kubelet = state.Error.String()
-	}
-
+	// We have a fully operational host, now we can check for details
 	ip, err := cluster.GetHostDriverIP(api, name)
 	if err != nil {
 		glog.Errorln("Error host driver ip status:", err)
@@ -175,20 +172,43 @@ func status(api libmachine.API, name string) (*Status, error) {
 		port = constants.APIServerPort
 	}
 
-	st.APIServer, err = bs.GetAPIServerStatus(ip, port)
+	st.Kubeconfig = Misconfigured
+	ok, err := kubeconfig.IsClusterInConfig(ip, name)
+	glog.Infof("%s is in kubeconfig at ip %s: %v (err=%v)", name, ip, ok, err)
+	if ok {
+		st.Kubeconfig = Configured
+	}
+
+	host, err := cluster.CheckIfHostExistsAndLoad(api, name)
 	if err != nil {
-		glog.Errorln("Error apiserver status:", err)
-		st.APIServer = state.Error.String()
+		return st, err
 	}
 
-	st.Kubeconfig = Misconfigured
-	ks, err := kubeconfig.IsClusterInConfig(ip, name)
+	cr, err := machine.CommandRunner(host)
 	if err != nil {
-		glog.Errorln("Error kubeconfig status:", err)
+		return st, err
 	}
-	if ks {
-		st.Kubeconfig = Configured
+
+	stk, err := kverify.KubeletStatus(cr)
+	glog.Infof("%s kubelet status = %s (err=%v)", name, stk, err)
+
+	if err != nil {
+		glog.Warningf("kubelet err: %v", err)
+		st.Kubelet = state.Error.String()
+	} else {
+		st.Kubelet = stk.String()
 	}
+
+	sta, err := kverify.APIServerStatus(cr, ip, port)
+	glog.Infof("%s apiserver status = %s (err=%v)", name, stk, err)
+
+	if err != nil {
+		glog.Errorln("Error apiserver status:", err)
+		st.APIServer = state.Error.String()
+	} else {
+		st.APIServer = sta.String()
+	}
+
 	return st, nil
 }
 

cmd/minikube/cmd/stop.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/docker/machine/libmachine/mcnerror"
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -54,6 +55,11 @@ func runStop(cmd *cobra.Command, args []string) {
 	nonexistent := false
 	stop := func() (err error) {
 		err = cluster.StopHost(api)
+		if err == nil {
+			return nil
+		}
+		glog.Warningf("stop host returned error: %v", err)
+
 		switch err := errors.Cause(err).(type) {
 		case mcnerror.ErrHostDoesNotExist:
 			out.T(out.Meh, `"{{.profile_name}}" VM does not exist, nothing to stop`, out.V{"profile_name": profile})

pkg/drivers/none/none.go

@@ -18,6 +18,7 @@ package none
 
 import (
 	"fmt"
+	"net"
 	"os/exec"
 	"strings"
 	"time"
@@ -26,10 +27,13 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"k8s.io/apimachinery/pkg/util/net"
+	knet "k8s.io/apimachinery/pkg/util/net"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/kubeconfig"
 	"k8s.io/minikube/pkg/minikube/vmpath"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -94,7 +98,7 @@ func (d *Driver) DriverName() string {
 
 // GetIP returns an IP or hostname that this host is available at
 func (d *Driver) GetIP() (string, error) {
-	ip, err := net.ChooseHostInterface()
+	ip, err := knet.ChooseHostInterface()
 	if err != nil {
 		return "", err
 	}
@@ -123,11 +127,30 @@ func (d *Driver) GetURL() (string, error) {
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
-	if err := checkKubelet(d.exec); err != nil {
-		glog.Infof("kubelet not running: %v", err)
-		return state.Stopped, nil
+	glog.Infof("GetState called")
+	ip, err := d.GetIP()
+	if err != nil {
+		return state.Error, err
+	}
+
+	port, err := kubeconfig.Port(d.BaseDriver.MachineName)
+	if err != nil {
+		glog.Warningf("unable to get port: %v", err)
+		port = constants.APIServerPort
 	}
-	return state.Running, nil
+
+	// Confusing logic, as libmachine.Stop will loop until the state == Stopped
+	ast, err := kverify.APIServerStatus(d.exec, net.ParseIP(ip), port)
+	if err != nil {
+		return ast, err
+	}
+
+	// If the apiserver is up, we'll claim to be up.
+	if ast == state.Paused || ast == state.Running {
+		return state.Running, nil
+	}
+
+	return kverify.KubeletStatus(d.exec)
 }
 
 // Kill stops a host forcefully, including any containers that we are managing.
@@ -197,17 +220,18 @@ func (d *Driver) Start() error {
 // Stop a host gracefully, including any containers that we are managing.
 func (d *Driver) Stop() error {
 	if err := stopKubelet(d.exec); err != nil {
-		return err
+		return errors.Wrap(err, "stop kubelet")
 	}
 	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "containers")
 	}
 	if len(containers) > 0 {
 		if err := d.runtime.StopContainers(containers); err != nil {
-			return errors.Wrap(err, "stop")
+			return errors.Wrap(err, "stop containers")
 		}
 	}
+	glog.Infof("none driver is stopped!")
 	return nil
 }
 
@@ -251,13 +275,3 @@ func restartKubelet(cr command.Runner) error {
 	}
 	return nil
 }
-
-// checkKubelet returns an error if the kubelet is not running.
-func checkKubelet(cr command.Runner) error {
-	glog.Infof("checking for running kubelet ...")
-	c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet")
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "check kubelet")
-	}
-	return nil
-}