diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index d095f9c768ac..0f315594b30e 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -29,10 +29,14 @@ import ( "strings" "time" + apiWait "k8s.io/apimachinery/pkg/util/wait" + "github.com/spf13/cobra" "k8s.io/klog/v2" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/driver" @@ -45,6 +49,8 @@ import ( "k8s.io/minikube/pkg/minikube/sysinit" ) +const minLogCheckTime = 60 * time.Second + var dockerEnvTCPTmpl = fmt.Sprintf( "{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}"+ "{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}"+ @@ -193,9 +199,37 @@ func mustRestartDockerd(name string, runner command.Runner) { // if we get to the point that we have to restart docker (instead of reload) // will need to wait for apisever container to come up, this usually takes 5 seconds // verifying apisever using kverify would add code complexity for a rare case. - klog.Warningf("waiting 5 seconds to ensure apisever container is up...") - time.Sleep(time.Second * 5) + klog.Warningf("waiting to ensure apisever container is up...") + startTime := time.Now() + if err = waitForAPIServerProcess(runner, startTime, time.Second*30); err != nil { + klog.Warningf("apiserver container isn't up, error: %v", err) + } + } +} + +func waitForAPIServerProcess(cr command.Runner, start time.Time, timeout time.Duration) error { + klog.Infof("waiting for apiserver process to appear ...") + err := apiWait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("cluster wait timed out during process check") + } + + if time.Since(start) > minLogCheckTime { + klog.Infof("waiting for apiserver process to appear ...") + time.Sleep(kconst.APICallRetryInterval * 5) + } + + if _, ierr := kverify.APIServerPID(cr); ierr != nil { + return false, nil + } + + return true, nil + }) + if err != nil { + return fmt.Errorf("apiserver process never appeared") } + klog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start)) + return nil } // dockerEnvCmd represents the docker-env command diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go index 9ca936309763..fe07aba87b8c 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go @@ -56,7 +56,7 @@ func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, c time.Sleep(kconst.APICallRetryInterval * 5) } - if _, ierr := apiServerPID(cr); ierr != nil { + if _, ierr := APIServerPID(cr); ierr != nil { return false, nil } @@ -69,8 +69,8 @@ func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, c return nil } -// apiServerPID returns our best guess to the apiserver pid -func apiServerPID(cr command.Runner) (int, error) { +// APIServerPID returns our best guess to the apiserver pid +func APIServerPID(cr command.Runner) (int, error) { rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "-xnf", "kube-apiserver.*minikube.*")) if err != nil { return 0, err @@ -145,7 +145,7 @@ func APIServerVersionMatch(client *kubernetes.Clientset, expected string) error func APIServerStatus(cr command.Runner, hostname string, port int) (state.State, error) { klog.Infof("Checking apiserver status ...") - pid, err := apiServerPID(cr) + pid, err := APIServerPID(cr) if err != nil { klog.Warningf("stopped: unable to get apiserver pid: %v", err) return state.Stopped, nil