From a31b851b315471ab44c61b6ada534a0b0775de64 Mon Sep 17 00:00:00 2001 From: Yanshu Zhao Date: Wed, 16 Dec 2020 07:52:06 +0000 Subject: [PATCH 1/5] docker-env cmd wait for api server to be up --- cmd/minikube/cmd/docker-env.go | 37 ++++++++++++++++++- .../bootstrapper/bsutil/kverify/api_server.go | 8 ++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index d095f9c768ac..80e68ea8fee6 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -29,10 +29,14 @@ import ( "strings" "time" + apiWait "k8s.io/apimachinery/pkg/util/wait" + "github.com/spf13/cobra" "k8s.io/klog/v2" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/driver" @@ -45,6 +49,8 @@ import ( "k8s.io/minikube/pkg/minikube/sysinit" ) +const minLogCheckTime = 60 * time.Second + var dockerEnvTCPTmpl = fmt.Sprintf( "{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}"+ "{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}"+ @@ -193,9 +199,36 @@ func mustRestartDockerd(name string, runner command.Runner) { // if we get to the point that we have to restart docker (instead of reload) // will need to wait for apisever container to come up, this usually takes 5 seconds // verifying apisever using kverify would add code complexity for a rare case. - klog.Warningf("waiting 5 seconds to ensure apisever container is up...") - time.Sleep(time.Second * 5) + klog.Warningf("waiting to ensure apisever container is up...") + startTime := time.Now() + //time.Sleep(time.Second * 5) + WaitForAPIServerProcess(runner, startTime, time.Second*5) + } +} + +func WaitForAPIServerProcess(cr command.Runner, start time.Time, timeout time.Duration) error { + klog.Infof("waiting for apiserver process to appear ...") + err := apiWait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("cluster wait timed out during process check") + } + + if time.Since(start) > minLogCheckTime { + klog.Infof("waiting for apiserver process to appear ...") + time.Sleep(kconst.APICallRetryInterval * 5) + } + + if _, ierr := kverify.ApiServerPID(cr); ierr != nil { + return false, nil + } + + return true, nil + }) + if err != nil { + return fmt.Errorf("apiserver process never appeared") } + klog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start)) + return nil } // dockerEnvCmd represents the docker-env command diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go index 9ca936309763..14f37ef2858a 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go @@ -56,7 +56,7 @@ func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, c time.Sleep(kconst.APICallRetryInterval * 5) } - if _, ierr := apiServerPID(cr); ierr != nil { + if _, ierr := ApiServerPID(cr); ierr != nil { return false, nil } @@ -69,8 +69,8 @@ func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, c return nil } -// apiServerPID returns our best guess to the apiserver pid -func apiServerPID(cr command.Runner) (int, error) { +// ApiServerPID returns our best guess to the apiserver pid +func ApiServerPID(cr command.Runner) (int, error) { rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "-xnf", "kube-apiserver.*minikube.*")) if err != nil { return 0, err @@ -145,7 +145,7 @@ func APIServerVersionMatch(client *kubernetes.Clientset, expected string) error func APIServerStatus(cr command.Runner, hostname string, port int) (state.State, error) { klog.Infof("Checking apiserver status ...") - pid, err := apiServerPID(cr) + pid, err := ApiServerPID(cr) if err != nil { klog.Warningf("stopped: unable to get apiserver pid: %v", err) return state.Stopped, nil From bb83ce154ecf957b035a9d9828afcd520d07d4bf Mon Sep 17 00:00:00 2001 From: Yanshu Zhao Date: Wed, 16 Dec 2020 17:56:00 +0000 Subject: [PATCH 2/5] Fix lint --- cmd/minikube/cmd/docker-env.go | 2 +- pkg/minikube/bootstrapper/bsutil/kverify/api_server.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 80e68ea8fee6..b7a6c33a5fb3 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -218,7 +218,7 @@ func WaitForAPIServerProcess(cr command.Runner, start time.Time, timeout time.Du time.Sleep(kconst.APICallRetryInterval * 5) } - if _, ierr := kverify.ApiServerPID(cr); ierr != nil { + if _, ierr := kverify.APIServerPID(cr); ierr != nil { return false, nil } diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go index 14f37ef2858a..fe07aba87b8c 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/api_server.go @@ -56,7 +56,7 @@ func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, c time.Sleep(kconst.APICallRetryInterval * 5) } - if _, ierr := ApiServerPID(cr); ierr != nil { + if _, ierr := APIServerPID(cr); ierr != nil { return false, nil } @@ -69,8 +69,8 @@ func WaitForAPIServerProcess(r cruntime.Manager, bs bootstrapper.Bootstrapper, c return nil } -// ApiServerPID returns our best guess to the apiserver pid -func ApiServerPID(cr command.Runner) (int, error) { +// APIServerPID returns our best guess to the apiserver pid +func APIServerPID(cr command.Runner) (int, error) { rr, err := cr.RunCmd(exec.Command("sudo", "pgrep", "-xnf", "kube-apiserver.*minikube.*")) if err != nil { return 0, err @@ -145,7 +145,7 @@ func APIServerVersionMatch(client *kubernetes.Clientset, expected string) error func APIServerStatus(cr command.Runner, hostname string, port int) (state.State, error) { klog.Infof("Checking apiserver status ...") - pid, err := ApiServerPID(cr) + pid, err := APIServerPID(cr) if err != nil { klog.Warningf("stopped: unable to get apiserver pid: %v", err) return state.Stopped, nil From 96104205aff5818552f794d7cfd89e8883cd5217 Mon Sep 17 00:00:00 2001 From: Yanshu Zhao Date: Thu, 17 Dec 2020 17:50:51 +0000 Subject: [PATCH 3/5] fix lint --- cmd/minikube/cmd/docker-env.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index b7a6c33a5fb3..ce6e347214d3 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -201,8 +201,10 @@ func mustRestartDockerd(name string, runner command.Runner) { // verifying apisever using kverify would add code complexity for a rare case. klog.Warningf("waiting to ensure apisever container is up...") startTime := time.Now() - //time.Sleep(time.Second * 5) - WaitForAPIServerProcess(runner, startTime, time.Second*5) + + if err = WaitForAPIServerProcess(runner, startTime, time.Second*5); err != nil { + exit.Message(reason.RuntimeRestart, `The api server within '{{.name}}' is not up`, out.V{"name": name}) + } } } From f698ccb52dfc6646910e1d495215c8a0d9ef39b3 Mon Sep 17 00:00:00 2001 From: Yanshu Zhao Date: Mon, 21 Dec 2020 20:00:43 +0000 Subject: [PATCH 4/5] Increase time out to 30 s --- cmd/minikube/cmd/docker-env.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index ce6e347214d3..fde8f224cc65 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -201,8 +201,7 @@ func mustRestartDockerd(name string, runner command.Runner) { // verifying apisever using kverify would add code complexity for a rare case. klog.Warningf("waiting to ensure apisever container is up...") startTime := time.Now() - - if err = WaitForAPIServerProcess(runner, startTime, time.Second*5); err != nil { + if err = WaitForAPIServerProcess(runner, startTime, time.Second*30); err != nil { exit.Message(reason.RuntimeRestart, `The api server within '{{.name}}' is not up`, out.V{"name": name}) } } From 3172a6ddfcdce23e7486e2733e27a047373ef750 Mon Sep 17 00:00:00 2001 From: Yanshu Zhao Date: Mon, 21 Dec 2020 23:19:41 +0000 Subject: [PATCH 5/5] change log level and change wait function to private --- cmd/minikube/cmd/docker-env.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index fde8f224cc65..0f315594b30e 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -201,13 +201,13 @@ func mustRestartDockerd(name string, runner command.Runner) { // verifying apisever using kverify would add code complexity for a rare case. klog.Warningf("waiting to ensure apisever container is up...") startTime := time.Now() - if err = WaitForAPIServerProcess(runner, startTime, time.Second*30); err != nil { - exit.Message(reason.RuntimeRestart, `The api server within '{{.name}}' is not up`, out.V{"name": name}) + if err = waitForAPIServerProcess(runner, startTime, time.Second*30); err != nil { + klog.Warningf("apiserver container isn't up, error: %v", err) } } } -func WaitForAPIServerProcess(cr command.Runner, start time.Time, timeout time.Duration) error { +func waitForAPIServerProcess(cr command.Runner, start time.Time, timeout time.Duration) error { klog.Infof("waiting for apiserver process to appear ...") err := apiWait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) { if time.Since(start) > timeout {