Restart unhealthy API servers when provisioning/upgrading clusters

Signed-off-by: Marko Mudrinić <mudrinic.mare@gmail.com>
kubermatic · Feb 8, 2021 · 8c87b06 · 8c87b06
1 parent 222201d
commit 8c87b06
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 0 deletions.
diff --git a/pkg/scripts/node.go b/pkg/scripts/node.go
@@ -29,6 +29,17 @@ var (
 		sudo KUBECONFIG=/etc/kubernetes/admin.conf \
 		kubectl drain {{ .NODE_NAME }} --ignore-daemonsets --delete-local-data
 	`)
+
+	restartKubeAPIServerTemplate = heredoc.Doc(`
+		apiserver_id=$(sudo crictl ps --name=kube-apiserver -q)
+		[ -z "$apiserver_id" ] && exit 1
+		
+		sudo crictl logs "$apiserver_id" > /tmp/kube-apiserver.log 2>&1
+		if grep -q "etcdserver: no leader" /tmp/kube-apiserver.log; then
+			sudo crictl rm "$apiserver_id"
+			sleep 10
+		fi
+	`)
 )
 
 func DrainNode(nodeName string) (string, error) {
@@ -40,3 +51,7 @@ func DrainNode(nodeName string) (string, error) {
 func Hostname() string {
 	return hostnameScript
 }
+
+func RestartKubeAPIServer() string {
+	return restartKubeAPIServerTemplate
+}
diff --git a/pkg/tasks/nodes.go b/pkg/tasks/nodes.go
@@ -56,3 +56,15 @@ func uncordonNode(s *state.State, host kubeoneapi.HostConfig) error {
 
 	return errors.WithStack(updateErr)
 }
+
+func restartKubeAPIServer(s *state.State) error {
+	s.Logger.Infoln("Restarting unhealthy API servers if needed...")
+	return s.RunTaskOnControlPlane(func(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connection) error {
+		_, _, err := s.Runner.Run(scripts.RestartKubeAPIServer(), nil)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}, state.RunSequentially)
+}
diff --git a/pkg/tasks/tasks.go b/pkg/tasks/tasks.go
@@ -114,6 +114,7 @@ func WithFullInstall(t Tasks) Tasks {
 			{Fn: repairClusterIfNeeded, ErrMsg: "failed to repair cluster"},
 			{Fn: joinControlplaneNode, ErrMsg: "failed to join other masters a cluster"},
 			{Fn: saveKubeconfig, ErrMsg: "failed to save kubeconfig to the local machine"},
+			{Fn: restartKubeAPIServer, ErrMsg: "failed to restart unhealthy kube-apiserver"},
 		}...).
 		append(kubernetesResources()...).
 		append(
@@ -188,6 +189,7 @@ func WithUpgrade(t Tasks) Tasks {
 		}...).
 		append(kubernetesResources()...).
 		append(
+			Task{Fn: restartKubeAPIServer, ErrMsg: "failed to restart unhealthy kube-apiserver"},
 			Task{Fn: upgradeStaticWorkers, ErrMsg: "unable to upgrade static worker nodes"},
 			Task{
 				Fn:         upgradeMachineDeployments,