From e8851fed03793059a0b289244e10631185fa27f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Thu, 28 Mar 2019 14:17:18 +0100 Subject: [PATCH] Add timeout after kubelet binary and kubeadm upgrade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/upgrader/upgrade/upgrade.go | 8 ++++++++ pkg/upgrader/upgrade/upgrade_follower.go | 9 ++++++--- pkg/upgrader/upgrade/upgrade_leader.go | 9 ++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pkg/upgrader/upgrade/upgrade.go b/pkg/upgrader/upgrade/upgrade.go index 13230a1ae..240dca3ea 100644 --- a/pkg/upgrader/upgrade/upgrade.go +++ b/pkg/upgrader/upgrade/upgrade.go @@ -17,6 +17,8 @@ limitations under the License. package upgrade import ( + "time" + "github.com/pkg/errors" "github.com/kubermatic/kubeone/pkg/certificate" @@ -28,6 +30,12 @@ import ( const ( labelUpgradeLock = "kubeone.io/upgrade-in-progress" labelControlPlaneNode = "node-role.kubernetes.io/master" + // timeoutKubeletUpgrade is time for how long kubeone will wait after upgrading kubelet + // and running the upgrade process on the node + timeoutKubeletUpgrade = 1 * time.Minute + // timeoutNodeUpgrade is time for how long kubeone will wait after finishing the upgrade + // process on the node + timeoutNodeUpgrade = 15 * time.Second ) // Upgrade performs all the steps required to upgrade Kubernetes on diff --git a/pkg/upgrader/upgrade/upgrade_follower.go b/pkg/upgrader/upgrade/upgrade_follower.go index 494e62f25..c50c6ff85 100644 --- a/pkg/upgrader/upgrade/upgrade_follower.go +++ b/pkg/upgrader/upgrade/upgrade_follower.go @@ -45,20 +45,23 @@ func upgradeFollowerExecutor(ctx *util.Context, node *config.HostConfig, conn ss return errors.Wrap(err, "failed to upgrade kubernetes binaries on follower control plane") } + logger.Infof("Waiting %v seconds to ensure kubelet is up…", timeoutKubeletUpgrade.String()) + time.Sleep(timeoutKubeletUpgrade) + logger.Infoln("Running 'kubeadm upgrade' on the follower control plane node…") err = upgradeFollowerControlPlane(ctx) if err != nil { return errors.Wrap(err, "failed to upgrade follower control plane") } + logger.Infof("Waiting %v seconds to ensure all components are up…", timeoutNodeUpgrade.String()) + time.Sleep(timeoutNodeUpgrade) + logger.Infoln("Unlabeling follower control plane…") err = unlabelNode(ctx.DynamicClient, node) if err != nil { return errors.Wrap(err, "failed to unlabel follower control plane node") } - logger.Infoln("Waiting 10 seconds to ensure all components are up…") - time.Sleep(10 * time.Second) - return nil } diff --git a/pkg/upgrader/upgrade/upgrade_leader.go b/pkg/upgrader/upgrade/upgrade_leader.go index b8aa71932..0bdcba255 100644 --- a/pkg/upgrader/upgrade/upgrade_leader.go +++ b/pkg/upgrader/upgrade/upgrade_leader.go @@ -43,6 +43,9 @@ func upgradeLeaderExecutor(ctx *util.Context, node *config.HostConfig, conn ssh. return errors.Wrap(err, "failed to upgrade kubernetes binaries on leader control plane") } + logger.Infof("Waiting %v seconds to ensure kubelet is up…", timeoutKubeletUpgrade.String()) + time.Sleep(timeoutKubeletUpgrade) + logger.Infoln("Generating kubeadm config …") if err := generateKubeadmConfig(ctx, node); err != nil { return errors.Wrap(err, "failed to generate kubeadm config") @@ -58,13 +61,13 @@ func upgradeLeaderExecutor(ctx *util.Context, node *config.HostConfig, conn ssh. return errors.Wrap(err, "failed to run 'kubeadm upgrade' on leader control plane") } + logger.Infof("Waiting %v seconds to ensure all components are up…", timeoutNodeUpgrade.String()) + time.Sleep(timeoutNodeUpgrade) + logger.Infoln("Unlabeling leader control plane…") if err := unlabelNode(ctx.DynamicClient, node); err != nil { return errors.Wrap(err, "failed to unlabel leader control plane node") } - logger.Infoln("Waiting 10 seconds to ensure all components are up…") - time.Sleep(10 * time.Second) - return nil }