From ee379fe8e91ba7960e8a4ee9caa4e3b1bbce9291 Mon Sep 17 00:00:00 2001 From: Loic Devulder Date: Wed, 26 Jul 2023 09:33:19 +0200 Subject: [PATCH] ci: fix sporadic issue during rollout check 1st or 2nd rollout command can sporadically fail. Signed-off-by: Loic Devulder --- tests/e2e/app_test.go | 8 +++++--- tests/e2e/upgrade_test.go | 33 ++++++++++++++++++--------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/e2e/app_test.go b/tests/e2e/app_test.go index f96501748..694d7f8ab 100644 --- a/tests/e2e/app_test.go +++ b/tests/e2e/app_test.go @@ -51,9 +51,11 @@ var _ = Describe("E2E - Checking a simple application", Label("check-app"), func By("Waiting for deployment to be rollout", func() { // Wait for application to be started - status, err := kubectl.Run("rollout", "status", "deployment/"+appName) - Expect(err).To(Not(HaveOccurred())) - Expect(status).To(ContainSubstring("successfully rolled out")) + // NOTE: 1st or 2nd rollout command can sporadically fail, so better to use Eventually here + Eventually(func() string { + status, _ := kubectl.Run("rollout", "status", "deployment/"+appName) + return status + }, tools.SetTimeout(2*time.Minute), 30*time.Second).Should(ContainSubstring("successfully rolled out")) }) By("Checking application", func() { diff --git a/tests/e2e/upgrade_test.go b/tests/e2e/upgrade_test.go index 541a1140e..ae360dd49 100644 --- a/tests/e2e/upgrade_test.go +++ b/tests/e2e/upgrade_test.go @@ -96,28 +96,31 @@ var _ = Describe("E2E - Upgrading Rancher Manager", Label("upgrade-rancher-manag Expect(err).To(Not(HaveOccurred())) // Wait for Rancher Manager to be restarted - status, err := kubectl.Run( - "rollout", - "--namespace", "cattle-system", - "status", "deployment/rancher", - ) - Expect(err).To(Not(HaveOccurred())) - Expect(status).To(ContainSubstring("successfully rolled out")) + // NOTE: 1st or 2nd rollout command can sporadically fail, so better to use Eventually here + Eventually(func() string { + status, _ := kubectl.Run( + "rollout", + "--namespace", "cattle-system", + "status", "deployment/rancher", + ) + return status + }, tools.SetTimeout(2*time.Minute), 30*time.Second).Should(ContainSubstring("successfully rolled out")) // Check that all Rancher Manager pods are running - checkList := [][]string{ - {"cattle-system", "app=rancher"}, - {"cattle-fleet-local-system", "app=fleet-agent"}, - {"cattle-system", "app=rancher-webhook"}, - } - err = rancher.CheckPod(k, checkList) - Expect(err).To(Not(HaveOccurred())) + Eventually(func() error { + checkList := [][]string{ + {"cattle-system", "app=rancher"}, + {"cattle-fleet-local-system", "app=fleet-agent"}, + {"cattle-system", "app=rancher-webhook"}, + } + return rancher.CheckPod(k, checkList) + }, tools.SetTimeout(3*time.Minute), 10*time.Second).Should(Not(HaveOccurred())) // Check that all pods are using the same version Eventually(func() int { out, _ := kubectl.Run(getImageVersion...) return len(strings.Fields(out)) - }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(Equal(1)) + }, tools.SetTimeout(3*time.Minute), 10*time.Second).Should(Equal(1)) // Get after-upgrade Rancher Manager version // and check that it's different to the before-upgrade version