From 1de000e375cefa1145e35dc603940d1a6c767e8c Mon Sep 17 00:00:00 2001 From: Yi Rae Kim Date: Mon, 6 Nov 2023 14:29:20 -0500 Subject: [PATCH 1/2] Fix flaky event recording config-policy-controller E2E test Signed-off-by: Yi Rae Kim --- test/e2e/case13_templatization_test.go | 4 +- test/e2e/case15_event_format_test.go | 75 +++++++++++-------- test/e2e/case17_evaluation_interval_test.go | 44 ++++++----- test/e2e/case20_delete_objects_test.go | 16 ++-- .../e2e/case21_alternative_kubeconfig_test.go | 8 +- test/e2e/case23_invalid_field_test.go | 4 +- test/e2e/case32_secret_stringdata_test.go | 2 +- test/e2e/case34_enforce_w_status_test.go | 6 +- test/e2e/case35_no_apiversion_test.go | 2 +- test/e2e/case7_no_spec_test.go | 2 +- test/utils/utils.go | 4 +- 11 files changed, 92 insertions(+), 75 deletions(-) diff --git a/test/e2e/case13_templatization_test.go b/test/e2e/case13_templatization_test.go index 29b9608d..954c5ee4 100644 --- a/test/e2e/case13_templatization_test.go +++ b/test/e2e/case13_templatization_test.go @@ -482,7 +482,7 @@ var _ = Describe("Test templatization", Ordered, func() { case13PruneTmpErr+"-configmap", "default", true, defaultTimeoutSeconds) return configmap - }, defaultTimeoutSeconds, 1).ShouldNot(BeNil()) + }, 30, 1).ShouldNot(BeNil()) By("Change to valid configmap") utils.Kubectl("patch", "configurationpolicy", case13PruneTmpErr, "--type=json", "-p", @@ -520,7 +520,7 @@ var _ = Describe("Test templatization", Ordered, func() { case13PruneTmpErr+"-configmap", "default", true, defaultTimeoutSeconds) return configmap - }, defaultTimeoutSeconds, 1).ShouldNot(BeNil()) + }, 30, 1).ShouldNot(BeNil()) }) AfterAll(func() { utils.Kubectl("delete", "configurationpolicy", case13PruneTmpErr, diff --git a/test/e2e/case15_event_format_test.go b/test/e2e/case15_event_format_test.go index 848434be..f620d1df 100644 --- a/test/e2e/case15_event_format_test.go +++ b/test/e2e/case15_event_format_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,18 +53,20 @@ var _ = Describe("Testing compliance event formatting", Ordered, func() { }, defaultTimeoutSeconds, 1).Should(Equal("Compliant")) By("Checking events on the configurationpolicy") - compPlcEvents := utils.GetMatchingEvents(clientManaged, testNamespace, - case15AlwaysCompliantName, "", "Policy status is Compliant", defaultTimeoutSeconds) - Expect(compPlcEvents).NotTo(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, + case15AlwaysCompliantName, "", "Policy status is Compliant", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) nonCompPlcEvents := utils.GetMatchingEvents(clientManaged, testNamespace, case15AlwaysCompliantName, "", "Policy status is NonCompliant", defaultTimeoutSeconds) Expect(nonCompPlcEvents).To(BeEmpty()) By("Checking events on the parent policy") - compParentEvents := utils.GetMatchingEvents(clientManaged, testNamespace, - case15AlwaysCompliantParentName, "policy: "+testNamespace+"/"+ - case15AlwaysCompliantName, "^Compliant;", defaultTimeoutSeconds) - Expect(compParentEvents).NotTo(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, + case15AlwaysCompliantParentName, "policy: "+testNamespace+"/"+ + case15AlwaysCompliantName, "^Compliant;", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) nonCompParentEvents := utils.GetMatchingEvents(clientManaged, testNamespace, case15AlwaysCompliantParentName, "policy: "+testNamespace+"/"+ case15AlwaysCompliantName, "^NonCompliant;", defaultTimeoutSeconds) @@ -87,19 +90,21 @@ var _ = Describe("Testing compliance event formatting", Ordered, func() { compPlcEvents := utils.GetMatchingEvents(clientManaged, testNamespace, case15NeverCompliantName, "", "Policy status is Compliant", defaultTimeoutSeconds) Expect(compPlcEvents).To(BeEmpty()) - nonCompPlcEvents := utils.GetMatchingEvents(clientManaged, testNamespace, - case15NeverCompliantName, "", "Policy status is NonCompliant", defaultTimeoutSeconds) - Expect(nonCompPlcEvents).NotTo(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, + case15NeverCompliantName, "", "Policy status is NonCompliant", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) By("Checking events on the parent policy") compParentEvents := utils.GetMatchingEvents(clientManaged, testNamespace, case15NeverCompliantParentName, "policy: "+testNamespace+"/"+case15NeverCompliantName, "^Compliant;", defaultTimeoutSeconds) Expect(compParentEvents).To(BeEmpty()) - nonCompParentEvents := utils.GetMatchingEvents(clientManaged, testNamespace, - case15NeverCompliantParentName, "policy: "+testNamespace+"/"+case15NeverCompliantName, - "^NonCompliant;", defaultTimeoutSeconds) - Expect(nonCompParentEvents).NotTo(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, + case15NeverCompliantParentName, "policy: "+testNamespace+"/"+case15NeverCompliantName, + "^NonCompliant;", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) }) It("Records events for a policy that becomes compliant", func() { createConfigPolicyWithParent(case15BecomesCompliantParentYaml, case15BecomesCompliantParentName, @@ -126,17 +131,20 @@ var _ = Describe("Testing compliance event formatting", Ordered, func() { }, defaultTimeoutSeconds, 1).Should(Equal("Compliant")) By("Checking for compliant events on the configurationpolicy and the parent policy") - compPlcEvents := utils.GetMatchingEvents(clientManaged, testNamespace, - case15BecomesCompliantName, "", "Policy status is Compliant", defaultTimeoutSeconds) - Expect(compPlcEvents).NotTo(BeEmpty()) - compParentEventsPreCreation := utils.GetMatchingEvents(clientManaged, testNamespace, - case15BecomesCompliantParentName, "policy: "+testNamespace+"/"+case15BecomesCompliantName, - "^NonCompliant;.*not found in namespace default$", defaultTimeoutSeconds) - Expect(compParentEventsPreCreation).NotTo(BeEmpty()) - compParentEvents := utils.GetMatchingEvents(clientManaged, testNamespace, case15BecomesCompliantParentName, - "policy: "+testNamespace+"/"+case15BecomesCompliantName, - "^Compliant;.*was created successfully in namespace default$", defaultTimeoutSeconds) - Expect(compParentEvents).NotTo(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, + case15BecomesCompliantName, "", "Policy status is Compliant", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, + case15BecomesCompliantParentName, "policy: "+testNamespace+"/"+case15BecomesCompliantName, + "^NonCompliant;.*not found in namespace default$", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, case15BecomesCompliantParentName, + "policy: "+testNamespace+"/"+case15BecomesCompliantName, + "^Compliant;.*was created successfully in namespace default$", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) }) It("Records events for a policy that becomes noncompliant", func() { createConfigPolicyWithParent(case15BecomesNonCompliantParentYaml, case15BecomesNonCompliantParentName, @@ -162,13 +170,16 @@ var _ = Describe("Testing compliance event formatting", Ordered, func() { }, defaultTimeoutSeconds, 1).Should(Equal("NonCompliant")) By("Checking for noncompliant events on the configurationpolicy and the parent policy") - nonCompPlcEvents := utils.GetMatchingEvents(clientManaged, testNamespace, - case15BecomesNonCompliantName, "", "Policy status is NonCompliant", defaultTimeoutSeconds) - Expect(nonCompPlcEvents).NotTo(BeEmpty()) - nonCompParentEvents := utils.GetMatchingEvents(clientManaged, testNamespace, - case15BecomesNonCompliantParentName, "policy: "+testNamespace+"/"+case15BecomesNonCompliantName, - "^NonCompliant;", defaultTimeoutSeconds) - Expect(nonCompParentEvents).NotTo(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, + case15BecomesNonCompliantName, "", "Policy status is NonCompliant", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) + + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, + case15BecomesNonCompliantParentName, "policy: "+testNamespace+"/"+case15BecomesNonCompliantName, + "^NonCompliant;", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) }) AfterAll(func() { policies := []string{ diff --git a/test/e2e/case17_evaluation_interval_test.go b/test/e2e/case17_evaluation_interval_test.go index c13ecc44..44df988b 100644 --- a/test/e2e/case17_evaluation_interval_test.go +++ b/test/e2e/case17_evaluation_interval_test.go @@ -70,28 +70,32 @@ var _ = Describe("Test evaluation interval", Ordered, func() { Expect(lastEvaluatedParsed.Before(lastEvalRefreshedParsed)).To(BeTrue()) By("Verifying that only one event was sent for the configuration policy") - events := utils.GetMatchingEvents( - clientManaged, - testNamespace, - case17PolicyName, - "", - "Policy status is NonCompliant", - defaultTimeoutSeconds, - ) - Expect(events).To(HaveLen(1)) - Expect(events[0].Count).To(Equal(int32(1))) + Eventually(func(g Gomega) { + events := utils.GetMatchingEvents( + clientManaged, + testNamespace, + case17PolicyName, + "", + "Policy status is NonCompliant", + defaultTimeoutSeconds, + ) + g.Expect(events).To(HaveLen(1)) + g.Expect(events[0].Count).To(Equal(int32(1))) + }, defaultTimeoutSeconds, 1).Should(Succeed()) By("Verifying that only one event was sent for the parent policy") - parentEvents := utils.GetMatchingEvents( - clientManaged, - testNamespace, - case17ParentPolicyName, - "policy: "+testNamespace+"/"+case17PolicyName, - "^NonCompliant;", - defaultTimeoutSeconds, - ) - Expect(parentEvents).To(HaveLen(1)) - Expect(parentEvents[0].Count).To(Equal(int32(1))) + Eventually(func(g Gomega) { + parentEvents := utils.GetMatchingEvents( + clientManaged, + testNamespace, + case17ParentPolicyName, + "policy: "+testNamespace+"/"+case17PolicyName, + "^NonCompliant;", + defaultTimeoutSeconds, + ) + g.Expect(parentEvents).To(HaveLen(1)) + g.Expect(parentEvents[0].Count).To(Equal(int32(1))) + }, defaultTimeoutSeconds, 1).Should(Succeed()) }) It("Verifies that a compliant policy is not reevaluated when set to never", func() { diff --git a/test/e2e/case20_delete_objects_test.go b/test/e2e/case20_delete_objects_test.go index f823197f..dbb1f7df 100644 --- a/test/e2e/case20_delete_objects_test.go +++ b/test/e2e/case20_delete_objects_test.go @@ -219,7 +219,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20PodName, "default", true, defaultTimeoutSeconds) return pod - }, defaultTimeoutSeconds, 1).Should(Not(BeNil())) + }, 30, 1).Should(Not(BeNil())) }) It("Should create DeleteIfCreated policy", func() { // delete pod to reset @@ -284,7 +284,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20PodName, "default", true, defaultTimeoutSeconds) return pod - }, defaultTimeoutSeconds, 1).Should(Not(BeNil())) + }, 30, 1).Should(Not(BeNil())) }) It("should handle deleteAll properly for created obj", func() { By("Creating " + case20ConfigPolicyNameExisting + " on managed") @@ -313,7 +313,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20ConfigPolicyNameExisting, testNamespace, false, defaultTimeoutSeconds) return managedPlc - }, defaultTimeoutSeconds, 1).Should(BeNil()) + }, 30, 1).Should(BeNil()) }) It("should handle deleteAll properly for non created obj", func() { By("Creating " + case20PodName + " on default") @@ -323,7 +323,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20PodName, "default", true, defaultTimeoutSeconds) return pod - }, defaultTimeoutSeconds, 1).Should(Not(BeNil())) + }, 30, 1).Should(Not(BeNil())) By("Creating " + case20ConfigPolicyNameExisting + " on managed") utils.Kubectl("apply", "-f", case20PolicyYamlExisting, "-n", testNamespace) plc := utils.GetWithTimeout(clientManagedDynamic, gvrConfigPolicy, @@ -378,13 +378,13 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20PodWithFinalizer, "default", true, defaultTimeoutSeconds) return pod - }, defaultTimeoutSeconds, 1).Should(Not(BeNil())) + }, 30, 1).Should(Not(BeNil())) Consistently(func() interface{} { managedPlc := utils.GetWithTimeout(clientManagedDynamic, gvrConfigPolicy, case20ConfigPolicyNameFinalizer, testNamespace, true, defaultTimeoutSeconds) return utils.GetComplianceState(managedPlc) - }, defaultTimeoutSeconds, 1).Should(Equal("Terminating")) + }, 30, 1).Should(Equal("Terminating")) }) It("should finish delete when pod finalizer is removed", func() { utils.Kubectl( @@ -405,7 +405,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20ConfigPolicyNameFinalizer, testNamespace, false, defaultTimeoutSeconds) return managedPlc - }, defaultTimeoutSeconds, 1).Should(BeNil()) + }, 30, 1).Should(BeNil()) }) It("should handle changing policy from inform to enforce", func() { By("Creating " + case20ConfigPolicyNameChange + " on managed") @@ -445,7 +445,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20ConfigPolicyNameChange, testNamespace, false, defaultTimeoutSeconds) return managedPlc - }, defaultTimeoutSeconds, 1).Should(BeNil()) + }, 30, 1).Should(BeNil()) }) It("should handle changing policy from enforce to inform", func() { By("Creating " + case20ConfigPolicyNameChange + " on managed") diff --git a/test/e2e/case21_alternative_kubeconfig_test.go b/test/e2e/case21_alternative_kubeconfig_test.go index df6f15e8..7ecd01f6 100644 --- a/test/e2e/case21_alternative_kubeconfig_test.go +++ b/test/e2e/case21_alternative_kubeconfig_test.go @@ -9,6 +9,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -74,8 +75,9 @@ var _ = Describe("Test an alternative kubeconfig for policy evaluation", Ordered Expect(err).ToNot(HaveOccurred()) By("Verifying that a compliance event was created on the parent policy") - compParentEvents := utils.GetMatchingEvents(clientManaged, testNamespace, parentPolicyName, - fmt.Sprintf("policy: %v/%v", testNamespace, policyName), "^Compliant;", defaultTimeoutSeconds) - Expect(compParentEvents).NotTo(BeEmpty()) + Eventually(func() []v1.Event { + return utils.GetMatchingEvents(clientManaged, testNamespace, parentPolicyName, + fmt.Sprintf("policy: %v/%v", testNamespace, policyName), "^Compliant;", defaultTimeoutSeconds) + }, defaultTimeoutSeconds, 1).ShouldNot(BeEmpty()) }) }) diff --git a/test/e2e/case23_invalid_field_test.go b/test/e2e/case23_invalid_field_test.go index f9d5af86..69d083c9 100644 --- a/test/e2e/case23_invalid_field_test.go +++ b/test/e2e/case23_invalid_field_test.go @@ -73,7 +73,7 @@ var _ = Describe("Test an objectDefinition with an invalid field", Ordered, func } return startTime.After(compPlcEvents[len(compPlcEvents)-1].LastTimestamp.Time) - }, defaultTimeoutSeconds, 1).Should(BeTrue()) + }, 30, 1).Should(BeTrue()) By("Verifying the message is correct when the " + configMapName + " ConfigMap already exists") configmap := &corev1.ConfigMap{ @@ -118,7 +118,7 @@ var _ = Describe("Test an objectDefinition with an invalid field", Ordered, func } return alreadyExistsStartTime.After(compPlcEvents[len(compPlcEvents)-1].LastTimestamp.Time) - }, defaultTimeoutSeconds, 1).Should(BeTrue()) + }, 30, 1).Should(BeTrue()) }) AfterAll(func() { diff --git a/test/e2e/case32_secret_stringdata_test.go b/test/e2e/case32_secret_stringdata_test.go index d503fbde..12ae10d1 100644 --- a/test/e2e/case32_secret_stringdata_test.go +++ b/test/e2e/case32_secret_stringdata_test.go @@ -54,7 +54,7 @@ var _ = Describe("Test converted stringData being decoded before comparison for "updated", defaultTimeoutSeconds)) return eventlen - }, 60, 2).Should(BeNumerically("<", 1)) + }, 30, 2).Should(BeNumerically("<", 1)) }) AfterAll(func() { diff --git a/test/e2e/case34_enforce_w_status_test.go b/test/e2e/case34_enforce_w_status_test.go index 99ca5046..44a02560 100644 --- a/test/e2e/case34_enforce_w_status_test.go +++ b/test/e2e/case34_enforce_w_status_test.go @@ -12,7 +12,7 @@ import ( "open-cluster-management.io/config-policy-controller/test/utils" ) -var _ = Describe("Test compliance events of enforced policies that define a status", func() { +var _ = Describe("Test compliance events of enforced policies that define a status", Serial, func() { const ( rsrcPath = "../resources/case34_enforce_w_status/" policyYAML = rsrcPath + "policy.yaml" @@ -38,7 +38,7 @@ var _ = Describe("Test compliance events of enforced policies that define a stat Consistently(func() interface{} { return utils.GetMatchingEvents(clientManaged, testNamespace, policyName, cfgPlcName, "^Compliant;", defaultTimeoutSeconds) - }, defaultTimeoutSeconds, 5).Should(BeEmpty()) + }, 30, 5).Should(BeEmpty()) By("Updating the policy") utils.Kubectl("apply", "-f", updatedCfgPlc, "-n", testNamespace) @@ -47,7 +47,7 @@ var _ = Describe("Test compliance events of enforced policies that define a stat Consistently(func() interface{} { return utils.GetMatchingEvents(clientManaged, testNamespace, policyName, cfgPlcName, "^Compliant;", defaultTimeoutSeconds) - }, defaultTimeoutSeconds, 5).Should(BeEmpty()) + }, 30, 5).Should(BeEmpty()) By("Updating the nested policy to increment its generation") utils.Kubectl("apply", "-f", nestedPlcYAML, "-n", testNamespace) diff --git a/test/e2e/case35_no_apiversion_test.go b/test/e2e/case35_no_apiversion_test.go index 0748d8fa..263d5868 100644 --- a/test/e2e/case35_no_apiversion_test.go +++ b/test/e2e/case35_no_apiversion_test.go @@ -33,7 +33,7 @@ var _ = Describe("Test a policy with an objectDefinition that is missing apiVers Consistently(func() interface{} { return utils.GetMatchingEvents(clientManaged, testNamespace, policyName, cfgPlcName, "^Compliant;", defaultTimeoutSeconds) - }, defaultTimeoutSeconds, 5).Should(BeEmpty()) + }, 30, 5).Should(BeEmpty()) }) AfterEach(func() { diff --git a/test/e2e/case7_no_spec_test.go b/test/e2e/case7_no_spec_test.go index c40f2768..2516340c 100644 --- a/test/e2e/case7_no_spec_test.go +++ b/test/e2e/case7_no_spec_test.go @@ -109,7 +109,7 @@ var _ = Describe("Test cluster version obj template handling", func() { case7ObjName, true, defaultTimeoutSeconds) return matchToExpected(managedObj) - }, defaultTimeoutSeconds, 1).Should(Equal(true)) + }, 30, 1).Should(Equal(true)) }) It("should handle change field to null", func() { By("Creating " + case7ConfigPolicyNameNull + " on managed") diff --git a/test/utils/utils.go b/test/utils/utils.go index 3aa17961..e31dbffb 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -99,8 +99,8 @@ func GetWithTimeout( EventuallyWithOffset(1, func() error { var err error - namespace := clientHubDynamic.Resource(gvr).Namespace(namespace) - obj, err = namespace.Get(context.TODO(), name, metav1.GetOptions{}) + obj, err = clientHubDynamic.Resource(gvr).Namespace(namespace). + Get(context.TODO(), name, metav1.GetOptions{}) if wantFound && err != nil { return err } From 492f449cbd1ccf62a7c2919768a2dac040a175f9 Mon Sep 17 00:00:00 2001 From: Yi Rae Kim Date: Mon, 13 Nov 2023 10:30:33 -0500 Subject: [PATCH 2/2] Reduce consistently timeout Signed-off-by: Yi Rae Kim --- test/e2e/case12_list_compare_test.go | 6 ++-- test/e2e/case13_templatization_test.go | 4 +-- test/e2e/case14_selection_test.go | 5 ++-- test/e2e/case20_delete_objects_test.go | 20 ++++++------- test/e2e/case31_policy_history_test.go | 4 +-- test/e2e/case32_secret_stringdata_test.go | 2 +- test/e2e/case34_enforce_w_status_test.go | 4 +-- test/e2e/case35_no_apiversion_test.go | 2 +- test/e2e/case3_imgvuln_test.go | 4 +-- test/e2e/case5_multi_test.go | 2 +- test/e2e/case7_no_spec_test.go | 2 +- test/e2e/e2e_suite_test.go | 36 ++++++++++++----------- 12 files changed, 45 insertions(+), 46 deletions(-) diff --git a/test/e2e/case12_list_compare_test.go b/test/e2e/case12_list_compare_test.go index adbe44ff..644bc94a 100644 --- a/test/e2e/case12_list_compare_test.go +++ b/test/e2e/case12_list_compare_test.go @@ -4,8 +4,6 @@ package e2e import ( - "time" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -334,7 +332,7 @@ var _ = Describe("Test list handling for musthave", func() { case12WhitespaceListCreate, testNamespace, true, defaultTimeoutSeconds) return utils.GetComplianceState(managedPlc) - }, time.Second*20, 1).Should(Equal("Compliant")) + }, defaultConsistentlyDuration, 1).Should(Equal("Compliant")) // Verify that the container list and its environment variable list is correct (there are no duplicates) deploy := utils.GetWithTimeout(clientManagedDynamic, gvrDeployment, @@ -388,7 +386,7 @@ var _ = Describe("Test list handling for musthave", func() { case12ByteCreate, testNamespace, true, defaultTimeoutSeconds) return utils.GetComplianceState(managedPlc) - }, time.Second*20, 1).Should(Equal("Compliant")) + }, defaultConsistentlyDuration, 1).Should(Equal("Compliant")) // Verify that the container list and its environment variable list is correct (there are no duplicates) utils.Kubectl("apply", "-f", case12ByteInformYaml, "-n", testNamespace) diff --git a/test/e2e/case13_templatization_test.go b/test/e2e/case13_templatization_test.go index 954c5ee4..920d01ee 100644 --- a/test/e2e/case13_templatization_test.go +++ b/test/e2e/case13_templatization_test.go @@ -482,7 +482,7 @@ var _ = Describe("Test templatization", Ordered, func() { case13PruneTmpErr+"-configmap", "default", true, defaultTimeoutSeconds) return configmap - }, 30, 1).ShouldNot(BeNil()) + }, defaultConsistentlyDuration, 1).ShouldNot(BeNil()) By("Change to valid configmap") utils.Kubectl("patch", "configurationpolicy", case13PruneTmpErr, "--type=json", "-p", @@ -520,7 +520,7 @@ var _ = Describe("Test templatization", Ordered, func() { case13PruneTmpErr+"-configmap", "default", true, defaultTimeoutSeconds) return configmap - }, 30, 1).ShouldNot(BeNil()) + }, defaultConsistentlyDuration, 1).ShouldNot(BeNil()) }) AfterAll(func() { utils.Kubectl("delete", "configurationpolicy", case13PruneTmpErr, diff --git a/test/e2e/case14_selection_test.go b/test/e2e/case14_selection_test.go index 08e6bfe6..51be0bd9 100644 --- a/test/e2e/case14_selection_test.go +++ b/test/e2e/case14_selection_test.go @@ -5,7 +5,6 @@ package e2e import ( "context" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -116,7 +115,7 @@ var _ = Describe("Test policy compliance with namespace selection", Ordered, fun policy.name, testNamespace, true, defaultTimeoutSeconds) return utils.GetComplianceState(managedPlc) - }, time.Second*20, 1).Should(Equal("NonCompliant")) + }, defaultConsistentlyDuration, 1).Should(Equal("NonCompliant")) } }) @@ -158,7 +157,7 @@ var _ = Describe("Test policy compliance with namespace selection", Ordered, fun policy.name, testNamespace, true, defaultTimeoutSeconds) return utils.GetComplianceState(managedPlc) - }, time.Second*20, 1).Should(Equal("NonCompliant")) + }, defaultConsistentlyDuration, 1).Should(Equal("NonCompliant")) By("Checking that " + policy.name + " has the correct relatedObjects") plc := utils.GetWithTimeout(clientManagedDynamic, gvrConfigPolicy, policy.name, testNamespace, true, defaultTimeoutSeconds) diff --git a/test/e2e/case20_delete_objects_test.go b/test/e2e/case20_delete_objects_test.go index dbb1f7df..e61c3d5b 100644 --- a/test/e2e/case20_delete_objects_test.go +++ b/test/e2e/case20_delete_objects_test.go @@ -219,7 +219,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20PodName, "default", true, defaultTimeoutSeconds) return pod - }, 30, 1).Should(Not(BeNil())) + }, defaultConsistentlyDuration, 1).Should(Not(BeNil())) }) It("Should create DeleteIfCreated policy", func() { // delete pod to reset @@ -284,7 +284,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20PodName, "default", true, defaultTimeoutSeconds) return pod - }, 30, 1).Should(Not(BeNil())) + }, defaultConsistentlyDuration, 1).Should(Not(BeNil())) }) It("should handle deleteAll properly for created obj", func() { By("Creating " + case20ConfigPolicyNameExisting + " on managed") @@ -313,7 +313,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20ConfigPolicyNameExisting, testNamespace, false, defaultTimeoutSeconds) return managedPlc - }, 30, 1).Should(BeNil()) + }, defaultConsistentlyDuration, 1).Should(BeNil()) }) It("should handle deleteAll properly for non created obj", func() { By("Creating " + case20PodName + " on default") @@ -378,13 +378,13 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20PodWithFinalizer, "default", true, defaultTimeoutSeconds) return pod - }, 30, 1).Should(Not(BeNil())) + }, defaultConsistentlyDuration, 1).Should(Not(BeNil())) Consistently(func() interface{} { managedPlc := utils.GetWithTimeout(clientManagedDynamic, gvrConfigPolicy, case20ConfigPolicyNameFinalizer, testNamespace, true, defaultTimeoutSeconds) return utils.GetComplianceState(managedPlc) - }, 30, 1).Should(Equal("Terminating")) + }, defaultConsistentlyDuration, 1).Should(Equal("Terminating")) }) It("should finish delete when pod finalizer is removed", func() { utils.Kubectl( @@ -405,7 +405,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20ConfigPolicyNameFinalizer, testNamespace, false, defaultTimeoutSeconds) return managedPlc - }, 30, 1).Should(BeNil()) + }, defaultConsistentlyDuration, 1).Should(BeNil()) }) It("should handle changing policy from inform to enforce", func() { By("Creating " + case20ConfigPolicyNameChange + " on managed") @@ -445,7 +445,7 @@ var _ = Describe("Test Object deletion", Ordered, func() { case20ConfigPolicyNameChange, testNamespace, false, defaultTimeoutSeconds) return managedPlc - }, 30, 1).Should(BeNil()) + }, defaultConsistentlyDuration, 1).Should(BeNil()) }) It("should handle changing policy from enforce to inform", func() { By("Creating " + case20ConfigPolicyNameChange + " on managed") @@ -735,7 +735,7 @@ var _ = Describe("Object Should not be deleted", Ordered, func() { Expect(oldPod).ShouldNot(BeNil()) return oldPod - }, 20, 1).ShouldNot(BeNil()) + }, defaultConsistentlyDuration, 1).ShouldNot(BeNil()) }) It("check pod is not removed when PruneObjectBehavior is none and namespace changed", func() { oldPod := utils.GetWithTimeout(clientManagedDynamic, gvrPod, @@ -755,7 +755,7 @@ var _ = Describe("Object Should not be deleted", Ordered, func() { Expect(oldPod).ShouldNot(BeNil()) return oldPod - }, 20, 1).ShouldNot(BeNil()) + }, defaultConsistentlyDuration, 1).ShouldNot(BeNil()) }) It("check pod is not removed when PruneObjectBehavior is DeleteAll and spec changed", func() { By("Add PruneObjectBehavior is DeleteAll") @@ -784,6 +784,6 @@ var _ = Describe("Object Should not be deleted", Ordered, func() { Expect(oldPod).ShouldNot(BeNil()) return oldPod - }, 20, 1).ShouldNot(BeNil()) + }, defaultConsistentlyDuration, 1).ShouldNot(BeNil()) }) }) diff --git a/test/e2e/case31_policy_history_test.go b/test/e2e/case31_policy_history_test.go index c4e9351c..6e6f7681 100644 --- a/test/e2e/case31_policy_history_test.go +++ b/test/e2e/case31_policy_history_test.go @@ -30,7 +30,7 @@ var _ = Describe("Test policy history messages when KubeAPI omits values in the configPolicyName, configPolicyName, "NonCompliant;", defaultTimeoutSeconds)) return eventlen - }, 30, 5).Should(BeNumerically("<", 2)) + }, defaultConsistentlyDuration, 5).Should(BeNumerically("<", 2)) By("Checking the events on the parent policy") // NOTE: pick policy event, these event's reason include ConfigPolicyName @@ -39,7 +39,7 @@ var _ = Describe("Test policy history messages when KubeAPI omits values in the policyName, configPolicyName, "NonCompliant;", defaultTimeoutSeconds)) return eventlen - }, 30, 5).Should(BeNumerically("<", 3)) + }, defaultConsistentlyDuration, 5).Should(BeNumerically("<", 3)) } const ( diff --git a/test/e2e/case32_secret_stringdata_test.go b/test/e2e/case32_secret_stringdata_test.go index 12ae10d1..a487915b 100644 --- a/test/e2e/case32_secret_stringdata_test.go +++ b/test/e2e/case32_secret_stringdata_test.go @@ -54,7 +54,7 @@ var _ = Describe("Test converted stringData being decoded before comparison for "updated", defaultTimeoutSeconds)) return eventlen - }, 30, 2).Should(BeNumerically("<", 1)) + }, defaultConsistentlyDuration, 2).Should(BeNumerically("<", 1)) }) AfterAll(func() { diff --git a/test/e2e/case34_enforce_w_status_test.go b/test/e2e/case34_enforce_w_status_test.go index 44a02560..796e98bf 100644 --- a/test/e2e/case34_enforce_w_status_test.go +++ b/test/e2e/case34_enforce_w_status_test.go @@ -38,7 +38,7 @@ var _ = Describe("Test compliance events of enforced policies that define a stat Consistently(func() interface{} { return utils.GetMatchingEvents(clientManaged, testNamespace, policyName, cfgPlcName, "^Compliant;", defaultTimeoutSeconds) - }, 30, 5).Should(BeEmpty()) + }, defaultConsistentlyDuration, 5).Should(BeEmpty()) By("Updating the policy") utils.Kubectl("apply", "-f", updatedCfgPlc, "-n", testNamespace) @@ -47,7 +47,7 @@ var _ = Describe("Test compliance events of enforced policies that define a stat Consistently(func() interface{} { return utils.GetMatchingEvents(clientManaged, testNamespace, policyName, cfgPlcName, "^Compliant;", defaultTimeoutSeconds) - }, 30, 5).Should(BeEmpty()) + }, defaultConsistentlyDuration, 5).Should(BeEmpty()) By("Updating the nested policy to increment its generation") utils.Kubectl("apply", "-f", nestedPlcYAML, "-n", testNamespace) diff --git a/test/e2e/case35_no_apiversion_test.go b/test/e2e/case35_no_apiversion_test.go index 263d5868..2cc4c917 100644 --- a/test/e2e/case35_no_apiversion_test.go +++ b/test/e2e/case35_no_apiversion_test.go @@ -33,7 +33,7 @@ var _ = Describe("Test a policy with an objectDefinition that is missing apiVers Consistently(func() interface{} { return utils.GetMatchingEvents(clientManaged, testNamespace, policyName, cfgPlcName, "^Compliant;", defaultTimeoutSeconds) - }, 30, 5).Should(BeEmpty()) + }, defaultConsistentlyDuration, 5).Should(BeEmpty()) }) AfterEach(func() { diff --git a/test/e2e/case3_imgvuln_test.go b/test/e2e/case3_imgvuln_test.go index af64b393..d7deee48 100644 --- a/test/e2e/case3_imgvuln_test.go +++ b/test/e2e/case3_imgvuln_test.go @@ -70,7 +70,7 @@ var _ = Describe("Test img vulnerability obj template handling", func() { case3ConfigPolicyNameVuln, testNamespace, true, defaultTimeoutSeconds) return utils.GetComplianceState(managedPlc) - }, 20, 1).Should(Equal("NonCompliant")) + }, defaultConsistentlyDuration, 1).Should(Equal("NonCompliant")) deleteConfigPolicies([]string{case3ConfigPolicyNameVuln}) }) @@ -91,7 +91,7 @@ var _ = Describe("Test img vulnerability obj template handling", func() { case3ConfigPolicyNameVulnObj, testNamespace, true, defaultTimeoutSeconds) return utils.GetComplianceState(managedPlc) - }, 20, 1).Should(Equal("NonCompliant")) + }, defaultConsistentlyDuration, 1).Should(Equal("NonCompliant")) deleteConfigPolicies([]string{case3ConfigPolicyNameVulnObj}) }) diff --git a/test/e2e/case5_multi_test.go b/test/e2e/case5_multi_test.go index 6d1c9d1d..d74dabfe 100644 --- a/test/e2e/case5_multi_test.go +++ b/test/e2e/case5_multi_test.go @@ -21,7 +21,7 @@ const ( case5ComboYaml string = "../resources/case5_multi/case5_multi_combo.yaml" ) -var _ = Describe("Test multiple obj template handling", func() { +var _ = Describe("Test multiple obj template handling", Ordered, func() { Describe("Create a policy on managed cluster in ns:"+testNamespace, Ordered, func() { It("should be created properly on the managed cluster", func() { By("Creating " + case5ConfigPolicyNameInform + " and " + case5ConfigPolicyNameCombo + " on managed") diff --git a/test/e2e/case7_no_spec_test.go b/test/e2e/case7_no_spec_test.go index 2516340c..bf6bce0f 100644 --- a/test/e2e/case7_no_spec_test.go +++ b/test/e2e/case7_no_spec_test.go @@ -109,7 +109,7 @@ var _ = Describe("Test cluster version obj template handling", func() { case7ObjName, true, defaultTimeoutSeconds) return matchToExpected(managedObj) - }, 30, 1).Should(Equal(true)) + }, defaultConsistentlyDuration, 1).Should(Equal(true)) }) It("should handle change field to null", func() { By("Creating " + case7ConfigPolicyNameNull + " on managed") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 9460b89b..6592aefc 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -29,23 +29,24 @@ import ( ) var ( - testNamespace string - defaultTimeoutSeconds int - kubeconfigManaged string - clientManaged kubernetes.Interface - clientManagedDynamic dynamic.Interface - gvrAPIService schema.GroupVersionResource - gvrConfigPolicy schema.GroupVersionResource - gvrCRD schema.GroupVersionResource - gvrPod schema.GroupVersionResource - gvrRole schema.GroupVersionResource - gvrNS schema.GroupVersionResource - gvrSCC schema.GroupVersionResource - gvrSecret schema.GroupVersionResource - gvrClusterClaim schema.GroupVersionResource - gvrConfigMap schema.GroupVersionResource - gvrDeployment schema.GroupVersionResource - gvrPolicy schema.GroupVersionResource + testNamespace string + defaultTimeoutSeconds int + defaultConsistentlyDuration int + kubeconfigManaged string + clientManaged kubernetes.Interface + clientManagedDynamic dynamic.Interface + gvrAPIService schema.GroupVersionResource + gvrConfigPolicy schema.GroupVersionResource + gvrCRD schema.GroupVersionResource + gvrPod schema.GroupVersionResource + gvrRole schema.GroupVersionResource + gvrNS schema.GroupVersionResource + gvrSCC schema.GroupVersionResource + gvrSecret schema.GroupVersionResource + gvrClusterClaim schema.GroupVersionResource + gvrConfigMap schema.GroupVersionResource + gvrDeployment schema.GroupVersionResource + gvrPolicy schema.GroupVersionResource defaultImageRegistry string ) @@ -107,6 +108,7 @@ var _ = BeforeSuite(func() { defaultImageRegistry = "quay.io/open-cluster-management" testNamespace = "managed" defaultTimeoutSeconds = 60 + defaultConsistentlyDuration = 25 By("Create watch namespace if needed") namespaces := clientManaged.CoreV1().Namespaces() if _, err := namespaces.Get(