Parallelize the Config Policy controller E2E tests

Signed-off-by: Yi Rae Kim <yikim@redhat.com>
open-cluster-management-io · Jun 28, 2023 · df14335 · df14335
1 parent 3237da5
commit df14335
Show file tree

Hide file tree

Showing 46 changed files with 548 additions and 492 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -27,89 +27,90 @@ jobs:
         # The "minimum" tag is set in the Makefile
         # KinD tags: https://hub.docker.com/r/kindest/node/tags
         kind:
-          - 'minimum'
-          - 'latest'
+          - "minimum"
+          - "latest"
     name: KinD tests
     steps:
-    - name: Checkout Config Policy Controller
-      uses: actions/checkout@v3
-      with:
-        path: config-policy-controller
-        fetch-depth: 0 # Fetch all history for all tags and branches
-
-    - name: Set up Go
-      uses: actions/setup-go@v3
-      id: go
-      with:
-        go-version-file: config-policy-controller/go.mod
-
-    - name: Verify modules
-      run: |
-        go mod verify
-
-    - name: Verify format
-      run: |
-        make fmt
-        git diff --exit-code
-        make lint
-
-    - name: Verify deploy/operator.yaml
-      run: |
-        make generate-operator-yaml
-        git diff --exit-code
-
-    - name: Unit and Integration Tests
-      run: |
-        make test
-
-    - name: Create K8s KinD Cluster - ${{ matrix.kind }}
-      env:
-        KIND_VERSION: ${{ matrix.kind }}
-      run: |
-        make kind-bootstrap-cluster-dev
-
-    - name: Ensure Service Account kubeconfig
-      run: |
-        KUBECONFIG=${PWD}/kubeconfig_managed make kind-ensure-sa
-
-    - name: E2E Tests
-      run: |
-        export GOPATH=$(go env GOPATH)
-        KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-coverage
-
-    - name: Create K8s KinD Cluster to simulate hosted mode - ${{ matrix.kind }}
-      env:
-        KIND_VERSION: ${{ matrix.kind }}
-      run: |
-        make kind-additional-cluster
-
-    - name: E2E tests that simulate hosted mode
-      run: |
-        export GOPATH=$(go env GOPATH)
-        KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-hosted-mode-coverage
-
-    - name: Verify Deployment Configuration
-      run: |
-        make build-images
-        KUBECONFIG=${PWD}/kubeconfig_managed_e2e make kind-deploy-controller-dev
-
-    - name: E2E tests that require the controller running in a cluster
-      run: |
-        export GOPATH=$(go env GOPATH)
-        KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-running-in-cluster
-
-    - name: Test Coverage Verification
-      if: ${{ github.event_name == 'pull_request' }}
-      run: |
-        make test-coverage
-        make coverage-verify
-
-    - name: Debug
-      if: ${{ failure() }}
-      run: |
-        make e2e-debug
-
-    - name: Clean up cluster
-      if: ${{ always() }}
-      run: |
-        make kind-delete-cluster
+      - name: Checkout Config Policy Controller
+        uses: actions/checkout@v3
+        with:
+          path: config-policy-controller
+          fetch-depth: 0 # Fetch all history for all tags and branches
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        id: go
+        with:
+          go-version-file: config-policy-controller/go.mod
+
+      - name: Verify modules
+        run: |
+          go mod verify
+
+      - name: Verify format
+        run: |
+          make fmt
+          git diff --exit-code
+          make lint
+
+      - name: Verify deploy/operator.yaml
+        run: |
+          make generate-operator-yaml
+          git diff --exit-code
+
+      - name: Unit and Integration Tests
+        run: |
+          make test
+
+      - name: Create K8s KinD Cluster - ${{ matrix.kind }}
+        env:
+          KIND_VERSION: ${{ matrix.kind }}
+        run: |
+          make kind-bootstrap-cluster-dev
+
+      - name: Ensure Service Account kubeconfig
+        run: |
+          KUBECONFIG=${PWD}/kubeconfig_managed make kind-ensure-sa
+
+      - name: E2E Tests
+        run: |
+          kubectl get pod -A
+          export GOPATH=$(go env GOPATH)
+          KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-coverage
+
+      - name: Create K8s KinD Cluster to simulate hosted mode - ${{ matrix.kind }}
+        env:
+          KIND_VERSION: ${{ matrix.kind }}
+        run: |
+          make kind-additional-cluster
+
+      - name: E2E tests that simulate hosted mode
+        run: |
+          export GOPATH=$(go env GOPATH)
+          KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-hosted-mode-coverage
+
+      - name: Verify Deployment Configuration
+        run: |
+          make build-images
+          KUBECONFIG=${PWD}/kubeconfig_managed_e2e make kind-deploy-controller-dev
+
+      - name: E2E tests that require the controller running in a cluster
+        run: |
+          export GOPATH=$(go env GOPATH)
+          KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-running-in-cluster
+
+      - name: Test Coverage Verification
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          make test-coverage
+          make coverage-verify
+
+      - name: Debug
+        if: ${{ failure() }}
+        run: |
+          make e2e-debug
+
+      - name: Clean up cluster
+        if: ${{ always() }}
+        run: |
+          make kind-delete-cluster
diff --git a/Makefile b/Makefile
@@ -205,7 +205,7 @@ install-resources:
 
 .PHONY: e2e-test
 e2e-test: e2e-dependencies
-	$(GINKGO) -v --timeout=2h --fail-fast $(E2E_TEST_ARGS) test/e2e
+	$(GINKGO) -v -p -procs=20 --fail-fast $(E2E_TEST_ARGS) test/e2e
 
 .PHONY: e2e-test-coverage
 e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter='!hosted-mode && !running-in-cluster' --output-dir=.

diff --git a/go.mod b/go.mod
@@ -6,8 +6,8 @@ require (
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
 	github.com/go-logr/zapr v1.2.3
 	github.com/google/go-cmp v0.5.9
-	github.com/onsi/ginkgo/v2 v2.9.4
-	github.com/onsi/gomega v1.27.6
+	github.com/onsi/ginkgo/v2 v2.9.7
+	github.com/onsi/gomega v1.27.8
 	github.com/prometheus/client_golang v1.15.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stolostron/go-log-utils v0.1.2
@@ -80,14 +80,14 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
-	golang.org/x/net v0.9.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/oauth2 v0.7.0 // indirect
 	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.8.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect

diff --git a/go.sum b/go.sum
@@ -156,10 +156,10 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
-github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
-github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
-github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
+github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss=
+github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
+github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
+github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -256,8 +256,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
@@ -303,8 +303,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
-golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

diff --git a/test/e2e/case10_kind_field_test.go b/test/e2e/case10_kind_field_test.go
@@ -20,7 +20,7 @@ const (
 )
 
 var _ = Describe("Test pod obj template handling", func() {
-	Describe("Create a pod policy on managed cluster in ns:"+testNamespace, func() {
+	Describe("Create a pod policy on managed cluster in ns:"+testNamespace, Ordered, func() {
 		It("should create a policy properly on the managed cluster", func() {
 			By("Creating " + case10ConfigPolicyNamePod + " on managed")
 			utils.Kubectl("apply", "-f", case10PolicyYamlPod, "-n", testNamespace)
@@ -60,7 +60,7 @@ var _ = Describe("Test pod obj template handling", func() {
 				return utils.GetComplianceState(managedPlc)
 			}, defaultTimeoutSeconds, 1).Should(Equal("NonCompliant"))
 		})
-		It("Cleans up", func() {
+		AfterAll(func() {
 			policies := []string{
 				case10ConfigPolicyNamePod,
 				case10ConfigPolicyNameCheck,

diff --git a/test/e2e/case11_apiserver_config_test.go b/test/e2e/case11_apiserver_config_test.go
@@ -24,8 +24,8 @@ const (
 	tlsProfileInformYaml             string = "../resources/case11_apiserver_config/tls_profile_inform.yaml"
 )
 
-var _ = Describe("Test APIServer Config policy", func() {
-	Describe("Test etcd encryption and tls profile", func() {
+var _ = Describe("Test APIServer Config policy", Serial, func() {
+	Describe("Test etcd encryption and tls profile", Ordered, func() {
 		It("should be noncompliant for no encryption", func() {
 			By("Creating " + etcdEncryptionInformYaml + " on managed")
 			utils.Kubectl("apply", "-f", etcdEncryptionInformYaml, "-n", testNamespace)
@@ -122,7 +122,7 @@ var _ = Describe("Test APIServer Config policy", func() {
 				return utils.GetComplianceState(informPlc)
 			}, defaultTimeoutSeconds, 1).Should(Equal("Compliant"))
 		})
-		It("Cleans up", func() {
+		AfterAll(func() {
 			policies := []string{
 				etcdEncryptionEnforceName,
 				etcdEncryptionInformName,

diff --git a/test/e2e/case12_list_compare_test.go b/test/e2e/case12_list_compare_test.go
@@ -15,6 +15,7 @@ import (
 const (
 	case12ConfigPolicyNameInform  string = "policy-pod-mh-listinform"
 	case12ConfigPolicyNameEnforce string = "policy-pod-create-listinspec"
+	case12PodName                 string = "nginx-pod-e2e-12"
 	case12InformYaml              string = "../resources/case12_list_compare/case12_pod_inform.yaml"
 	case12EnforceYaml             string = "../resources/case12_list_compare/case12_pod_create.yaml"
 )
@@ -77,7 +78,7 @@ const (
 )
 
 var _ = Describe("Test list handling for musthave", func() {
-	Describe("Create a policy with a nested list on managed cluster in ns:"+testNamespace, func() {
+	Describe("Create a policy with a nested list on managed cluster in ns:"+testNamespace, Ordered, func() {
 		It("should be created properly on the managed cluster", func() {
 			By("Creating " + case12ConfigPolicyNameEnforce + " and " + case12ConfigPolicyNameInform + " on managed")
 			utils.Kubectl("apply", "-f", case12EnforceYaml, "-n", testNamespace)
@@ -101,16 +102,18 @@ var _ = Describe("Test list handling for musthave", func() {
 				return utils.GetComplianceState(managedPlc)
 			}, defaultTimeoutSeconds, 1).Should(Equal("Compliant"))
 		})
-		It("Cleans up", func() {
+		AfterAll(func() {
 			policies := []string{
 				case12ConfigPolicyNameInform,
 				case12ConfigPolicyNameEnforce,
 			}
 
 			deleteConfigPolicies(policies)
+
+			utils.Kubectl("delete", "pod", case12PodName, "-n", "default", "--ignore-not-found")
 		})
 	})
-	Describe("Create a policy with a list field on managed cluster in ns:"+testNamespace, func() {
+	Describe("Create a policy with a list field on managed cluster in ns:"+testNamespace, Ordered, func() {
 		It("should be created properly on the managed cluster", func() {
 			By("Creating " + case12ConfigPolicyNameRoleEnforce + " and " +
 				case12ConfigPolicyNameRoleInform + " on managed")
@@ -135,7 +138,7 @@ var _ = Describe("Test list handling for musthave", func() {
 				return utils.GetComplianceState(managedPlc)
 			}, defaultTimeoutSeconds, 1).Should(Equal("NonCompliant"))
 		})
-		It("Cleans up", func() {
+		AfterAll(func() {
 			policies := []string{
 				case12ConfigPolicyNameRoleInform,
 				case12ConfigPolicyNameRoleEnforce,
@@ -144,7 +147,7 @@ var _ = Describe("Test list handling for musthave", func() {
 			deleteConfigPolicies(policies)
 		})
 	})
-	Describe("Create and patch a role on managed cluster in ns:"+testNamespace, func() {
+	Describe("Create and patch a role on managed cluster in ns:"+testNamespace, Ordered, func() {
 		It("should be created properly on the managed cluster", func() {
 			By("Creating " + case12RoleToPatch + " and " + case12RolePatchEnforce + " on managed")
 			utils.Kubectl("apply", "-f", case12RoleToPatchYaml, "-n", testNamespace)
@@ -178,7 +181,7 @@ var _ = Describe("Test list handling for musthave", func() {
 				return utils.GetComplianceState(managedPlc)
 			}, defaultTimeoutSeconds, 1).Should(Equal("Compliant"))
 		})
-		It("Cleans up", func() {
+		AfterAll(func() {
 			policies := []string{
 				case12RoleToPatch,
 				case12RolePatchEnforce,
@@ -188,7 +191,7 @@ var _ = Describe("Test list handling for musthave", func() {
 			deleteConfigPolicies(policies)
 		})
 	})
-	Describe("Create and patch an oauth object on managed cluster in ns:"+testNamespace, func() {
+	Describe("Create and patch an oauth object on managed cluster in ns:"+testNamespace, Ordered, func() {
 		It("should be created properly on the managed cluster", func() {
 			By("Creating " + case12OauthCreate + " and " + case12OauthPatch + " on managed")
 			utils.Kubectl("apply", "-f", case12OauthCreateYaml, "-n", testNamespace)
@@ -295,7 +298,7 @@ var _ = Describe("Test list handling for musthave", func() {
 			}, defaultTimeoutSeconds, 1).Should(Equal("Compliant"))
 		})
 
-		It("Cleans up", func() {
+		AfterAll(func() {
 			policies := []string{
 				case12OauthCreate,
 				case12OauthPatch,
@@ -311,7 +314,7 @@ var _ = Describe("Test list handling for musthave", func() {
 			deleteConfigPolicies(policies)
 		})
 	})
-	Describe("Create a deployment object with env vars on managed cluster in ns:"+testNamespace, func() {
+	Describe("Create a deployment object with env vars on managed cluster in ns:"+testNamespace, Ordered, func() {
 		It("should only add the list item with prefix and suffix whitespace once", func() {
 			By("Creating " + case12WhitespaceListCreate + " and " + case12WhitespaceListInform + " on managed")
 			utils.Kubectl("apply", "-f", case12WhitespaceListCreateYaml, "-n", testNamespace)
@@ -347,7 +350,7 @@ var _ = Describe("Test list handling for musthave", func() {
 			Expect(envvars).To(HaveLen(1))
 		})
 
-		It("Cleans up", func() {
+		AfterAll(func() {
 			policies := []string{
 				case12WhitespaceListCreate,
 				case12WhitespaceListInform,