Parallelize the Config Policy controller E2E tests

Signed-off-by: Yi Rae Kim <yikim@redhat.com>

.github/workflows/kind.yml

@@ -27,89 +27,90 @@ jobs:
  # The "minimum" tag is set in the Makefile
  # KinD tags: https://hub.docker.com/r/kindest/node/tags
  kind:
- - 'minimum'
- - 'latest'
+ - "minimum"
+ - "latest"
  name: KinD tests
  steps:
- - name: Checkout Config Policy Controller
- uses: actions/checkout@v3
- with:
- path: config-policy-controller
- fetch-depth: 0 # Fetch all history for all tags and branches
-
- - name: Set up Go
- uses: actions/setup-go@v3
- id: go
- with:
- go-version-file: config-policy-controller/go.mod
-
- - name: Verify modules
- run: |
- go mod verify
-
- - name: Verify format
- run: |
- make fmt
- git diff --exit-code
- make lint
-
- - name: Verify deploy/operator.yaml
- run: |
- make generate-operator-yaml
- git diff --exit-code
-
- - name: Unit and Integration Tests
- run: |
- make test
-
- - name: Create K8s KinD Cluster - ${{ matrix.kind }}
- env:
- KIND_VERSION: ${{ matrix.kind }}
- run: |
- make kind-bootstrap-cluster-dev
-
- - name: Ensure Service Account kubeconfig
- run: |
- KUBECONFIG=${PWD}/kubeconfig_managed make kind-ensure-sa
-
- - name: E2E Tests
- run: |
- export GOPATH=$(go env GOPATH)
- KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-coverage
-
- - name: Create K8s KinD Cluster to simulate hosted mode - ${{ matrix.kind }}
- env:
- KIND_VERSION: ${{ matrix.kind }}
- run: |
- make kind-additional-cluster
-
- - name: E2E tests that simulate hosted mode
- run: |
- export GOPATH=$(go env GOPATH)
- KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-hosted-mode-coverage
-
- - name: Verify Deployment Configuration
- run: |
- make build-images
- KUBECONFIG=${PWD}/kubeconfig_managed_e2e make kind-deploy-controller-dev
-
- - name: E2E tests that require the controller running in a cluster
- run: |
- export GOPATH=$(go env GOPATH)
- KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-running-in-cluster
-
- - name: Test Coverage Verification
- if: ${{ github.event_name == 'pull_request' }}
- run: |
- make test-coverage
- make coverage-verify
-
- - name: Debug
- if: ${{ failure() }}
- run: |
- make e2e-debug
-
- - name: Clean up cluster
- if: ${{ always() }}
- run: |
- make kind-delete-cluster
+ - name: Checkout Config Policy Controller
+ uses: actions/checkout@v3
+ with:
+ path: config-policy-controller
+ fetch-depth: 0 # Fetch all history for all tags and branches
+
+ - name: Set up Go
+ uses: actions/setup-go@v3
+ id: go
+ with:
+ go-version-file: config-policy-controller/go.mod
+
+ - name: Verify modules
+ run: |
+ go mod verify
+
+ - name: Verify format
+ run: |
+ make fmt
+ git diff --exit-code
+ make lint
+
+ - name: Verify deploy/operator.yaml
+ run: |
+ make generate-operator-yaml
+ git diff --exit-code
+
+ - name: Unit and Integration Tests
+ run: |
+ make test
+
+ - name: Create K8s KinD Cluster - ${{ matrix.kind }}
+ env:
+ KIND_VERSION: ${{ matrix.kind }}
+ run: |
+ make kind-bootstrap-cluster-dev
+
+ - name: Ensure Service Account kubeconfig
+ run: |
+ KUBECONFIG=${PWD}/kubeconfig_managed make kind-ensure-sa
+
+ - name: E2E Tests
+ run: |
+ kubectl get pod -A
+ export GOPATH=$(go env GOPATH)
+ KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-coverage
+
+ - name: Create K8s KinD Cluster to simulate hosted mode - ${{ matrix.kind }}
+ env:
+ KIND_VERSION: ${{ matrix.kind }}
+ run: |
+ make kind-additional-cluster
+
+ - name: E2E tests that simulate hosted mode
+ run: |
+ export GOPATH=$(go env GOPATH)
+ KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-hosted-mode-coverage
+
+ - name: Verify Deployment Configuration
+ run: |
+ make build-images
+ KUBECONFIG=${PWD}/kubeconfig_managed_e2e make kind-deploy-controller-dev
+
+ - name: E2E tests that require the controller running in a cluster
+ run: |
+ export GOPATH=$(go env GOPATH)
+ KUBECONFIG=${PWD}/kubeconfig_managed make e2e-test-running-in-cluster
+
+ - name: Test Coverage Verification
+ if: ${{ github.event_name == 'pull_request' }}
+ run: |
+ make test-coverage
+ make coverage-verify
+
+ - name: Debug
+ if: ${{ failure() }}
+ run: |
+ make e2e-debug
+
+ - name: Clean up cluster
+ if: ${{ always() }}
+ run: |
+ make kind-delete-cluster

Makefile

@@ -205,7 +205,7 @@ install-resources:
 
 .PHONY: e2e-test
 e2e-test: e2e-dependencies
- $(GINKGO) -v --fail-fast $(E2E_TEST_ARGS) test/e2e
+ $(GINKGO) -v -p -procs=7 --fail-fast $(E2E_TEST_ARGS) test/e2e
 
 .PHONY: e2e-test-coverage
 e2e-test-coverage: E2E_TEST_ARGS = --json-report=report_e2e.json --label-filter='!hosted-mode && !running-in-cluster' --output-dir=.

go.mod

@@ -6,8 +6,8 @@ require (
  github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32
  github.com/go-logr/zapr v1.2.3
  github.com/google/go-cmp v0.5.9
- github.com/onsi/ginkgo/v2 v2.9.4
- github.com/onsi/gomega v1.27.6
+ github.com/onsi/ginkgo/v2 v2.9.7
+ github.com/onsi/gomega v1.27.8
  github.com/prometheus/client_golang v1.15.1
  github.com/spf13/pflag v1.0.5
  github.com/stolostron/go-log-utils v0.1.2
@@ -80,14 +80,14 @@ require (
  go.uber.org/multierr v1.11.0 // indirect
  go.uber.org/zap v1.24.0 // indirect
  golang.org/x/crypto v0.8.0 // indirect
- golang.org/x/net v0.9.0 // indirect
+ golang.org/x/net v0.10.0 // indirect
  golang.org/x/oauth2 v0.7.0 // indirect
  golang.org/x/sync v0.2.0 // indirect
  golang.org/x/sys v0.8.0 // indirect
  golang.org/x/term v0.8.0 // indirect
  golang.org/x/text v0.9.0 // indirect
  golang.org/x/time v0.3.0 // indirect
- golang.org/x/tools v0.8.0 // indirect
+ golang.org/x/tools v0.9.1 // indirect
  gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
  google.golang.org/appengine v1.6.7 // indirect
  google.golang.org/protobuf v1.30.0 // indirect

go.sum

@@ -156,10 +156,10 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE=
-github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
-github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
-github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
+github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss=
+github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
+github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
+github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -256,8 +256,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
@@ -303,8 +303,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
-golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

test/e2e/case10_kind_field_test.go

@@ -20,7 +20,7 @@ const (
 )
 
 var _ = Describe("Test pod obj template handling", func() {
- Describe("Create a pod policy on managed cluster in ns:"+testNamespace, func() {
+ Describe("Create a pod policy on managed cluster in ns:"+testNamespace, Ordered, func() {
  It("should create a policy properly on the managed cluster", func() {
  By("Creating " + case10ConfigPolicyNamePod + " on managed")
  utils.Kubectl("apply", "-f", case10PolicyYamlPod, "-n", testNamespace)
@@ -60,7 +60,7 @@ var _ = Describe("Test pod obj template handling", func() {
  return utils.GetComplianceState(managedPlc)
  }, defaultTimeoutSeconds, 1).Should(Equal("NonCompliant"))
  })
- It("Cleans up", func() {
+ AfterAll(func() {
  policies := []string{
  case10ConfigPolicyNamePod,
  case10ConfigPolicyNameCheck,

test/e2e/case11_apiserver_config_test.go

@@ -24,8 +24,8 @@ const (
  tlsProfileInformYaml string = "../resources/case11_apiserver_config/tls_profile_inform.yaml"
 )
 
-var _ = Describe("Test APIServer Config policy", func() {
- Describe("Test etcd encryption and tls profile", func() {
+var _ = Describe("Test APIServer Config policy", Serial, func() {
+ Describe("Test etcd encryption and tls profile", Ordered, func() {
  It("should be noncompliant for no encryption", func() {
  By("Creating " + etcdEncryptionInformYaml + " on managed")
  utils.Kubectl("apply", "-f", etcdEncryptionInformYaml, "-n", testNamespace)
@@ -122,7 +122,7 @@ var _ = Describe("Test APIServer Config policy", func() {
  return utils.GetComplianceState(informPlc)
  }, defaultTimeoutSeconds, 1).Should(Equal("Compliant"))
  })
- It("Cleans up", func() {
+ AfterAll(func() {
  policies := []string{
  etcdEncryptionEnforceName,
  etcdEncryptionInformName,

test/e2e/case12_list_compare_test.go

@@ -15,6 +15,7 @@ import (
 const (
  case12ConfigPolicyNameInform string = "policy-pod-mh-listinform"
  case12ConfigPolicyNameEnforce string = "policy-pod-create-listinspec"
+ case12PodName string = "nginx-pod-e2e-12"
  case12InformYaml string = "../resources/case12_list_compare/case12_pod_inform.yaml"
  case12EnforceYaml string = "../resources/case12_list_compare/case12_pod_create.yaml"
 )
@@ -77,7 +78,7 @@ const (
 )
 
 var _ = Describe("Test list handling for musthave", func() {
- Describe("Create a policy with a nested list on managed cluster in ns:"+testNamespace, func() {
+ Describe("Create a policy with a nested list on managed cluster in ns:"+testNamespace, Ordered, func() {
  It("should be created properly on the managed cluster", func() {
  By("Creating " + case12ConfigPolicyNameEnforce + " and " + case12ConfigPolicyNameInform + " on managed")
  utils.Kubectl("apply", "-f", case12EnforceYaml, "-n", testNamespace)
@@ -101,16 +102,18 @@ var _ = Describe("Test list handling for musthave", func() {
  return utils.GetComplianceState(managedPlc)
  }, defaultTimeoutSeconds, 1).Should(Equal("Compliant"))
  })
- It("Cleans up", func() {
+ AfterAll(func() {
  policies := []string{
  case12ConfigPolicyNameInform,
  case12ConfigPolicyNameEnforce,
  }
 
  deleteConfigPolicies(policies)
+
+ utils.Kubectl("delete", "pod", case12PodName, "-n", "default", "--ignore-not-found")
  })
  })
- Describe("Create a policy with a list field on managed cluster in ns:"+testNamespace, func() {
+ Describe("Create a policy with a list field on managed cluster in ns:"+testNamespace, Ordered, func() {
  It("should be created properly on the managed cluster", func() {
  By("Creating " + case12ConfigPolicyNameRoleEnforce + " and " +
  case12ConfigPolicyNameRoleInform + " on managed")
@@ -135,7 +138,7 @@ var _ = Describe("Test list handling for musthave", func() {
  return utils.GetComplianceState(managedPlc)
  }, defaultTimeoutSeconds, 1).Should(Equal("NonCompliant"))
  })
- It("Cleans up", func() {
+ AfterAll(func() {
  policies := []string{
  case12ConfigPolicyNameRoleInform,
  case12ConfigPolicyNameRoleEnforce,
@@ -144,7 +147,7 @@ var _ = Describe("Test list handling for musthave", func() {
  deleteConfigPolicies(policies)
  })
  })
- Describe("Create and patch a role on managed cluster in ns:"+testNamespace, func() {
+ Describe("Create and patch a role on managed cluster in ns:"+testNamespace, Ordered, func() {
  It("should be created properly on the managed cluster", func() {
  By("Creating " + case12RoleToPatch + " and " + case12RolePatchEnforce + " on managed")
  utils.Kubectl("apply", "-f", case12RoleToPatchYaml, "-n", testNamespace)
@@ -178,7 +181,7 @@ var _ = Describe("Test list handling for musthave", func() {
  return utils.GetComplianceState(managedPlc)
  }, defaultTimeoutSeconds, 1).Should(Equal("Compliant"))
  })
- It("Cleans up", func() {
+ AfterAll(func() {
  policies := []string{
  case12RoleToPatch,
  case12RolePatchEnforce,
@@ -188,7 +191,7 @@ var _ = Describe("Test list handling for musthave", func() {
  deleteConfigPolicies(policies)
  })
  })
- Describe("Create and patch an oauth object on managed cluster in ns:"+testNamespace, func() {
+ Describe("Create and patch an oauth object on managed cluster in ns:"+testNamespace, Ordered, func() {
  It("should be created properly on the managed cluster", func() {
  By("Creating " + case12OauthCreate + " and " + case12OauthPatch + " on managed")
  utils.Kubectl("apply", "-f", case12OauthCreateYaml, "-n", testNamespace)
@@ -295,7 +298,7 @@ var _ = Describe("Test list handling for musthave", func() {
  }, defaultTimeoutSeconds, 1).Should(Equal("Compliant"))
  })
 
- It("Cleans up", func() {
+ AfterAll(func() {
  policies := []string{
  case12OauthCreate,
  case12OauthPatch,
@@ -311,7 +314,7 @@ var _ = Describe("Test list handling for musthave", func() {
  deleteConfigPolicies(policies)
  })
  })
- Describe("Create a deployment object with env vars on managed cluster in ns:"+testNamespace, func() {
+ Describe("Create a deployment object with env vars on managed cluster in ns:"+testNamespace, Ordered, func() {
  It("should only add the list item with prefix and suffix whitespace once", func() {
  By("Creating " + case12WhitespaceListCreate + " and " + case12WhitespaceListInform + " on managed")
  utils.Kubectl("apply", "-f", case12WhitespaceListCreateYaml, "-n", testNamespace)
@@ -347,7 +350,7 @@ var _ = Describe("Test list handling for musthave", func() {
  Expect(envvars).To(HaveLen(1))
  })
 
- It("Cleans up", func() {
+ AfterAll(func() {
  policies := []string{
  case12WhitespaceListCreate,
  case12WhitespaceListInform,