ref(e2e): divide tests into buckets (openservicemesh#1917)

This change adds the capability to run e2e tests in parallel in CI.
Tests are now organized into buckets, where each bucket runs in parallel
and tests within each bucket run sequentially. How e2e tests are defined
at the top-level has been refactored to include a new parameter
identifying in which bucket a test should run.

Tests were not organized into buckets initially based on any heuristic,
so rebalancing may be necessary. Adding more buckets is also simple.

.github/workflows/main.yml

@@ -113,9 +113,12 @@ jobs:
     name: Go test e2e
     runs-on: ubuntu-latest
     needs: build
+    strategy:
+      matrix:
+        bucket: [1, 2]
     env:
       CTR_TAG: ${{ github.sha }}
-      CTR_REGISTRY: "localhost:5000"   # unused for kind, but currently required in framework
+      CTR_REGISTRY: "localhost:5000" # unused for kind, but currently required in framework
     steps:
       - name: Checkout
         uses: actions/checkout@v1
@@ -139,10 +142,10 @@ jobs:
         run: make docker-build-osm-controller docker-build-init build-osm
       - name: Run PR tests
         if: ${{ github.event_name == 'pull_request' }}
-        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -installType=KindCluster -test.timeout 0 -test.failfast -ginkgo.failFast -ginkgo.focus='\[Tier 1\]'
+        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -installType=KindCluster -test.timeout 0 -test.failfast -ginkgo.failFast -ginkgo.focus='\[Tier 1\]\[Bucket ${{ matrix.bucket }}\]'
       - name: Run tests for push to main
         if: ${{ github.event_name == 'push' }}
-        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -installType=KindCluster -test.timeout 0 -test.failfast -ginkgo.failFast
+        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -installType=KindCluster -test.timeout 0 -test.failfast -ginkgo.failFast -ginkgo.focus='\[Bucket ${{ matrix.bucket }}\]'
 
   integration-tresor:
     name: Integration Test with Tresor, SMI traffic policies, and egress disabled

tests/e2e/README.md

@@ -21,9 +21,11 @@ Tests are organized by top-level `Describe` blocks into tiers based on priority.
 - Tier 1: run against every PR and should pass before being merged
 - Tier 2: run against every merge into the main branch
 
-**Note**: These tiers and which tests fall into each are likely to change as the test suite grows.
+Independent of tiers, tests are also organized into buckets. Each bucket runs in parallel, and individual tests in the bucket run sequentially.
 
-To help organize the tests, custom `Describe` blocks named after the tiers like `DescribeTier1` are provided to help name tests accordingly and should be used in place of Ginkgo's original `Describe` to ensure the right tests are run at the right times in CI.
+**Note**: These tiers and buckets and which tests fall into each are likely to change as the test suite grows.
+
+To help organize the tests, a custom `Describe` block named `OSMDescribe` is provided which accepts an additional struct parameter which contains fields for test metadata like tier and bucket. `OSMDescribe` will construct a well-formatted name including the test metadata which can be used in CI to run tests accordingly. Ginkgo's original `Describe` should not be used directly at the top-level and `OSMDescribe` should be used instead.
 
 ## Running the tests
 Running the tests will require a running Kubernetes cluster. If you do not have a Kubernetes cluster to run the tests onto, you can choose to run them using `Kind`, which will make the test framework initialize a cluster on a local accessible docker client.

tests/e2e/common.go

@@ -67,14 +67,22 @@ var (
 	defaultDeployFluentbit = false
 )
 
-func DescribeTierN(tier uint) func(string, func()) bool {
-	return func(name string, body func()) bool {
-		return Describe(fmt.Sprintf("[Tier %d] %s", tier, name), body)
-	}
+type OSMDescribeInfo struct {
+	// tier represents the priority of the test. Lower value indicates higher priority.
+	tier int
+
+	// bucket indicates in which test bucket the test will run in for CI. Each
+	// bucket is run in parallel while tests in the same bucket run sequentially.
+	bucket int
 }
 
-var DescribeTier1 = DescribeTierN(1)
-var DescribeTier2 = DescribeTierN(2)
+func (o OSMDescribeInfo) String() string {
+	return fmt.Sprintf("[Tier %d][Bucket %d]", o.tier, o.bucket)
+}
+
+func OSMDescribe(name string, opts OSMDescribeInfo, body func()) bool {
+	return Describe(opts.String()+" "+name, body)
+}
 
 // InstallType defines several OSM test deployment scenarios
 type InstallType string

tests/e2e/e2e_certmanager_test.go

@@ -8,103 +8,108 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = DescribeTier2("1 Client pod -> 1 Server pod test using cert-manager", func() {
-	Context("CertManagerSimpleClientServer", func() {
-		const sourceNs = "client"
-		const destNs = "server"
-		var ns []string = []string{sourceNs, destNs}
-
-		It("Tests HTTP traffic for client pod -> server pod", func() {
-			// Install OSM
-			installOpts := td.GetOSMInstallOpts()
-			installOpts.certManager = "cert-manager"
-			Expect(td.InstallOSM(installOpts)).To(Succeed())
-			Expect(td.WaitForPodsRunningReady(td.osmNamespace, 60*time.Second, 4)).To(Succeed())
-
-			// Create Test NS
-			for _, n := range ns {
-				Expect(td.CreateNs(n, nil)).To(Succeed())
-				Expect(td.AddNsToMesh(true, n)).To(Succeed())
-			}
-
-			// Get simple pod definitions for the HTTP server
-			svcAccDef, podDef, svcDef := td.SimplePodApp(
-				SimplePodAppDef{
-					name:      "server",
-					namespace: destNs,
-					image:     "kennethreitz/httpbin",
-					ports:     []int{80},
-				})
+var _ = OSMDescribe("1 Client pod -> 1 Server pod test using cert-manager",
+	OSMDescribeInfo{
+		tier:   2,
+		bucket: 2,
+	},
+	func() {
+		Context("CertManagerSimpleClientServer", func() {
+			const sourceNs = "client"
+			const destNs = "server"
+			var ns []string = []string{sourceNs, destNs}
+
+			It("Tests HTTP traffic for client pod -> server pod", func() {
+				// Install OSM
+				installOpts := td.GetOSMInstallOpts()
+				installOpts.certManager = "cert-manager"
+				Expect(td.InstallOSM(installOpts)).To(Succeed())
+				Expect(td.WaitForPodsRunningReady(td.osmNamespace, 60*time.Second, 4)).To(Succeed())
+
+				// Create Test NS
+				for _, n := range ns {
+					Expect(td.CreateNs(n, nil)).To(Succeed())
+					Expect(td.AddNsToMesh(true, n)).To(Succeed())
+				}
 
-			_, err := td.CreateServiceAccount(destNs, &svcAccDef)
-			Expect(err).NotTo(HaveOccurred())
-			dstPod, err := td.CreatePod(destNs, podDef)
-			Expect(err).NotTo(HaveOccurred())
-			_, err = td.CreateService(destNs, svcDef)
-			Expect(err).NotTo(HaveOccurred())
-
-			// Expect it to be up and running in it's receiver namespace
-			Expect(td.WaitForPodsRunningReady(destNs, 60*time.Second, 1)).To(Succeed())
-
-			// Get simple Pod definitions for the client
-			svcAccDef, podDef, svcDef = td.SimplePodApp(SimplePodAppDef{
-				name:      "client",
-				namespace: sourceNs,
-				command:   []string{"/bin/bash", "-c", "--"},
-				args:      []string{"while true; do sleep 30; done;"},
-				image:     "songrgg/alpine-debug",
-				ports:     []int{80},
-			})
+				// Get simple pod definitions for the HTTP server
+				svcAccDef, podDef, svcDef := td.SimplePodApp(
+					SimplePodAppDef{
+						name:      "server",
+						namespace: destNs,
+						image:     "kennethreitz/httpbin",
+						ports:     []int{80},
+					})
 
-			_, err = td.CreateServiceAccount(sourceNs, &svcAccDef)
-			Expect(err).NotTo(HaveOccurred())
-			srcPod, err := td.CreatePod(sourceNs, podDef)
-			Expect(err).NotTo(HaveOccurred())
-			_, err = td.CreateService(sourceNs, svcDef)
-			Expect(err).NotTo(HaveOccurred())
+				_, err := td.CreateServiceAccount(destNs, &svcAccDef)
+				Expect(err).NotTo(HaveOccurred())
+				dstPod, err := td.CreatePod(destNs, podDef)
+				Expect(err).NotTo(HaveOccurred())
+				_, err = td.CreateService(destNs, svcDef)
+				Expect(err).NotTo(HaveOccurred())
+
+				// Expect it to be up and running in it's receiver namespace
+				Expect(td.WaitForPodsRunningReady(destNs, 60*time.Second, 1)).To(Succeed())
+
+				// Get simple Pod definitions for the client
+				svcAccDef, podDef, svcDef = td.SimplePodApp(SimplePodAppDef{
+					name:      "client",
+					namespace: sourceNs,
+					command:   []string{"/bin/bash", "-c", "--"},
+					args:      []string{"while true; do sleep 30; done;"},
+					image:     "songrgg/alpine-debug",
+					ports:     []int{80},
+				})
 
-			// Expect it to be up and running in it's receiver namespace
-			Expect(td.WaitForPodsRunningReady(sourceNs, 60*time.Second, 1)).To(Succeed())
+				_, err = td.CreateServiceAccount(sourceNs, &svcAccDef)
+				Expect(err).NotTo(HaveOccurred())
+				srcPod, err := td.CreatePod(sourceNs, podDef)
+				Expect(err).NotTo(HaveOccurred())
+				_, err = td.CreateService(sourceNs, svcDef)
+				Expect(err).NotTo(HaveOccurred())
 
-			// Deploy allow rule client->server
-			httpRG, trafficTarget := td.CreateSimpleAllowPolicy(
-				SimpleAllowPolicy{
-					RouteGroupName:    "routes",
-					TrafficTargetName: "test-target",
+				// Expect it to be up and running in it's receiver namespace
+				Expect(td.WaitForPodsRunningReady(sourceNs, 60*time.Second, 1)).To(Succeed())
 
-					SourceNamespace:      sourceNs,
-					SourceSVCAccountName: "client",
+				// Deploy allow rule client->server
+				httpRG, trafficTarget := td.CreateSimpleAllowPolicy(
+					SimpleAllowPolicy{
+						RouteGroupName:    "routes",
+						TrafficTargetName: "test-target",
 
-					DestinationNamespace:      destNs,
-					DestinationSvcAccountName: "server",
-				})
+						SourceNamespace:      sourceNs,
+						SourceSVCAccountName: "client",
 
-			// Configs have to be put into a monitored NS, and osm-system can't be by cli
-			_, err = td.CreateHTTPRouteGroup(sourceNs, httpRG)
-			Expect(err).NotTo(HaveOccurred())
-			_, err = td.CreateTrafficTarget(sourceNs, trafficTarget)
-			Expect(err).NotTo(HaveOccurred())
-
-			// All ready. Expect client to reach server
-			// Need to get the pod though.
-			cond := td.WaitForRepeatedSuccess(func() bool {
-				result :=
-					td.HTTPRequest(HTTPRequestDef{
-						SourceNs:        srcPod.Namespace,
-						SourcePod:       srcPod.Name,
-						SourceContainer: "client",
-
-						Destination: fmt.Sprintf("%s.%s", dstPod.Name, dstPod.Namespace),
+						DestinationNamespace:      destNs,
+						DestinationSvcAccountName: "server",
 					})
 
-				if result.Err != nil || result.StatusCode != 200 {
-					td.T.Logf("> REST req failed (status: %d) %v", result.StatusCode, result.Err)
-					return false
-				}
-				td.T.Logf("> REST req succeeded: %d", result.StatusCode)
-				return true
-			}, 5 /*consecutive success threshold*/, 90*time.Second /*timeout*/)
-			Expect(cond).To(BeTrue())
+				// Configs have to be put into a monitored NS, and osm-system can't be by cli
+				_, err = td.CreateHTTPRouteGroup(sourceNs, httpRG)
+				Expect(err).NotTo(HaveOccurred())
+				_, err = td.CreateTrafficTarget(sourceNs, trafficTarget)
+				Expect(err).NotTo(HaveOccurred())
+
+				// All ready. Expect client to reach server
+				// Need to get the pod though.
+				cond := td.WaitForRepeatedSuccess(func() bool {
+					result :=
+						td.HTTPRequest(HTTPRequestDef{
+							SourceNs:        srcPod.Namespace,
+							SourcePod:       srcPod.Name,
+							SourceContainer: "client",
+
+							Destination: fmt.Sprintf("%s.%s", dstPod.Name, dstPod.Namespace),
+						})
+
+					if result.Err != nil || result.StatusCode != 200 {
+						td.T.Logf("> REST req failed (status: %d) %v", result.StatusCode, result.Err)
+						return false
+					}
+					td.T.Logf("> REST req succeeded: %d", result.StatusCode)
+					return true
+				}, 5 /*consecutive success threshold*/, 90*time.Second /*timeout*/)
+				Expect(cond).To(BeTrue())
+			})
 		})
 	})
-})