ref(e2e): organize e2e tests into tiers (#1902)

This change organizes tests into tiers based on priority to keep CI
running reasonably quickly against PRs while still regularly running all
tests. Currently 2 tiers are defined. Tier 1 tests have the highest
priority and run against PRs. Tier 2 tests are run on each merge into
the main branch. More tiers will likely be defined and tests may move
between tiers as the test suite grows.

This change also updates the e2e developer docs with the latest info and
guidance for these changes.

.github/workflows/main.yml

@@ -113,6 +113,9 @@ jobs:
     name: Go test e2e
     runs-on: ubuntu-latest
     needs: build
+    env:
+      CTR_TAG: ${{ github.sha }}
+      CTR_REGISTRY: "localhost:5000"   # unused for kind, but currently required in framework
     steps:
       - name: Checkout
         uses: actions/checkout@v1
@@ -132,13 +135,14 @@ jobs:
         uses: actions/setup-go@v1
         with:
           go-version: 1.15
-      - name: Run E2E tests
-        env:
-          CTR_TAG: ${{ github.sha }}
-          CTR_REGISTRY: "localhost:5000"   # unused for kind, but currently required in framework
-        run: |
-          make docker-build-osm-controller docker-build-init build-osm
-          go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -kindCluster -test.timeout 0 -test.failfast
+      - name: Build test dependencies
+        run: make docker-build-osm-controller docker-build-init build-osm
+      - name: Run PR tests
+        if: ${{ github.event_name == 'pull_request' }}
+        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -kindCluster -test.timeout 0 -test.failfast -ginkgo.focus='\[Tier 1\]'
+      - name: Run tests for push to main
+        if: ${{ github.event_name == 'push' }}
+        run: go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -kindCluster -test.timeout 0 -test.failfast
 
   integration-tresor:
     name: Integration Test with Tresor, SMI traffic policies, and egress disabled

tests/e2e/README.md

@@ -16,6 +16,15 @@ The tests are written using Ginkgo and Gomega so they may also be directly invok
 
 OSM's framework has an init mechanism which will automatically initialize and parse flags and variables from both `env` and `go test flags` if any are passed to the test. The hooks for initialization and cleanup are set at Ginkgo's `BeforeEach` at the top level of test execution (between Ginkgo `Describes`); we henceforth recommend keeping every test in its own `Describe` section, as well as on a separate file for clarity. You can refer to [suite_test](tests/e2e/suite_test.go) for more details about the init process.
 
+Tests are organized by top-level `Describe` blocks into tiers based on priority. A tier's tests will also be run as a part of all the tiers below it.
+
+- Tier 1: run against every PR and should pass before being merged
+- Tier 2: run against every merge into the main branch
+
+**Note**: These tiers and which tests fall into each are likely to change as the test suite grows.
+
+To help organize the tests, custom `Describe` blocks named after the tiers like `DescribeTier1` are provided to help name tests accordingly and should be used in place of Ginkgo's original `Describe` to ensure the right tests are run at the right times in CI.
+
 ## Running the tests
 Running the tests will require a running Kubernetes cluster. If you do not have a Kubernetes cluster to run the tests onto, you can choose to run them using `Kind`, which will make the test framework initialize a cluster on a local accessible docker client.
 

tests/e2e/common.go

@@ -53,6 +53,15 @@ const (
 	osmTest = "osmTest"
 )
 
+func DescribeTierN(tier uint) func(string, func()) bool {
+	return func(name string, body func()) bool {
+		return Describe(fmt.Sprintf("[Tier %d] %s", tier, name), body)
+	}
+}
+
+var DescribeTier1 = DescribeTierN(1)
+var DescribeTier2 = DescribeTierN(2)
+
 // OsmTestData stores common state, variables and flags for the test at hand
 type OsmTestData struct {
 	T GinkgoTInterface // for common test logging

tests/e2e/e2e_certmanager_test.go

@@ -8,7 +8,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("1 Client pod -> 1 Server pod test using cert-manager", func() {
+var _ = DescribeTier2("1 Client pod -> 1 Server pod test using cert-manager", func() {
 	Context("CertManagerSimpleClientServer", func() {
 		const sourceNs = "client"
 		const destNs = "server"

tests/e2e/e2e_deployment_client_server_test.go

@@ -12,7 +12,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var _ = Describe("Test HTTP traffic from N deployment client -> 1 deployment server", func() {
+var _ = DescribeTier1("Test HTTP traffic from N deployment client -> 1 deployment server", func() {
 	Context("DeploymentsClientServer", func() {
 		destApp := "server"
 		sourceAppBaseName := "client"

tests/e2e/e2e_egress_test.go

@@ -8,7 +8,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("HTTP and HTTPS Egress", func() {
+var _ = DescribeTier1("HTTP and HTTPS Egress", func() {
 	Context("Egress", func() {
 		sourceNs := "client"
 

tests/e2e/e2e_fluentbit_test.go

@@ -11,7 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 )
 
-var _ = Describe("Test deployment of Fluent Bit sidecar", func() {
+var _ = DescribeTier2("Test deployment of Fluent Bit sidecar", func() {
 	Context("Fluentbit", func() {
 		It("Deploys a Fluent Bit sidecar only when enabled", func() {
 			// Install OSM with Fluentbit

tests/e2e/e2e_hashivault_test.go

@@ -8,7 +8,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("1 Client pod -> 1 Server pod test using Vault", func() {
+var _ = DescribeTier2("1 Client pod -> 1 Server pod test using Vault", func() {
 	Context("HashivaultSimpleClientServer", func() {
 		sourceNs := "client"
 		destNs := "server"

tests/e2e/e2e_permissive_test.go

@@ -9,7 +9,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("Permissive Traffic Policy Mode", func() {
+var _ = DescribeTier1("Permissive Traffic Policy Mode", func() {
 	Context("PermissiveMode", func() {
 		const sourceNs = "client"
 		const destNs = "server"

tests/e2e/e2e_pod_client_server_test.go

@@ -11,7 +11,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var _ = Describe("Test HTTP traffic from 1 pod client -> 1 pod server", func() {
+var _ = DescribeTier1("Test HTTP traffic from 1 pod client -> 1 pod server", func() {
 	Context("SimpleClientServer", func() {
 		sourceName := "client"
 		destName := "server"

tests/e2e/e2e_trafficsplit_same_sa_test.go

@@ -14,7 +14,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var _ = Describe("Test TrafficSplit where each backend shares the same ServiceAccount", func() {
+var _ = DescribeTier1("Test TrafficSplit where each backend shares the same ServiceAccount", func() {
 	Context("ClientServerTrafficSplitSameSA", func() {
 		const (
 			// to name the header we will use to identify the server that replies

tests/e2e/e2e_trafficsplit_test.go

@@ -13,7 +13,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-var _ = Describe("Test HTTP from N Clients deployments to 1 Server deployment backed with Traffic split test", func() {
+var _ = DescribeTier1("Test HTTP from N Clients deployments to 1 Server deployment backed with Traffic split test", func() {
 	Context("ClientServerTrafficSplit", func() {
 		const (
 			// to name the header we will use to identify the server that replies