Support GKE E2E tests and use (actual) E2E tests to gate releases

.github/workflows/e2e.yaml

@@ -149,10 +149,68 @@ jobs:
         name: tests-report
         path: istio-${{ matrix.kubernetes-version }}-${{ matrix.istio-version }}-tests.xml
 
+  e2e-gke-tests:
+    environment: "gcloud"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes-version: # the GKE setup script ignores the patch version here and uses the latest, but we still include it for consistency with the other E2E jobs
+          - 'v1.24.2'
+    steps:
+    - name: setup golang
+      uses: actions/setup-go@v3
+      with:
+        go-version: '^1.19'
+
+    - name: cache go modules
+      uses: actions/cache@v3
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-build-codegen-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-build-codegen-
+
+    - name: checkout repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - uses: Kong/kong-license@master
+      id: license
+      with:
+        # PULP_PASSWORD secret is set in "Configure ci" environment
+        password: ${{ secrets.PULP_PASSWORD }}
+
+    - name: run ${{ matrix.kubernetes-version }}
+      run: make test.e2e.gke
+      env:
+        GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+        GOOGLE_PROJECT: ${{ secrets.GOOGLE_PROJECT }}
+        GOOGLE_LOCATION: ${{ secrets.GOOGLE_LOCATION }}
+        KONG_CLUSTER_VERSION: ${{ matrix.kubernetes-version }}
+        KONG_LICENSE_DATA: ${{ steps.license.outputs.license }}
+        GOTESTSUM_JUNITFILE: "gke-${{ matrix.kubernetes-version }}-tests.xml"
+
+    - name: upload diagnostics
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: diagnostics-e2e-tests
+        path: /tmp/ktf-diag*
+        if-no-files-found: ignore
+
+    - name: collect test report
+      uses: actions/upload-artifact@v3
+      with:
+        name: tests-report
+        path: gke-${{ matrix.kubernetes-version }}-tests.xml
+
   buildpulse-report:
     environment: "Configure ci"
     needs:
       - "e2e-tests"
+      - "e2e-gke-tests"
       - "istio-tests"
     if: ${{ always() }}
     runs-on: ubuntu-latest

.github/workflows/release.yaml

@@ -206,9 +206,6 @@ jobs:
       matrix:
         kubernetes-version:
           - 'v1.24.3'
-        dbmode:
-          - 'dbless'
-          - 'postgres'
     steps:
       - name: setup golang
         uses: actions/setup-go@v3
@@ -225,7 +222,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Kubernetes ${{ matrix.kubernetes-version }} ${{ matrix.dbmode }} Integration Tests
+      - name: Kubernetes ${{ matrix.kubernetes-version }} E2E Tests
         run: KONG_CLUSTER_VERSION=${{ matrix.kubernetes-version }} make test.integration.${{ matrix.dbmode }}
 
   test-previous-kubernetes:
@@ -238,9 +235,6 @@ jobs:
           - '21'
           - '22'
           - '23'
-        dbmode:
-          - 'dbless'
-          - 'postgres'
     steps:
       - name: setup golang
         uses: actions/setup-go@v3
@@ -257,8 +251,8 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: test ${{ matrix.dbmode }} on GKE v1.${{ matrix.minor }}
-        run: make test.integration.gke
+      - name: E2E on GKE v1.${{ matrix.minor }}
+        run: make test.e2e.gke
         env:
           KUBERNETES_MAJOR_VERSION: 1
           KUBERNETES_MINOR_VERSION: ${{ matrix.minor }}

Makefile

@@ -338,6 +338,20 @@ test.integration.kind:
 	@$(MAKE) _test.integration.cp \
 		CP="kind"
 
+.PHONY: test.e2e.gke
+test.e2e.gke:
+	CLUSTER_NAME="e2e-$(uuidgen)" \
+		KUBERNETES_CLUSTER_NAME="${CLUSTER_NAME}" go run hack/e2e/cluster/deploy/main.go \
+		KONG_TEST_CLUSTER="gke:${CLUSTER_NAME}" \
+		GOFLAGS="-tags=e2e_tests" $(GOTESTSUM) -- $(GOTESTFLAGS) \
+			-race \
+			-run $(E2E_TEST_RUN) \
+			-parallel $(NCPU) \
+			-timeout $(E2E_TEST_TIMEOUT) \
+			./test/e2e/... \
+		go run hack/e2e/cluster/cleanup/main.go ${CLUSTER_NAME} \
+		trap cleanup EXIT SIGINT SIGQUIT
+
 .PHONY: test.e2e
 test.e2e: gotestsum
 	GOFLAGS="-tags=e2e_tests" \

hack/e2e/cluster/deploy/main.go

@@ -8,16 +8,17 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strconv"
+	"strings"
 
+	"github.com/blang/semver/v4"
+	"github.com/google/uuid"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/types/gke"
 )
 
 const (
-	k8sNameVar  = "KUBERNETES_CLUSTER_NAME"
-	k8sMajorVar = "KUBERNETES_MAJOR_VERSION"
-	k8sMinorVar = "KUBERNETES_MINOR_VERSION"
+	k8sNameVar    = "KUBERNETES_CLUSTER_NAME"
+	k8sVersionVar = "KONG_CLUSTER_VERSION"
 )
 
 var (
@@ -28,23 +29,22 @@ var (
 	gkeProject  = os.Getenv(gke.GKEProjectVar)
 	gkeLocation = os.Getenv(gke.GKELocationVar)
 	k8sName     = os.Getenv(k8sNameVar)
-	k8sMajor    = os.Getenv(k8sMajorVar)
-	k8sMinor    = os.Getenv(k8sMinorVar)
+	k8sVersion  = semver.MustParse(strings.TrimPrefix(os.Getenv(k8sVersionVar), "v"))
 )
 
 func main() {
 	mustNotBeEmpty(gke.GKECredsVar, gkeCreds)
 	mustNotBeEmpty(gke.GKEProjectVar, gkeProject)
 	mustNotBeEmpty(gke.GKELocationVar, gkeLocation)
-	mustNotBeEmpty(k8sNameVar, k8sName)
-	mustNotBeEmpty(k8sMajorVar, k8sMajor)
-	mustNotBeEmpty(k8sMinorVar, k8sMinor)
+	mustNotBeEmpty(k8sVersionVar, k8sVersion.String())
+	if k8sName == "" {
+		k8sName = "kic-" + uuid.NewString()
+		fmt.Println("INFO: no cluster name provided, using generated name " + k8sName)
+	}
 
 	fmt.Println("INFO: validating cluster version requirements")
-	major, err := strconv.Atoi(k8sMajor)
-	mustNotError(err)
-	minor, err := strconv.Atoi(k8sMinor)
-	mustNotError(err)
+	major := k8sVersion.Major
+	minor := k8sVersion.Minor
 
 	if len(os.Args) > 1 && os.Args[1] == "cleanup" {
 		fmt.Printf("INFO: cleanup called, deleting GKE cluster %s\n", k8sName)
@@ -57,10 +57,10 @@ func main() {
 
 	fmt.Printf("INFO: configuring the GKE cluster NAME=(%s) VERSION=(v%d.%d) PROJECT=(%s) LOCATION=(%s)\n", k8sName, major, minor, gkeProject, gkeLocation)
 	builder := gke.NewBuilder([]byte(gkeCreds), gkeProject, gkeLocation).WithName(k8sName)
-	builder.WithClusterMinorVersion(uint64(major), uint64(minor))
+	builder.WithClusterMinorVersion(major, minor)
 
 	fmt.Printf("INFO: building cluster %s (this can take some time)\n", builder.Name)
-	cluster, err = builder.Build(ctx)
+	cluster, err := builder.Build(ctx)
 	mustNotError(err)
 
 	fmt.Println("INFO: verifying that the cluster can be communicated with")

test/e2e/all_in_one_test.go

@@ -13,8 +13,6 @@ import (
 
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters"
 	"github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/kong"
-	"github.com/kong/kubernetes-testing-framework/pkg/clusters/addons/metallb"
-	"github.com/kong/kubernetes-testing-framework/pkg/environments"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
@@ -46,13 +44,8 @@ func TestDeployAllInOneDBLESS(t *testing.T) {
 	defer cancel()
 
 	t.Log("building test cluster and environment")
-	addons := []clusters.Addon{}
-	addons = append(addons, metallb.New())
-
-	addons = append(addons, buildImageLoadAddons(t, imageLoad, kongImageLoad)...)
-
-	builder := setBuilderKubernetesVersion(t,
-		environments.NewBuilder().WithAddons(addons...), clusterVersionStr)
+	builder, err := getEnvironmentBuilder(ctx)
+	require.NoError(t, err)
 	env, err := builder.Build(ctx)
 	require.NoError(t, err)
 	defer func() {
@@ -112,13 +105,8 @@ func TestDeployAndUpgradeAllInOneDBLESS(t *testing.T) {
 	defer cancel()
 
 	t.Log("building test cluster and environment")
-	addons := []clusters.Addon{}
-	addons = append(addons, metallb.New())
-
-	addons = append(addons, buildImageLoadAddons(t, imageLoad, kongImageLoad)...)
-
-	builder := setBuilderKubernetesVersion(t,
-		environments.NewBuilder().WithAddons(addons...), clusterVersionStr)
+	builder, err := getEnvironmentBuilder(ctx)
+	require.NoError(t, err)
 
 	env, err := builder.Build(ctx)
 	require.NoError(t, err)
@@ -164,13 +152,8 @@ func TestDeployAllInOneEnterpriseDBLESS(t *testing.T) {
 	defer cancel()
 
 	t.Log("building test cluster and environment")
-	addons := []clusters.Addon{}
-	addons = append(addons, metallb.New())
-
-	addons = append(addons, buildImageLoadAddons(t, imageLoad, kongImageLoad)...)
-
-	builder := setBuilderKubernetesVersion(t,
-		environments.NewBuilder().WithAddons(addons...), clusterVersionStr)
+	builder, err := getEnvironmentBuilder(ctx)
+	require.NoError(t, err)
 	env, err := builder.Build(ctx)
 	require.NoError(t, err)
 
@@ -220,13 +203,8 @@ func TestDeployAllInOnePostgres(t *testing.T) {
 	defer cancel()
 
 	t.Log("building test cluster and environment")
-	addons := []clusters.Addon{}
-	addons = append(addons, metallb.New())
-
-	addons = append(addons, buildImageLoadAddons(t, imageLoad, kongImageLoad)...)
-
-	builder := setBuilderKubernetesVersion(t,
-		environments.NewBuilder().WithAddons(addons...), clusterVersionStr)
+	builder, err := getEnvironmentBuilder(ctx)
+	require.NoError(t, err)
 	env, err := builder.Build(ctx)
 	require.NoError(t, err)
 
@@ -261,13 +239,8 @@ func TestDeployAllInOnePostgresWithMultipleReplicas(t *testing.T) {
 	defer cancel()
 
 	t.Log("building test cluster and environment")
-	addons := []clusters.Addon{}
-	addons = append(addons, metallb.New())
-
-	addons = append(addons, buildImageLoadAddons(t, imageLoad, kongImageLoad)...)
-
-	builder := setBuilderKubernetesVersion(t,
-		environments.NewBuilder().WithAddons(addons...), clusterVersionStr)
+	builder, err := getEnvironmentBuilder(ctx)
+	require.NoError(t, err)
 	env, err := builder.Build(ctx)
 	require.NoError(t, err)
 
@@ -417,13 +390,8 @@ func TestDeployAllInOneEnterprisePostgres(t *testing.T) {
 	defer cancel()
 
 	t.Log("building test cluster and environment")
-	addons := []clusters.Addon{}
-	addons = append(addons, metallb.New())
-
-	addons = append(addons, buildImageLoadAddons(t, imageLoad, kongImageLoad)...)
-
-	builder := setBuilderKubernetesVersion(t,
-		environments.NewBuilder().WithAddons(addons...), clusterVersionStr)
+	builder, err := getEnvironmentBuilder(ctx)
+	require.NoError(t, err)
 	env, err := builder.Build(ctx)
 	require.NoError(t, err)
 	createKongImagePullSecret(ctx, t, env)

test/e2e/environment.go

@@ -0,0 +1,19 @@
+package e2e
+
+import "os"
+
+var (
+	// clusterVersionStr indicates the Kubernetes cluster version to use when
+	// generating a testing environment and allows the caller to provide a specific
+	// version. If no version is provided the default version for the cluster
+	// provisioner in the testing framework will be used.
+	clusterVersionStr = os.Getenv("KONG_CLUSTER_VERSION")
+
+	imageOverride         = os.Getenv("TEST_KONG_CONTROLLER_IMAGE_OVERRIDE")
+	imageLoad             = os.Getenv("TEST_KONG_CONTROLLER_IMAGE_LOAD")
+	kongImageOverride     = os.Getenv("TEST_KONG_IMAGE_OVERRIDE")
+	kongImageLoad         = os.Getenv("TEST_KONG_IMAGE_LOAD")
+	kongImagePullUsername = os.Getenv("TEST_KONG_PULL_USERNAME")
+	kongImagePullPassword = os.Getenv("TEST_KONG_PULL_PASSWORD")
+	existingCluster       = os.Getenv("KONG_TEST_CLUSTER")
+)