tests: delete migrations job before upgrade in e2e test (#4100)

To prevent failures caused by not being allowed to update jobs' spec when 
upgrading with kubectl apply -f to a newer version of all-in-one manifests, 
delete the job just before the upgrade.

test/e2e/features_test.go

@@ -71,7 +71,6 @@ nodes:
 `
 	validationWebhookName = "kong-validation-webhook"
 	kongNamespace         = "kong"
-	admissionScriptPath   = "../../hack/deploy-admission-controller.sh"
 )
 
 // openssl req -new -x509 -nodes -newkey ec:<(openssl ecparam -name secp384r1) -keyout cert.key -out cert.crt -days 3650 -subj '/CN=first.example/'

test/e2e/helpers_test.go

@@ -81,6 +81,9 @@ const (
 
 	// proxyContainerName is the name of the proxy container in all manifests variants.
 	proxyContainerName = "proxy"
+
+	// migrationsJobName is the name of the migrations job in postgres manifests variant.
+	migrationsJobName = "kong-migrations"
 )
 
 // setupE2ETest builds a testing environment for the E2E test. It also sets up the environment's teardown and test

test/e2e/upgrade_test.go

@@ -3,10 +3,14 @@
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"testing"
 
+	"github.com/kong/kubernetes-testing-framework/pkg/environments"
+	"github.com/samber/lo"
 	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/kong/kubernetes-ingress-controller/v2/test/internal/helpers"
 )
@@ -25,16 +29,33 @@ const (
 func TestDeployAndUpgradeAllInOneDBLESS(t *testing.T) {
 	fromManifestURL := fmt.Sprintf(dblessURLTemplate, upgradeTestFromTag)
 	toManifestPath := dblessPath
-	testManifestsUpgrade(t, fromManifestURL, toManifestPath)
+	testManifestsUpgrade(t, fromManifestURL, toManifestPath, nil)
 }
 
 func TestDeployAndUpgradeAllInOnePostgres(t *testing.T) {
 	fromManifestURL := fmt.Sprintf(postgresURLTemplate, upgradeTestFromTag)
 	toManifestPath := postgresPath
-	testManifestsUpgrade(t, fromManifestURL, toManifestPath)
+
+	// Injecting a beforeUpgradeHook to delete the old migrations job before the upgrade. This is necessary because it's
+	// not allowed to modify the existing job's spec.
+	beforeUpgradeHook := func(ctx context.Context, t *testing.T, env environments.Environment) {
+		err := env.Cluster().Client().BatchV1().Jobs(namespace).Delete(ctx, migrationsJobName, metav1.DeleteOptions{
+			PropagationPolicy: lo.ToPtr(metav1.DeletePropagationBackground),
+		})
+		require.NoError(t, err, "failed to delete old migrations job before upgrade")
+	}
+	testManifestsUpgrade(t, fromManifestURL, toManifestPath, beforeUpgradeHook)
 }
 
-func testManifestsUpgrade(t *testing.T, fromManifestURL string, toManifestPath string) {
+// beforeUpgradeFn is a function that is run before the upgrade to clean up any resources that may interfere with the upgrade.
+type beforeUpgradeFn func(ctx context.Context, t *testing.T, env environments.Environment)
+
+func testManifestsUpgrade(
+	t *testing.T,
+	fromManifestURL string,
+	toManifestPath string,
+	beforeUpgradeHook beforeUpgradeFn,
+) {
 	t.Parallel()
 
 	httpClient := helpers.RetryableHTTPClient(helpers.DefaultHTTPClient())
@@ -52,6 +73,11 @@ func testManifestsUpgrade(t *testing.T, fromManifestURL string, toManifestPath s
 	deployIngressWithEchoBackends(ctx, t, env, numberOfEchoBackends)
 	verifyIngressWithEchoBackends(ctx, t, env, numberOfEchoBackends)
 
+	if beforeUpgradeHook != nil {
+		t.Log("running before upgrade hook")
+		beforeUpgradeHook(ctx, t, env)
+	}
+
 	t.Logf("deploying target version of kong manifests: %s", toManifestPath)
 	manifest := getTestManifest(t, toManifestPath)
 	deployKong(ctx, t, env, manifest)