e2e: graceful envoy shutdown

go.mod

@@ -4,6 +4,7 @@ go 1.21
 
 require (
 	fortio.org/fortio v1.63.4
+	fortio.org/log v1.12.0
 	github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa
 	github.com/davecgh/go-spew v1.1.1
 	github.com/envoyproxy/go-control-plane v0.12.0
@@ -53,7 +54,6 @@ require (
 
 require (
 	fortio.org/dflag v1.7.0 // indirect
-	fortio.org/log v1.12.0 // indirect
 	fortio.org/sets v1.0.3 // indirect
 	fortio.org/struct2env v0.4.0 // indirect
 	fortio.org/version v1.0.3 // indirect

test/config/gatewayclass.yaml

@@ -62,3 +62,30 @@ spec:
   shutdown:
     drainTimeout: 5s
     minDrainDuration: 1s
+---
+kind: GatewayClass
+apiVersion: gateway.networking.k8s.io/v1
+metadata:
+  name: upgrade
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+  parametersRef:
+    group: gateway.envoyproxy.io
+    kind: EnvoyProxy
+    name: upgrade-config
+    namespace: envoy-gateway-system
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: upgrade-config
+  namespace: envoy-gateway-system
+spec:
+  shutdown:
+    drainTimeout: 15s
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        replicas: 2
+

test/e2e/base/manifests.yaml

@@ -642,3 +642,72 @@ spec:
   - protocol: TCP
     port: 8000
     targetPort: 8000
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gateway-upgrade-infra
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: ha-gateway
+  namespace: gateway-upgrade-infra
+spec:
+  gatewayClassName: upgrade
+  listeners:
+  - allowedRoutes:
+      namespaces:
+        from: Same
+    name: http1
+    port: 80
+    protocol: HTTP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: infra-backend
+  namespace: gateway-upgrade-infra
+spec:
+  selector:
+    app: infra-backend
+  ports:
+  - protocol: TCP
+    port: 8080
+    targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: infra-backend
+  namespace: gateway-upgrade-infra
+  labels:
+    app: infra-backend
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: infra-backend
+  template:
+    metadata:
+      labels:
+        app: infra-backend
+    spec:
+      containers:
+      - name: infra-backend
+        # From https://github.com/kubernetes-sigs/gateway-api/blob/main/conformance/echo-basic/echo-basic.go
+        image: gcr.io/k8s-staging-gateway-api/echo-basic:v20231214-v1.0.0-140-gf544a46e
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: SERVICE_NAME
+          value: infra-backend
+        resources:
+          requests:
+            cpu: 10m

test/e2e/testdata/eg-upgrade.yaml

@@ -1,38 +1,16 @@
-apiVersion: gateway.envoyproxy.io/v1alpha1
-kind: BackendTrafficPolicy
-metadata:
-  name: eg-upgrade-example
-  namespace: gateway-conformance-infra
-spec:
-  targetRef:
-    group: gateway.networking.k8s.io
-    kind: HTTPRoute
-    name: http-backend-eg-upgrade
-    namespace: gateway-conformance-infra
-  circuitBreaker:
-    maxParallelRequests: 10000
-    maxConnections: 10000
-    maxPendingRequests: 10000
-  retry:
-    retryOn:
-      triggers:
-        - connect-failure
-        - reset
-    numRetries: 10
----
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata:
   name: http-backend-eg-upgrade
-  namespace: gateway-conformance-infra
+  namespace: gateway-upgrade-infra
 spec:
   parentRefs:
-    - name: same-namespace
+  - name: ha-gateway
   rules:
-    - matches:
-        - path:
-            type: PathPrefix
-            value: /backend-upgrade
-      backendRefs:
-        - name: infra-backend-v1
-          port: 8080
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /eg-upgrade
+    backendRefs:
+    - name: infra-backend
+      port: 8080

test/e2e/testdata/envoy-shutdown.yaml

@@ -0,0 +1,16 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-envoy-shutdown
+  namespace: gateway-upgrade-infra
+spec:
+  parentRefs:
+  - name: ha-gateway
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /envoy-shutdown
+    backendRefs:
+    - name: infra-backend
+      port: 8080

test/e2e/tests/backend_upgrade.go

@@ -10,21 +10,19 @@ package tests
 
 import (
 	"context"
-	"io"
 	"net/url"
 	"testing"
 	"time"
 
-	"fortio.org/fortio/fhttp"
 	"fortio.org/fortio/periodic"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/gateway-api/conformance/utils/config"
 	"sigs.k8s.io/gateway-api/conformance/utils/http"
 	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
@@ -44,10 +42,15 @@ var BackendUpgradeTest = suite.ConformanceTest{
 			ns := "gateway-conformance-infra"
 			routeNN := types.NamespacedName{Name: "http-backend-upgrade", Namespace: ns}
 			gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
-			dNN := types.NamespacedName{Name: "infra-backend-v1", Namespace: ns}
 			gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)
 			reqURL := url.URL{Scheme: "http", Host: http.CalculateHost(t, gwAddr, "http"), Path: "/backend-upgrade"}
 
+			// get deployment to restart
+			dp, err := getDeploymentByNN(ns, "infra-backend-v1", suite.Client)
+			if err != nil {
+				t.Errorf("Failed to get backend deployment")
+			}
+
 			// can be used to abort the test after deployment restart is complete or failed
 			aborter := periodic.NewAborter()
 			// will contain indication on success or failure of load test
@@ -58,7 +61,7 @@ var BackendUpgradeTest = suite.ConformanceTest{
 			go runLoadAndWait(t, suite.TimeoutConfig, loadSuccess, aborter, reqURL.String())
 
 			t.Log("Restarting deployment")
-			err := restartDeploymentAndWaitForNewPods(t, suite.TimeoutConfig, suite.Client, dNN)
+			err = restartDeploymentAndWaitForNewPods(t, suite.TimeoutConfig, suite.Client, dp)
 
 			t.Log("Stopping load generation and collecting results")
 			aborter.Abort(false) // abort the load either way
@@ -76,74 +79,39 @@ var BackendUpgradeTest = suite.ConformanceTest{
 	},
 }
 
-// runs a load test with options described in opts
-// the done channel is used to notify caller of execution result
-// the execution may end due to an external abort or timeout
-func runLoadAndWait(t *testing.T, timeoutConfig config.TimeoutConfig, done chan bool, aborter *periodic.Aborter, reqURL string) {
-	opts := fhttp.HTTPRunnerOptions{
-		RunnerOptions: periodic.RunnerOptions{
-			QPS: 5000,
-			// allow some overhead time for setting up workers and tearing down after restart
-			Duration:   timeoutConfig.CreateTimeout + timeoutConfig.CreateTimeout/2,
-			NumThreads: 50,
-			Stop:       aborter,
-			Out:        io.Discard,
-		},
-		HTTPOptions: fhttp.HTTPOptions{
-			URL: reqURL,
-		},
-	}
-	res, err := fhttp.RunHTTPTest(&opts)
-	if err != nil {
-		done <- false
-		t.Logf("failed to create load: %v", err)
-	}
-
-	// collect stats
-	okReq := res.RetCodes[200]
-	totalReq := res.DurationHistogram.Count
-	failedReq := totalReq - okReq
-	errorReq := res.ErrorsDurationHistogram.Count
-	timedOut := res.ActualDuration == opts.Duration
-	t.Logf("Backend upgrade completed after %s with %d requests, %d success, %d failures and %d errors", res.ActualDuration, totalReq, okReq, failedReq, errorReq)
+func getDeploymentByNN(namespace, name string, c client.Client) (*appsv1.Deployment, error) {
+	ctx := context.Background()
+	dp := &appsv1.Deployment{}
 
-	if okReq == totalReq && errorReq == 0 && !timedOut {
-		done <- true
-	}
-	done <- false
+	err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, dp)
+	return dp, err
 }
 
-func restartDeploymentAndWaitForNewPods(t *testing.T, timeoutConfig config.TimeoutConfig, c client.Client, dNN types.NamespacedName) error {
+func restartDeploymentAndWaitForNewPods(t *testing.T, timeoutConfig config.TimeoutConfig, c client.Client, dp *appsv1.Deployment) error {
 	t.Helper()
 	const kubeRestartAnnotation = "kubectl.kubernetes.io/restartedAt"
 
 	ctx := context.Background()
-	dp := &appsv1.Deployment{}
-
-	err := c.Get(ctx, dNN, dp)
-	if err != nil {
-		return err
-	}
 
 	if dp.Spec.Template.ObjectMeta.Annotations == nil {
 		dp.Spec.Template.ObjectMeta.Annotations = make(map[string]string)
 	}
 	restartTime := time.Now().Format(time.RFC3339)
 	dp.Spec.Template.ObjectMeta.Annotations[kubeRestartAnnotation] = restartTime
 
-	if err = c.Update(ctx, dp); err != nil {
+	if err := c.Update(ctx, dp); err != nil {
 		return err
 	}
 
 	return wait.PollUntilContextTimeout(ctx, 1*time.Second, timeoutConfig.CreateTimeout, true, func(ctx context.Context) (bool, error) {
 		// wait for replicaset with the same annotation to reach ready status
 		podList := &corev1.PodList{}
 		listOpts := []client.ListOption{
-			client.InNamespace(dNN.Namespace),
-			client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(labels.Set{"app": dNN.Name})},
+			client.InNamespace(dp.Namespace),
+			client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(dp.Spec.Selector.MatchLabels)},
 		}
 
-		err = c.List(ctx, podList, listOpts...)
+		err := c.List(ctx, podList, listOpts...)
 		if err != nil {
 			return false, err
 		}

test/e2e/tests/eg_upgrade.go

@@ -37,15 +37,15 @@ var EGUpgradeTest = suite.ConformanceTest{
 				lastVersionTag = "v0.6.0" // Default version tag if not specified
 			}
 
-			ns := "gateway-conformance-infra"
+			ns := "gateway-upgrade-infra"
 			routeNN := types.NamespacedName{Name: "http-backend-eg-upgrade", Namespace: ns}
-			gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
+			gwNN := types.NamespacedName{Name: "ha-gateway", Namespace: ns}
 			gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)
-			reqURL := url.URL{Scheme: "http", Host: http.CalculateHost(t, gwAddr, "http"), Path: "/backend-upgrade"}
+			reqURL := url.URL{Scheme: "http", Host: http.CalculateHost(t, gwAddr, "http"), Path: "/eg-upgrade"}
 			kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{depNS})
 			expectOkResp := http.ExpectedResponse{
 				Request: http.Request{
-					Path: "/backend-upgrade",
+					Path: "/eg-upgrade",
 				},
 				Response: http.Response{
 					StatusCode: 200,