-
Notifications
You must be signed in to change notification settings - Fork 324
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
203c9d1
commit 4323b3c
Showing
20 changed files
with
1,321 additions
and
221 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| ```release-note:feature | ||
| Add support for configuring graceful shutdown proxy lifecycle management settings. | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
221 changes: 221 additions & 0 deletions
221
acceptance/tests/connect/connect_proxy_lifecycle_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,221 @@ | ||
| // Copyright (c) HashiCorp, Inc. | ||
| // SPDX-License-Identifier: MPL-2.0 | ||
|
|
||
| package connect | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strconv" | ||
| "strings" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/hashicorp/consul-k8s/acceptance/framework/connhelper" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/consul" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" | ||
| "github.com/hashicorp/consul-k8s/acceptance/framework/logger" | ||
| "github.com/hashicorp/consul/sdk/testutil/retry" | ||
| "github.com/hashicorp/go-version" | ||
| "github.com/stretchr/testify/require" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| type LifecycleShutdownConfig struct { | ||
| secure bool | ||
| helmValues map[string]string | ||
| } | ||
|
|
||
| const ( | ||
| helmDrainListenersKey = "connectInject.sidecarProxy.lifecycle.defaultEnableShutdownDrainListeners" | ||
| helmGracePeriodSecondsKey = "connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds" | ||
| ) | ||
|
|
||
| // Test the endpoints controller cleans up force-killed pods. | ||
| func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { | ||
| cfg := suite.Config() | ||
|
|
||
| ver, err := version.NewVersion("1.2.0") | ||
| require.NoError(t, err) | ||
| if cfg.ConsulDataplaneVersion != nil && cfg.ConsulDataplaneVersion.LessThan(ver) { | ||
| t.Skipf("skipping this test because proxy lifecycle management is not supported in consul-dataplane version %v", cfg.ConsulDataplaneVersion.String()) | ||
| } | ||
|
|
||
| for _, testCfg := range []LifecycleShutdownConfig{ | ||
| {secure: false, helmValues: map[string]string{}}, | ||
| {secure: true, helmValues: map[string]string{}}, | ||
| {secure: false, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "true", | ||
| helmGracePeriodSecondsKey: "30", | ||
| }}, | ||
| {secure: true, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "true", | ||
| helmGracePeriodSecondsKey: "30", | ||
| }}, | ||
| {secure: false, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "false", | ||
| helmGracePeriodSecondsKey: "30", | ||
| }}, | ||
| {secure: true, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "false", | ||
| helmGracePeriodSecondsKey: "30", | ||
| }}, | ||
| {secure: false, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "false", | ||
| helmGracePeriodSecondsKey: "0", | ||
| }}, | ||
| {secure: true, helmValues: map[string]string{ | ||
| helmDrainListenersKey: "false", | ||
| helmGracePeriodSecondsKey: "0", | ||
| }}, | ||
| } { | ||
| // Determine if listeners should be expected to drain inbound connections | ||
| var drainListenersEnabled bool | ||
| val, ok := testCfg.helmValues[helmDrainListenersKey] | ||
| if ok { | ||
| drainListenersEnabled, err = strconv.ParseBool(val) | ||
| require.NoError(t, err) | ||
| } else { | ||
| // Helm defaults | ||
| drainListenersEnabled = true | ||
| } | ||
|
|
||
| // Determine expected shutdown grace period | ||
| var gracePeriodSeconds int64 | ||
| val, ok = testCfg.helmValues[helmGracePeriodSecondsKey] | ||
| if ok { | ||
| gracePeriodSeconds, err = strconv.ParseInt(val, 10, 64) | ||
| require.NoError(t, err) | ||
| } else { | ||
| // Helm defaults | ||
| gracePeriodSeconds = 30 | ||
| } | ||
|
|
||
| name := fmt.Sprintf("secure: %t, drainListeners: %t, gracePeriodSeconds: %d", testCfg.secure, drainListenersEnabled, gracePeriodSeconds) | ||
| t.Run(name, func(t *testing.T) { | ||
| ctx := suite.Environment().DefaultContext(t) | ||
| releaseName := helpers.RandomName() | ||
|
|
||
| connHelper := connhelper.ConnectHelper{ | ||
| ClusterKind: consul.Helm, | ||
| Secure: testCfg.secure, | ||
| ReleaseName: releaseName, | ||
| Ctx: ctx, | ||
| Cfg: cfg, | ||
| HelmValues: testCfg.helmValues, | ||
| } | ||
|
|
||
| connHelper.Setup(t) | ||
| connHelper.Install(t) | ||
| connHelper.DeployClientAndServer(t) | ||
|
|
||
| // TODO: should this move into connhelper.DeployClientAndServer? | ||
| logger.Log(t, "waiting for static-client and static-server to be registered with Consul") | ||
| retry.Run(t, func(r *retry.R) { | ||
| for _, name := range []string{ | ||
| "static-client", | ||
| "static-client-sidecar-proxy", | ||
| "static-server", | ||
| "static-server-sidecar-proxy", | ||
| } { | ||
| logger.Logf(t, "checking for %s service in Consul catalog", name) | ||
| instances, _, err := connHelper.ConsulClient.Catalog().Service(name, "", nil) | ||
| r.Check(err) | ||
|
|
||
| if len(instances) != 1 { | ||
| r.Errorf("expected 1 instance of %s", name) | ||
| } | ||
| } | ||
| }) | ||
|
|
||
| if testCfg.secure { | ||
| connHelper.TestConnectionFailureWithoutIntention(t) | ||
| connHelper.CreateIntention(t) | ||
| } | ||
|
|
||
| connHelper.TestConnectionSuccess(t) | ||
|
|
||
| // Get static-client pod name | ||
| // TODO: is this necessary? | ||
| ns := ctx.KubectlOptions(t).Namespace | ||
| pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List( | ||
| context.Background(), | ||
| metav1.ListOptions{ | ||
| LabelSelector: "app=static-client", | ||
| }, | ||
| ) | ||
| require.NoError(t, err) | ||
| require.Len(t, pods.Items, 1) | ||
| clientPodName := pods.Items[0].Name | ||
|
|
||
| var terminationGracePeriod int64 = 60 | ||
| logger.Logf(t, "killing the %q pod with %dseconds termination grace period", clientPodName, terminationGracePeriod) | ||
| err = ctx.KubernetesClient(t).CoreV1().Pods(ns).Delete(context.Background(), clientPodName, metav1.DeleteOptions{GracePeriodSeconds: &terminationGracePeriod}) | ||
| require.NoError(t, err) | ||
|
|
||
| // Exec into terminating pod, not just any static-client pod | ||
| args := []string{"exec", clientPodName, "-c", connhelper.StaticClientName, "--", "curl", "-vvvsSf"} | ||
|
|
||
| if cfg.EnableTransparentProxy { | ||
| args = append(args, "http://static-server") | ||
| } else { | ||
| args = append(args, "http://localhost:1234") | ||
| } | ||
|
|
||
| if gracePeriodSeconds > 0 { | ||
| // Ensure outbound requests are still successful during grace | ||
| // period. | ||
| retry.RunWith(&retry.Timer{Timeout: time.Duration(gracePeriodSeconds) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { | ||
| output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) | ||
| require.NoError(r, err) | ||
| require.Condition(r, func() bool { | ||
| exists := false | ||
| if strings.Contains(output, "curl: (7) Failed to connect") { | ||
| exists = true | ||
| } | ||
| return !exists | ||
| }) | ||
| }) | ||
|
|
||
| // TODO: If listener draining is enabled, ensure inbound | ||
| // requests are rejected during grace period. | ||
| // connHelper.TestConnectionSuccess(t) | ||
|
|
||
| // TODO: Test that outbound requests start failing after | ||
| // shutdown grace period has elapsed but before pod terminates. | ||
| } else { | ||
| // Ensure outbound requests fail because proxy has terminated | ||
| retry.RunWith(&retry.Timer{Timeout: time.Duration(terminationGracePeriod) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { | ||
| output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) | ||
| require.Error(r, err) | ||
| require.Condition(r, func() bool { | ||
| exists := false | ||
| if strings.Contains(output, "curl: (7) Failed to connect") { | ||
| exists = true | ||
| } | ||
| return exists | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| logger.Log(t, "ensuring pod is deregistered after termination") | ||
| retry.Run(t, func(r *retry.R) { | ||
| for _, name := range []string{ | ||
| "static-client", | ||
| "static-client-sidecar-proxy", | ||
| } { | ||
| logger.Logf(t, "checking for %s service in Consul catalog", name) | ||
| instances, _, err := connHelper.ConsulClient.Catalog().Service(name, "", nil) | ||
| r.Check(err) | ||
|
|
||
| for _, instance := range instances { | ||
| if strings.Contains(instance.ServiceID, clientPodName) { | ||
| r.Errorf("%s is still registered", instance.ServiceID) | ||
| } | ||
| } | ||
| } | ||
| }) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.