diff --git a/acceptance/framework/consul/cli_cluster.go b/acceptance/framework/consul/cli_cluster.go index a9b1dbe74f..ba4cfc93ab 100644 --- a/acceptance/framework/consul/cli_cluster.go +++ b/acceptance/framework/consul/cli_cluster.go @@ -200,9 +200,14 @@ func (c *CLICluster) Destroy(t *testing.T) { require.NoError(t, err) } -func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) (*api.Client, string) { +func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool, release ...string) (*api.Client, string) { t.Helper() + releaseName := c.releaseName + if len(release) > 0 { + releaseName = release[0] + } + namespace := c.kubectlOptions.Namespace config := api.DefaultConfig() localPort := terratestk8s.GetAvailablePort(t) @@ -222,13 +227,13 @@ func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) (*api.Client, // In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary. // Instead, we provide a replication token that serves the role of the bootstrap token. - aclSecretName := fmt.Sprintf("%s-consul-bootstrap-acl-token", c.releaseName) + aclSecretName := fmt.Sprintf("%s-consul-bootstrap-acl-token", releaseName) if c.releaseName == CLIReleaseName { aclSecretName = "consul-bootstrap-acl-token" } aclSecret, err := c.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), aclSecretName, metav1.GetOptions{}) if err != nil && errors.IsNotFound(err) { - federationSecret := fmt.Sprintf("%s-consul-federation", c.releaseName) + federationSecret := fmt.Sprintf("%s-consul-federation", releaseName) if c.releaseName == CLIReleaseName { federationSecret = "consul-federation" } @@ -242,8 +247,8 @@ func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) (*api.Client, } } - serverPod := fmt.Sprintf("%s-consul-server-0", c.releaseName) - if c.releaseName == CLIReleaseName { + serverPod := fmt.Sprintf("%s-consul-server-0", releaseName) + if releaseName == CLIReleaseName { serverPod = "consul-server-0" } tunnel := terratestk8s.NewTunnelWithLogger( diff --git a/acceptance/framework/consul/cluster.go b/acceptance/framework/consul/cluster.go index 734f07f36e..1b9a543245 100644 --- a/acceptance/framework/consul/cluster.go +++ b/acceptance/framework/consul/cluster.go @@ -12,7 +12,7 @@ import ( // Cluster represents a consul cluster object. type Cluster interface { // SetupConsulClient returns a new Consul client. - SetupConsulClient(t *testing.T, secure bool) (*api.Client, string) + SetupConsulClient(t *testing.T, secure bool, release ...string) (*api.Client, string) // Create creates a new Consul Cluster. Create(t *testing.T) diff --git a/acceptance/framework/consul/helm_cluster.go b/acceptance/framework/consul/helm_cluster.go index 613b69da91..0c84229e65 100644 --- a/acceptance/framework/consul/helm_cluster.go +++ b/acceptance/framework/consul/helm_cluster.go @@ -362,14 +362,23 @@ func (h *HelmCluster) Upgrade(t *testing.T, helmValues map[string]string) { k8s.WaitForAllPodsToBeReady(t, h.kubernetesClient, h.helmOptions.KubectlOptions.Namespace, fmt.Sprintf("release=%s", h.releaseName)) } -func (h *HelmCluster) CreatePortForwardTunnel(t *testing.T, remotePort int) string { - serverPod := fmt.Sprintf("%s-consul-server-0", h.releaseName) +func (h *HelmCluster) CreatePortForwardTunnel(t *testing.T, remotePort int, release ...string) string { + releaseName := h.releaseName + if len(release) > 0 { + releaseName = release[0] + } + serverPod := fmt.Sprintf("%s-consul-server-0", releaseName) return portforward.CreateTunnelToResourcePort(t, serverPod, remotePort, h.helmOptions.KubectlOptions, h.logger) } -func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) (client *api.Client, configAddress string) { +func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool, release ...string) (client *api.Client, configAddress string) { t.Helper() + releaseName := h.releaseName + if len(release) > 0 { + releaseName = release[0] + } + namespace := h.helmOptions.KubectlOptions.Namespace config := api.DefaultConfig() remotePort := 8500 // use non-secure by default @@ -392,9 +401,9 @@ func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) (client *api. // and will try to read the replication token from the federation secret. // In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary. // Instead, we provide a replication token that serves the role of the bootstrap token. - aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), h.releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{}) + aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{}) if err != nil && errors.IsNotFound(err) { - federationSecret := fmt.Sprintf("%s-consul-federation", h.releaseName) + federationSecret := fmt.Sprintf("%s-consul-federation", releaseName) aclSecret, err = h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), federationSecret, metav1.GetOptions{}) require.NoError(r, err) config.Token = string(aclSecret.Data["replicationToken"]) @@ -408,7 +417,7 @@ func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) (client *api. } } - config.Address = h.CreatePortForwardTunnel(t, remotePort) + config.Address = h.CreatePortForwardTunnel(t, remotePort, release...) consulClient, err := api.NewClient(config) require.NoError(t, err) diff --git a/acceptance/tests/api-gateway/api_gateway_external_servers_test.go b/acceptance/tests/api-gateway/api_gateway_external_servers_test.go new file mode 100644 index 0000000000..c0fa8bbcca --- /dev/null +++ b/acceptance/tests/api-gateway/api_gateway_external_servers_test.go @@ -0,0 +1,133 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package apigateway + +import ( + "context" + "fmt" + "testing" + + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// TestAPIGateway_ExternalServers tests that connect works when using external servers. +// It sets up an external Consul server in the same cluster but a different Helm installation +// and then treats this server as external. +func TestAPIGateway_ExternalServers(t *testing.T) { + cfg := suite.Config() + ctx := suite.Environment().DefaultContext(t) + + serverHelmValues := map[string]string{ + "global.acls.manageSystemACLs": "true", + "global.tls.enabled": "true", + + // Don't install injector, controller and cni on this cluster so that it's not installed twice. + "connectInject.enabled": "false", + "connectInject.cni.enabled": "false", + } + serverReleaseName := helpers.RandomName() + consulServerCluster := consul.NewHelmCluster(t, serverHelmValues, ctx, cfg, serverReleaseName) + + consulServerCluster.Create(t) + + helmValues := map[string]string{ + "server.enabled": "false", + "global.acls.manageSystemACLs": "true", + "global.tls.enabled": "true", + "connectInject.enabled": "true", + "externalServers.enabled": "true", + "externalServers.hosts[0]": fmt.Sprintf("%s-consul-server", serverReleaseName), + "externalServers.httpsPort": "8501", + "global.tls.caCert.secretName": fmt.Sprintf("%s-consul-ca-cert", serverReleaseName), + "global.tls.caCert.secretKey": "tls.crt", + "global.acls.bootstrapToken.secretName": fmt.Sprintf("%s-consul-bootstrap-acl-token", serverReleaseName), + "global.acls.bootstrapToken.secretKey": "token", + } + + releaseName := helpers.RandomName() + consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName) + consulCluster.SkipCheckForPreviousInstallations = true + + consulCluster.Create(t) + + logger.Log(t, "creating static-server and static-client deployments") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject") + + // Override the default proxy config settings for this test + consulClient, _ := consulCluster.SetupConsulClient(t, true, serverReleaseName) + logger.Log(t, "have consul client") + _, _, err := consulClient.ConfigEntries().Set(&api.ProxyConfigEntry{ + Kind: api.ProxyDefaults, + Name: api.ProxyConfigGlobal, + Config: map[string]interface{}{ + "protocol": "http", + }, + }, nil) + require.NoError(t, err) + logger.Log(t, "set consul config entry") + + logger.Log(t, "creating api-gateway resources") + out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") + require.NoError(t, err, out) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + // Ignore errors here because if the test ran as expected + // the custom resources will have been deleted. + k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-k", "../fixtures/bases/api-gateway") + }) + + logger.Log(t, "patching route to target server") + k8s.RunKubectl(t, ctx.KubectlOptions(t), "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"name":"static-server","port":80}]}]}}`, "--type=merge") + + // Grab a kubernetes client so that we can verify binding + // behavior prior to issuing requests through the gateway. + k8sClient := ctx.ControllerRuntimeClient(t) + + // On startup, the controller can take upwards of 1m to perform + // leader election so we may need to wait a long time for + // the reconcile loop to run (hence a ~1m timeout here). + var gatewayAddress string + retryCheck(t, 30, func(r *retry.R) { + var gateway gwv1beta1.Gateway + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "gateway", Namespace: "default"}, &gateway) + require.NoError(r, err) + + // check that we have an address to use + require.Len(r, gateway.Status.Addresses, 1) + // now we know we have an address, set it so we can use it + gatewayAddress = gateway.Status.Addresses[0].Value + }) + + k8sOptions := ctx.KubectlOptions(t) + targetAddress := fmt.Sprintf("http://%s/", gatewayAddress) + + // check that intentions keep our connection from happening + k8s.CheckStaticServerHTTPConnectionFailing(t, k8sOptions, StaticClientName, targetAddress) + + // Now we create the allow intention. + _, _, err = consulClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ + Kind: api.ServiceIntentions, + Name: "static-server", + Sources: []*api.SourceIntention{ + { + Name: "gateway", + Action: api.IntentionActionAllow, + }, + }, + }, nil) + require.NoError(t, err) + + // Test that we can make a call to the api gateway + // via the static-client pod. It should route to the static-server pod. + logger.Log(t, "trying calls to api gateway") + k8s.CheckStaticServerConnectionSuccessful(t, k8sOptions, StaticClientName, targetAddress) +}