[API Gateway] Add external consul servers test

acceptance/framework/consul/cli_cluster.go

@@ -200,9 +200,14 @@ func (c *CLICluster) Destroy(t *testing.T) {
 	require.NoError(t, err)
 }
 
-func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) (*api.Client, string) {
+func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool, release ...string) (*api.Client, string) {
 	t.Helper()
 
+	releaseName := c.releaseName
+	if len(release) > 0 {
+		releaseName = release[0]
+	}
+
 	namespace := c.kubectlOptions.Namespace
 	config := api.DefaultConfig()
 	localPort := terratestk8s.GetAvailablePort(t)
@@ -222,13 +227,13 @@ func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) (*api.Client,
 		// In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary.
 		// Instead, we provide a replication token that serves the role of the bootstrap token.
 
-		aclSecretName := fmt.Sprintf("%s-consul-bootstrap-acl-token", c.releaseName)
+		aclSecretName := fmt.Sprintf("%s-consul-bootstrap-acl-token", releaseName)
 		if c.releaseName == CLIReleaseName {
 			aclSecretName = "consul-bootstrap-acl-token"
 		}
 		aclSecret, err := c.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), aclSecretName, metav1.GetOptions{})
 		if err != nil && errors.IsNotFound(err) {
-			federationSecret := fmt.Sprintf("%s-consul-federation", c.releaseName)
+			federationSecret := fmt.Sprintf("%s-consul-federation", releaseName)
 			if c.releaseName == CLIReleaseName {
 				federationSecret = "consul-federation"
 			}
@@ -242,8 +247,8 @@ func (c *CLICluster) SetupConsulClient(t *testing.T, secure bool) (*api.Client,
 		}
 	}
 
-	serverPod := fmt.Sprintf("%s-consul-server-0", c.releaseName)
-	if c.releaseName == CLIReleaseName {
+	serverPod := fmt.Sprintf("%s-consul-server-0", releaseName)
+	if releaseName == CLIReleaseName {
 		serverPod = "consul-server-0"
 	}
 	tunnel := terratestk8s.NewTunnelWithLogger(

acceptance/framework/consul/cluster.go

@@ -12,7 +12,7 @@ import (
 // Cluster represents a consul cluster object.
 type Cluster interface {
 	// SetupConsulClient returns a new Consul client.
-	SetupConsulClient(t *testing.T, secure bool) (*api.Client, string)
+	SetupConsulClient(t *testing.T, secure bool, release ...string) (*api.Client, string)
 
 	// Create creates a new Consul Cluster.
 	Create(t *testing.T)

acceptance/framework/consul/helm_cluster.go

@@ -362,14 +362,23 @@ func (h *HelmCluster) Upgrade(t *testing.T, helmValues map[string]string) {
 	k8s.WaitForAllPodsToBeReady(t, h.kubernetesClient, h.helmOptions.KubectlOptions.Namespace, fmt.Sprintf("release=%s", h.releaseName))
 }
 
-func (h *HelmCluster) CreatePortForwardTunnel(t *testing.T, remotePort int) string {
-	serverPod := fmt.Sprintf("%s-consul-server-0", h.releaseName)
+func (h *HelmCluster) CreatePortForwardTunnel(t *testing.T, remotePort int, release ...string) string {
+	releaseName := h.releaseName
+	if len(release) > 0 {
+		releaseName = release[0]
+	}
+	serverPod := fmt.Sprintf("%s-consul-server-0", releaseName)
 	return portforward.CreateTunnelToResourcePort(t, serverPod, remotePort, h.helmOptions.KubectlOptions, h.logger)
 }
 
-func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) (client *api.Client, configAddress string) {
+func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool, release ...string) (client *api.Client, configAddress string) {
 	t.Helper()
 
+	releaseName := h.releaseName
+	if len(release) > 0 {
+		releaseName = release[0]
+	}
+
 	namespace := h.helmOptions.KubectlOptions.Namespace
 	config := api.DefaultConfig()
 	remotePort := 8500 // use non-secure by default
@@ -392,9 +401,9 @@ func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) (client *api.
 				// and will try to read the replication token from the federation secret.
 				// In secondary servers, we don't create a bootstrap token since ACLs are only bootstrapped in the primary.
 				// Instead, we provide a replication token that serves the role of the bootstrap token.
-				aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), h.releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{})
+				aclSecret, err := h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), releaseName+"-consul-bootstrap-acl-token", metav1.GetOptions{})
 				if err != nil && errors.IsNotFound(err) {
-					federationSecret := fmt.Sprintf("%s-consul-federation", h.releaseName)
+					federationSecret := fmt.Sprintf("%s-consul-federation", releaseName)
 					aclSecret, err = h.kubernetesClient.CoreV1().Secrets(namespace).Get(context.Background(), federationSecret, metav1.GetOptions{})
 					require.NoError(r, err)
 					config.Token = string(aclSecret.Data["replicationToken"])
@@ -408,7 +417,7 @@ func (h *HelmCluster) SetupConsulClient(t *testing.T, secure bool) (client *api.
 		}
 	}
 
-	config.Address = h.CreatePortForwardTunnel(t, remotePort)
+	config.Address = h.CreatePortForwardTunnel(t, remotePort, release...)
 	consulClient, err := api.NewClient(config)
 	require.NoError(t, err)
 

acceptance/tests/api-gateway/api_gateway_external_servers_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package apigateway
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/sdk/testutil/retry"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/types"
+	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+)
+
+// TestAPIGateway_ExternalServers tests that connect works when using external servers.
+// It sets up an external Consul server in the same cluster but a different Helm installation
+// and then treats this server as external.
+func TestAPIGateway_ExternalServers(t *testing.T) {
+	cfg := suite.Config()
+	ctx := suite.Environment().DefaultContext(t)
+
+	serverHelmValues := map[string]string{
+		"global.acls.manageSystemACLs": "true",
+		"global.tls.enabled":           "true",
+
+		// Don't install injector, controller and cni on this cluster so that it's not installed twice.
+		"connectInject.enabled":     "false",
+		"connectInject.cni.enabled": "false",
+	}
+	serverReleaseName := helpers.RandomName()
+	consulServerCluster := consul.NewHelmCluster(t, serverHelmValues, ctx, cfg, serverReleaseName)
+
+	consulServerCluster.Create(t)
+
+	helmValues := map[string]string{
+		"server.enabled":                        "false",
+		"global.acls.manageSystemACLs":          "true",
+		"global.tls.enabled":                    "true",
+		"connectInject.enabled":                 "true",
+		"externalServers.enabled":               "true",
+		"externalServers.hosts[0]":              fmt.Sprintf("%s-consul-server", serverReleaseName),
+		"externalServers.httpsPort":             "8501",
+		"global.tls.caCert.secretName":          fmt.Sprintf("%s-consul-ca-cert", serverReleaseName),
+		"global.tls.caCert.secretKey":           "tls.crt",
+		"global.acls.bootstrapToken.secretName": fmt.Sprintf("%s-consul-bootstrap-acl-token", serverReleaseName),
+		"global.acls.bootstrapToken.secretKey":  "token",
+	}
+
+	releaseName := helpers.RandomName()
+	consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName)
+	consulCluster.SkipCheckForPreviousInstallations = true
+
+	consulCluster.Create(t)
+
+	logger.Log(t, "creating static-server and static-client deployments")
+	k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject")
+	k8s.DeployKustomize(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-inject")
+
+	// Override the default proxy config settings for this test
+	consulClient, _ := consulCluster.SetupConsulClient(t, true, serverReleaseName)
+	logger.Log(t, "have consul client")
+	_, _, err := consulClient.ConfigEntries().Set(&api.ProxyConfigEntry{
+		Kind: api.ProxyDefaults,
+		Name: api.ProxyConfigGlobal,
+		Config: map[string]interface{}{
+			"protocol": "http",
+		},
+	}, nil)
+	require.NoError(t, err)
+	logger.Log(t, "set consul config entry")
+
+	logger.Log(t, "creating api-gateway resources")
+	out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway")
+	require.NoError(t, err, out)
+	helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() {
+		// Ignore errors here because if the test ran as expected
+		// the custom resources will have been deleted.
+		k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "delete", "-k", "../fixtures/bases/api-gateway")
+	})
+
+	logger.Log(t, "patching route to target server")
+	k8s.RunKubectl(t, ctx.KubectlOptions(t), "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"name":"static-server","port":80}]}]}}`, "--type=merge")
+
+	// Grab a kubernetes client so that we can verify binding
+	// behavior prior to issuing requests through the gateway.
+	k8sClient := ctx.ControllerRuntimeClient(t)
+
+	// On startup, the controller can take upwards of 1m to perform
+	// leader election so we may need to wait a long time for
+	// the reconcile loop to run (hence a ~1m timeout here).
+	var gatewayAddress string
+	retryCheck(t, 30, func(r *retry.R) {
+		var gateway gwv1beta1.Gateway
+		err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "gateway", Namespace: "default"}, &gateway)
+		require.NoError(r, err)
+
+		// check that we have an address to use
+		require.Len(r, gateway.Status.Addresses, 1)
+		// now we know we have an address, set it so we can use it
+		gatewayAddress = gateway.Status.Addresses[0].Value
+	})
+
+	k8sOptions := ctx.KubectlOptions(t)
+	targetAddress := fmt.Sprintf("http://%s/", gatewayAddress)
+
+	// check that intentions keep our connection from happening
+	k8s.CheckStaticServerHTTPConnectionFailing(t, k8sOptions, StaticClientName, targetAddress)
+
+	// Now we create the allow intention.
+	_, _, err = consulClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{
+		Kind: api.ServiceIntentions,
+		Name: "static-server",
+		Sources: []*api.SourceIntention{
+			{
+				Name:   "gateway",
+				Action: api.IntentionActionAllow,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	// Test that we can make a call to the api gateway
+	// via the static-client pod. It should route to the static-server pod.
+	logger.Log(t, "trying calls to api gateway")
+	k8s.CheckStaticServerConnectionSuccessful(t, k8sOptions, StaticClientName, targetAddress)
+}