diff --git a/acceptance/tests/fixtures/cases/api-gateways/dc1-to-dc2-resolver/kustomization.yaml b/acceptance/tests/fixtures/cases/api-gateways/dc1-to-dc2-resolver/kustomization.yaml new file mode 100644 index 0000000000..cdbcd688c0 --- /dev/null +++ b/acceptance/tests/fixtures/cases/api-gateways/dc1-to-dc2-resolver/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - serviceresolver.yaml diff --git a/acceptance/tests/fixtures/cases/api-gateways/dc1-to-dc2-resolver/serviceresolver.yaml b/acceptance/tests/fixtures/cases/api-gateways/dc1-to-dc2-resolver/serviceresolver.yaml new file mode 100644 index 0000000000..ca009754b4 --- /dev/null +++ b/acceptance/tests/fixtures/cases/api-gateways/dc1-to-dc2-resolver/serviceresolver.yaml @@ -0,0 +1,11 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceResolver +metadata: + name: static-server +spec: + redirect: + service: static-server + datacenter: dc2 \ No newline at end of file diff --git a/acceptance/tests/fixtures/cases/api-gateways/dc2-to-dc1-resolver/kustomization.yaml b/acceptance/tests/fixtures/cases/api-gateways/dc2-to-dc1-resolver/kustomization.yaml new file mode 100644 index 0000000000..cdbcd688c0 --- /dev/null +++ b/acceptance/tests/fixtures/cases/api-gateways/dc2-to-dc1-resolver/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - serviceresolver.yaml diff --git a/acceptance/tests/fixtures/cases/api-gateways/dc2-to-dc1-resolver/serviceresolver.yaml b/acceptance/tests/fixtures/cases/api-gateways/dc2-to-dc1-resolver/serviceresolver.yaml new file mode 100644 index 0000000000..af8cdb72ed --- /dev/null +++ b/acceptance/tests/fixtures/cases/api-gateways/dc2-to-dc1-resolver/serviceresolver.yaml @@ -0,0 +1,11 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceResolver +metadata: + name: static-server +spec: + redirect: + service: static-server + datacenter: dc1 \ No newline at end of file diff --git a/acceptance/tests/wan-federation/wan_federation_gateway_test.go b/acceptance/tests/wan-federation/wan_federation_gateway_test.go new file mode 100644 index 0000000000..0ef48b9920 --- /dev/null +++ b/acceptance/tests/wan-federation/wan_federation_gateway_test.go @@ -0,0 +1,241 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package wanfederation + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/serf/testutil/retry" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +func TestWANFederation_Gateway(t *testing.T) { + env := suite.Environment() + cfg := suite.Config() + + if cfg.UseKind { + // the only way this test can currently run on kind, at least on a Mac, is via leveraging MetalLB, which + // isn't in CI, so we just skip for now. + t.Skipf("skipping wan federation tests as they currently fail on Kind even though they work on other clouds.") + } + + primaryContext := env.DefaultContext(t) + secondaryContext := env.Context(t, environment.SecondaryContextName) + + primaryHelmValues := map[string]string{ + "global.datacenter": "dc1", + + "global.tls.enabled": "true", + "global.tls.httpsOnly": "true", + + "global.federation.enabled": "true", + "global.federation.createFederationSecret": "true", + + "global.acls.manageSystemACLs": "true", + "global.acls.createReplicationToken": "true", + + "connectInject.enabled": "true", + "connectInject.replicas": "1", + + "meshGateway.enabled": "true", + "meshGateway.replicas": "1", + } + + releaseName := helpers.RandomName() + + // Install the primary consul cluster in the default kubernetes context + primaryConsulCluster := consul.NewHelmCluster(t, primaryHelmValues, primaryContext, cfg, releaseName) + primaryConsulCluster.Create(t) + + // Get the federation secret from the primary cluster and apply it to secondary cluster + federationSecretName := fmt.Sprintf("%s-consul-federation", releaseName) + logger.Logf(t, "retrieving federation secret %s from the primary cluster and applying to the secondary", federationSecretName) + federationSecret, err := primaryContext.KubernetesClient(t).CoreV1().Secrets(primaryContext.KubectlOptions(t).Namespace).Get(context.Background(), federationSecretName, metav1.GetOptions{}) + require.NoError(t, err) + federationSecret.ResourceVersion = "" + _, err = secondaryContext.KubernetesClient(t).CoreV1().Secrets(secondaryContext.KubectlOptions(t).Namespace).Create(context.Background(), federationSecret, metav1.CreateOptions{}) + require.NoError(t, err) + + var k8sAuthMethodHost string + // When running on kind, the kube API address in kubeconfig will have a localhost address + // which will not work from inside the container. That's why we need to use the endpoints address instead + // which will point the node IP. + if cfg.UseKind { + // The Kubernetes AuthMethod host is read from the endpoints for the Kubernetes service. + kubernetesEndpoint, err := secondaryContext.KubernetesClient(t).CoreV1().Endpoints("default").Get(context.Background(), "kubernetes", metav1.GetOptions{}) + require.NoError(t, err) + k8sAuthMethodHost = fmt.Sprintf("%s:%d", kubernetesEndpoint.Subsets[0].Addresses[0].IP, kubernetesEndpoint.Subsets[0].Ports[0].Port) + } else { + k8sAuthMethodHost = k8s.KubernetesAPIServerHostFromOptions(t, secondaryContext.KubectlOptions(t)) + } + + // Create secondary cluster + secondaryHelmValues := map[string]string{ + "global.datacenter": "dc2", + + "global.tls.enabled": "true", + "global.tls.httpsOnly": "false", + "global.acls.manageSystemACLs": "true", + "global.tls.caCert.secretName": federationSecretName, + "global.tls.caCert.secretKey": "caCert", + "global.tls.caKey.secretName": federationSecretName, + "global.tls.caKey.secretKey": "caKey", + + "global.federation.enabled": "true", + + "server.extraVolumes[0].type": "secret", + "server.extraVolumes[0].name": federationSecretName, + "server.extraVolumes[0].load": "true", + "server.extraVolumes[0].items[0].key": "serverConfigJSON", + "server.extraVolumes[0].items[0].path": "config.json", + + "connectInject.enabled": "true", + "connectInject.replicas": "1", + + "meshGateway.enabled": "true", + "meshGateway.replicas": "1", + + "global.acls.replicationToken.secretName": federationSecretName, + "global.acls.replicationToken.secretKey": "replicationToken", + "global.federation.k8sAuthMethodHost": k8sAuthMethodHost, + "global.federation.primaryDatacenter": "dc1", + } + + // Install the secondary consul cluster in the secondary kubernetes context + secondaryConsulCluster := consul.NewHelmCluster(t, secondaryHelmValues, secondaryContext, cfg, releaseName) + secondaryConsulCluster.Create(t) + + primaryClient, _ := primaryConsulCluster.SetupConsulClient(t, true) + secondaryClient, _ := secondaryConsulCluster.SetupConsulClient(t, true) + + // Verify federation between servers + logger.Log(t, "verifying federation was successful") + helpers.VerifyFederation(t, primaryClient, secondaryClient, releaseName, true) + + // Create a ProxyDefaults resource to configure services to use the mesh + // gateways. + logger.Log(t, "creating proxy-defaults config in dc1") + kustomizeDir := "../fixtures/cases/api-gateways/mesh" + k8s.KubectlApplyK(t, primaryContext.KubectlOptions(t), kustomizeDir) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + k8s.KubectlDeleteK(t, primaryContext.KubectlOptions(t), kustomizeDir) + }) + + // these clients are just there so we can exec in and curl on them. + logger.Log(t, "creating static-client in dc1") + k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") + + logger.Log(t, "creating static-client in dc2") + k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-client-multi-dc") + + t.Run("from primary to secondary", func(t *testing.T) { + logger.Log(t, "creating static-server in dc2") + k8s.DeployKustomize(t, secondaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + + logger.Log(t, "creating api-gateway resources in dc1") + out, err := k8s.RunKubectlAndGetOutputE(t, primaryContext.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") + require.NoError(t, err, out) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + // Ignore errors here because if the test ran as expected + // the custom resources will have been deleted. + k8s.RunKubectlAndGetOutputE(t, primaryContext.KubectlOptions(t), "delete", "-k", "../fixtures/bases/api-gateway") + }) + + // create a service resolver for doing cross-dc redirects. + k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), "../fixtures/cases/api-gateways/dc1-to-dc2-resolver") + helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), "../fixtures/cases/api-gateways/dc1-to-dc2-resolver") + }) + + // patching the route to target a MeshService since we don't have the corresponding Kubernetes service in this + // cluster. + k8s.RunKubectl(t, primaryContext.KubectlOptions(t), "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"group":"consul.hashicorp.com","kind":"MeshService","name":"mesh-service","port":80}]}]}}`, "--type=merge") + + checkConnectivity(t, primaryContext, primaryClient) + }) + + t.Run("from secondary to primary", func(t *testing.T) { + // Check that we can connect services over the mesh gateways + logger.Log(t, "creating static-server in dc1") + k8s.DeployKustomize(t, primaryContext.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.DebugDirectory, "../fixtures/cases/static-server-inject") + + logger.Log(t, "creating api-gateway resources in dc2") + out, err := k8s.RunKubectlAndGetOutputE(t, secondaryContext.KubectlOptions(t), "apply", "-k", "../fixtures/bases/api-gateway") + require.NoError(t, err, out) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + // Ignore errors here because if the test ran as expected + // the custom resources will have been deleted. + k8s.RunKubectlAndGetOutputE(t, secondaryContext.KubectlOptions(t), "delete", "-k", "../fixtures/bases/api-gateway") + }) + + // create a service resolver for doing cross-dc redirects. + k8s.KubectlApplyK(t, secondaryContext.KubectlOptions(t), "../fixtures/cases/api-gateways/dc2-to-dc1-resolver") + helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + k8s.KubectlDeleteK(t, secondaryContext.KubectlOptions(t), "../fixtures/cases/api-gateways/dc2-to-dc1-resolver") + }) + + // patching the route to target a MeshService since we don't have the corresponding Kubernetes service in this + // cluster. + k8s.RunKubectl(t, secondaryContext.KubectlOptions(t), "patch", "httproute", "http-route", "-p", `{"spec":{"rules":[{"backendRefs":[{"group":"consul.hashicorp.com","kind":"MeshService","name":"mesh-service","port":80}]}]}}`, "--type=merge") + + checkConnectivity(t, secondaryContext, primaryClient) + }) +} + +func checkConnectivity(t *testing.T, ctx environment.TestContext, client *api.Client) { + k8sClient := ctx.ControllerRuntimeClient(t) + + // On startup, the controller can take upwards of 1m to perform + // leader election so we may need to wait a long time for + // the reconcile loop to run (hence the 1m timeout here). + var gatewayAddress string + counter := &retry.Counter{Count: 600, Wait: 2 * time.Second} + retry.RunWith(counter, t, func(r *retry.R) { + var gateway gwv1beta1.Gateway + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "gateway", Namespace: "default"}, &gateway) + require.NoError(r, err) + + // check that we have an address to use + require.Len(r, gateway.Status.Addresses, 1) + // now we know we have an address, set it so we can use it + gatewayAddress = gateway.Status.Addresses[0].Value + }) + + targetAddress := fmt.Sprintf("http://%s/", gatewayAddress) + + logger.Log(t, "checking that the connection is not successful because there's no intention") + k8s.CheckStaticServerHTTPConnectionFailing(t, ctx.KubectlOptions(t), StaticClientName, targetAddress) + + logger.Log(t, "creating intention") + _, _, err := client.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ + Kind: api.ServiceIntentions, + Name: "static-server", + Sources: []*api.SourceIntention{ + { + Name: "gateway", + Action: api.IntentionActionAllow, + }, + }, + }, nil) + require.NoError(t, err) + defer func() { + _, err := client.ConfigEntries().Delete(api.ServiceIntentions, "static-server", &api.WriteOptions{}) + require.NoError(t, err) + }() + + logger.Log(t, "checking that connection is successful") + k8s.CheckStaticServerConnectionSuccessful(t, ctx.KubectlOptions(t), StaticClientName, targetAddress) +} diff --git a/control-plane/api-gateway/cache/consul.go b/control-plane/api-gateway/cache/consul.go index e47df71522..7737e80d57 100644 --- a/control-plane/api-gateway/cache/consul.go +++ b/control-plane/api-gateway/cache/consul.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "github.com/hashicorp/consul-k8s/control-plane/api-gateway/common" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/hashicorp/consul-k8s/control-plane/namespaces" "github.com/hashicorp/consul/api" @@ -60,6 +61,7 @@ type Config struct { ConsulClientConfig *consul.Config ConsulServerConnMgr consul.ServerConnectionManager NamespacesEnabled bool + Datacenter string CrossNamespaceACLPolicy string Logger logr.Logger } @@ -83,6 +85,8 @@ type Cache struct { synced chan struct{} kinds []string + + datacenter string } func New(config Config) *Cache { @@ -104,6 +108,7 @@ func New(config Config) *Cache { synced: make(chan struct{}, len(Kinds)), logger: config.Logger, crossNamespaceACLPolicy: config.CrossNamespaceACLPolicy, + datacenter: config.Datacenter, } } @@ -216,6 +221,19 @@ func (c *Cache) updateAndNotify(ctx context.Context, once *sync.Once, kind strin cache := common.NewReferenceMap() for _, entry := range entries { + meta := entry.GetMeta() + if meta[constants.MetaKeyKubeName] == "" || meta[constants.MetaKeyDatacenter] != c.datacenter { + // Don't process things that don't belong to us. The main reason + // for this is so that we don't garbage collect config entries that + // are either user-created or that another controller running in a + // federated datacenter creates. While we still allow for competing controllers + // syncing/overriding each other due to conflicting Kubernetes objects in + // two federated clusters (which is what the rest of the controllers also allow + // for), we don't want to delete a config entry just because we don't have + // its corresponding Kubernetes object if we know it belongs to another datacenter. + continue + } + cache.Set(common.EntryToReference(entry), entry) } @@ -336,6 +354,7 @@ func (c *Cache) ensureRole(client *api.Client) (string, error) { } aclRoleName := "managed-gateway-acl-role" + aclRole, _, err := client.ACL().RoleReadByName(aclRoleName, &api.QueryOptions{}) if err != nil { return "", err diff --git a/control-plane/api-gateway/cache/consul_test.go b/control-plane/api-gateway/cache/consul_test.go index 555e13b6c2..59570e532f 100644 --- a/control-plane/api-gateway/cache/consul_test.go +++ b/control-plane/api-gateway/cache/consul_test.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "github.com/hashicorp/consul-k8s/control-plane/api-gateway/common" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/hashicorp/consul-k8s/control-plane/helper/test" "github.com/hashicorp/consul/api" @@ -119,8 +120,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, })[api.HTTPRoute], args: args{ @@ -203,8 +206,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, })[api.HTTPRoute], }, @@ -291,8 +296,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, &api.HTTPRouteConfigEntry{ Kind: api.HTTPRoute, @@ -372,8 +379,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, })[api.HTTPRoute], args: args{ @@ -456,8 +465,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, })[api.HTTPRoute], }, @@ -540,8 +551,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, }, }, @@ -626,8 +639,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, })[api.HTTPRoute], args: args{ @@ -710,8 +725,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, &api.HTTPRouteConfigEntry{ Kind: api.HTTPRoute, @@ -791,8 +808,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, })[api.HTTPRoute], }, @@ -875,8 +894,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, }, }, @@ -962,8 +983,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, })[api.HTTPRoute], args: args{ @@ -1047,8 +1070,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, })[api.HTTPRoute], }, @@ -1132,8 +1157,10 @@ func Test_resourceCache_diff(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, }, }, }, @@ -1378,8 +1405,10 @@ func TestCache_Write(t *testing.T) { }, }, Hostnames: []string{"hostname.com"}, - Meta: map[string]string{}, - Status: api.ConfigEntryStatus{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, + Status: api.ConfigEntryStatus{}, } err = c.Write(context.Background(), entry) @@ -1410,18 +1439,24 @@ func TestCache_Get(t *testing.T) { want: &api.APIGatewayConfigEntry{ Kind: api.APIGateway, Name: "api-gw", - Meta: map[string]string{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, }, cache: loadedReferenceMaps([]api.ConfigEntry{ &api.APIGatewayConfigEntry{ Kind: api.APIGateway, Name: "api-gw", - Meta: map[string]string{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, }, &api.APIGatewayConfigEntry{ Kind: api.APIGateway, Name: "api-gw-2", - Meta: map[string]string{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, }, }), }, @@ -1438,12 +1473,16 @@ func TestCache_Get(t *testing.T) { &api.APIGatewayConfigEntry{ Kind: api.APIGateway, Name: "api-gw", - Meta: map[string]string{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, }, &api.APIGatewayConfigEntry{ Kind: api.APIGateway, Name: "api-gw-2", - Meta: map[string]string{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, }, }), }, @@ -1460,7 +1499,9 @@ func TestCache_Get(t *testing.T) { &api.HTTPRouteConfigEntry{ Kind: api.HTTPRoute, Name: "route", - Meta: map[string]string{}, + Meta: map[string]string{ + constants.MetaKeyKubeName: "name", + }, }, }), }, @@ -1766,7 +1807,8 @@ func setupHTTPRoutes() (*api.HTTPRouteConfigEntry, *api.HTTPRouteConfigEntry) { }, Hostnames: []string{"hostname.com"}, Meta: map[string]string{ - "metaKey": "metaVal", + "metaKey": "metaVal", + constants.MetaKeyKubeName: "name", }, Status: api.ConfigEntryStatus{}, } @@ -1849,7 +1891,8 @@ func setupHTTPRoutes() (*api.HTTPRouteConfigEntry, *api.HTTPRouteConfigEntry) { }, Hostnames: []string{"hostname.com"}, Meta: map[string]string{ - "metakey": "meta val", + "metakey": "meta val", + constants.MetaKeyKubeName: "name", }, } return routeOne, routeTwo @@ -1860,7 +1903,8 @@ func setupGateway() *api.APIGatewayConfigEntry { Kind: api.APIGateway, Name: "api-gw", Meta: map[string]string{ - "metakey": "meta val", + "metakey": "meta val", + constants.MetaKeyKubeName: "name", }, Listeners: []api.APIGatewayListener{ { @@ -1891,7 +1935,8 @@ func setupTCPRoute() *api.TCPRouteConfigEntry { }, }, Meta: map[string]string{ - "metakey": "meta val", + "metakey": "meta val", + constants.MetaKeyKubeName: "name", }, Status: api.ConfigEntryStatus{}, } @@ -1904,7 +1949,8 @@ func setupInlineCertificate() *api.InlineCertificateConfigEntry { Certificate: "cert", PrivateKey: "super secret", Meta: map[string]string{ - "metaKey": "meta val", + "metaKey": "meta val", + constants.MetaKeyKubeName: "name", }, } } diff --git a/control-plane/api-gateway/common/helm_config.go b/control-plane/api-gateway/common/helm_config.go index 2a6cc8211b..f0d4dc7988 100644 --- a/control-plane/api-gateway/common/helm_config.go +++ b/control-plane/api-gateway/common/helm_config.go @@ -3,7 +3,12 @@ package common -import "time" +import ( + "strings" + "time" +) + +const componentAuthMethod = "k8s-component-auth-method" // HelmConfig is the configuration of gateways that comes in from the user's Helm values. type HelmConfig struct { @@ -33,3 +38,20 @@ type ConsulConfig struct { HTTPPort int APITimeout time.Duration } + +func (h HelmConfig) Normalize() HelmConfig { + if h.AuthMethod != "" { + // strip off any DC naming off the back in case we're + // in a secondary DC, in which case our auth method is + // going to be a globally scoped auth method, and we want + // to target the locally scoped one, which is the auth + // method without the DC-specific suffix. + tokens := strings.Split(h.AuthMethod, componentAuthMethod) + if len(tokens) != 2 { + // skip the normalization if we can't do it. + return h + } + h.AuthMethod = tokens[0] + componentAuthMethod + } + return h +} diff --git a/control-plane/api-gateway/common/translation.go b/control-plane/api-gateway/common/translation.go index 5e577470d6..8644b64716 100644 --- a/control-plane/api-gateway/common/translation.go +++ b/control-plane/api-gateway/common/translation.go @@ -23,6 +23,7 @@ type ResourceTranslator struct { EnableK8sMirroring bool MirroringPrefix string ConsulPartition string + Datacenter string } func (t ResourceTranslator) NonNormalizedConfigEntryReference(kind string, id types.NamespacedName) api.ResourceReference { @@ -65,10 +66,10 @@ func (t ResourceTranslator) ToAPIGateway(gateway gwv1beta1.Gateway, resources *R Name: gateway.Name, Namespace: namespace, Partition: t.ConsulPartition, - Meta: map[string]string{ + Meta: t.addDatacenterToMeta(map[string]string{ constants.MetaKeyKubeNS: gateway.Namespace, constants.MetaKeyKubeName: gateway.Name, - }, + }), Listeners: listeners, } } @@ -128,10 +129,10 @@ func (t ResourceTranslator) ToHTTPRoute(route gwv1beta1.HTTPRoute, resources *Re Name: route.Name, Namespace: namespace, Partition: t.ConsulPartition, - Meta: map[string]string{ + Meta: t.addDatacenterToMeta(map[string]string{ constants.MetaKeyKubeNS: route.Namespace, constants.MetaKeyKubeName: route.Name, - }, + }), Hostnames: hostnames, Rules: rules, } @@ -292,10 +293,10 @@ func (t ResourceTranslator) ToTCPRoute(route gwv1alpha2.TCPRoute, resources *Res Name: route.Name, Namespace: namespace, Partition: t.ConsulPartition, - Meta: map[string]string{ + Meta: t.addDatacenterToMeta(map[string]string{ constants.MetaKeyKubeNS: route.Namespace, constants.MetaKeyKubeName: route.Name, - }, + }), Services: services, } } @@ -346,10 +347,10 @@ func (t ResourceTranslator) ToInlineCertificate(secret corev1.Secret) (*api.Inli Partition: t.ConsulPartition, Certificate: strings.TrimSpace(certificate), PrivateKey: strings.TrimSpace(privateKey), - Meta: map[string]string{ + Meta: t.addDatacenterToMeta(map[string]string{ constants.MetaKeyKubeNS: secret.Namespace, constants.MetaKeyKubeName: secret.Name, - }, + }), }, nil } @@ -361,3 +362,11 @@ func EntryToNamespacedName(entry api.ConfigEntry) types.NamespacedName { Name: meta[constants.MetaKeyKubeName], } } + +func (t ResourceTranslator) addDatacenterToMeta(meta map[string]string) map[string]string { + if t.Datacenter == "" { + return meta + } + meta[constants.MetaKeyDatacenter] = t.Datacenter + return meta +} diff --git a/control-plane/api-gateway/controllers/gateway_controller.go b/control-plane/api-gateway/controllers/gateway_controller.go index ec8c2e9af0..664bb98d3c 100644 --- a/control-plane/api-gateway/controllers/gateway_controller.go +++ b/control-plane/api-gateway/controllers/gateway_controller.go @@ -46,6 +46,7 @@ type GatewayControllerConfig struct { NamespacesEnabled bool CrossNamespaceACLPolicy string Partition string + Datacenter string AllowK8sNamespacesSet mapset.Set DenyK8sNamespacesSet mapset.Set } @@ -317,6 +318,7 @@ func SetupGatewayControllerWithManager(ctx context.Context, mgr ctrl.Manager, co ConsulClientConfig: config.ConsulClientConfig, ConsulServerConnMgr: config.ConsulServerConnMgr, NamespacesEnabled: config.NamespacesEnabled, + Datacenter: config.Datacenter, CrossNamespaceACLPolicy: config.CrossNamespaceACLPolicy, Logger: mgr.GetLogger(), } @@ -332,13 +334,14 @@ func SetupGatewayControllerWithManager(ctx context.Context, mgr ctrl.Manager, co r := &GatewayController{ Client: mgr.GetClient(), Log: mgr.GetLogger(), - HelmConfig: config.HelmConfig, + HelmConfig: config.HelmConfig.Normalize(), Translator: common.ResourceTranslator{ EnableConsulNamespaces: config.HelmConfig.EnableNamespaces, ConsulDestNamespace: config.HelmConfig.ConsulDestinationNamespace, EnableK8sMirroring: config.HelmConfig.EnableNamespaceMirroring, MirroringPrefix: config.HelmConfig.NamespaceMirroringPrefix, ConsulPartition: config.HelmConfig.ConsulPartition, + Datacenter: config.Datacenter, }, denyK8sNamespacesSet: config.DenyK8sNamespacesSet, allowK8sNamespacesSet: config.AllowK8sNamespacesSet, diff --git a/control-plane/api/v1alpha1/serviceresolver_types.go b/control-plane/api/v1alpha1/serviceresolver_types.go index 75aa44f6b9..d00821275d 100644 --- a/control-plane/api/v1alpha1/serviceresolver_types.go +++ b/control-plane/api/v1alpha1/serviceresolver_types.go @@ -425,7 +425,7 @@ func (in *ServiceResolverRedirect) validate(path *field.Path, consulMeta common. "service resolver redirect cannot be empty")) } - if consulMeta.Partition != "default" && in.Datacenter != "" { + if consulMeta.Partition != "default" && consulMeta.Partition != "" && in.Datacenter != "" { errs = append(errs, field.Invalid(path.Child("datacenter"), in.Datacenter, "cross-datacenter redirect is only supported in the default partition")) } diff --git a/control-plane/connect-inject/constants/constants.go b/control-plane/connect-inject/constants/constants.go index e576816a6a..05e5d1a3bc 100644 --- a/control-plane/connect-inject/constants/constants.go +++ b/control-plane/connect-inject/constants/constants.go @@ -20,6 +20,9 @@ const ( // MetaKeyKubeName is the meta key name for Kubernetes object name used for a Consul object. MetaKeyKubeName = "k8s-name" + // MetaKeyDatacenter is the datacenter that this object was registered from. + MetaKeyDatacenter = "datacenter" + // MetaKeyKubeServiceName is the meta key name for Kubernetes service name used for the Consul services. MetaKeyKubeServiceName = "k8s-service-name" diff --git a/control-plane/subcommand/inject-connect/command.go b/control-plane/subcommand/inject-connect/command.go index a745cbd915..2379948b35 100644 --- a/control-plane/subcommand/inject-connect/command.go +++ b/control-plane/subcommand/inject-connect/command.go @@ -521,7 +521,9 @@ func (c *Command) Run(args []string) int { NamespacesEnabled: c.flagEnableNamespaces, CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy, Partition: c.consul.Partition, + Datacenter: c.consul.Datacenter, }) + if err != nil { setupLog.Error(err, "unable to create controller", "controller", "Gateway") return 1