From 78a00f2e1c66589aa0a298c30574a461a324952b Mon Sep 17 00:00:00 2001 From: Paul Banks Date: Tue, 28 Sep 2021 12:25:48 +0100 Subject: [PATCH 1/4] Add support for enabling connect-based ingress TLS per listener. --- agent/proxycfg/ingress_gateway.go | 27 +++- agent/proxycfg/state_test.go | 122 ++++++++++++++++-- agent/xds/listeners.go | 43 ------ agent/xds/listeners_ingress.go | 54 +++++++- agent/xds/listeners_test.go | 53 ++++++++ ...th-tls-mixed-listeners.envoy-1-18-x.golden | 120 +++++++++++++++++ ...xed-listeners.v2compat.envoy-1-16-x.golden | 120 +++++++++++++++++ 7 files changed, 484 insertions(+), 55 deletions(-) create mode 100644 agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.envoy-1-18-x.golden create mode 100644 agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.v2compat.envoy-1-16-x.golden diff --git a/agent/proxycfg/ingress_gateway.go b/agent/proxycfg/ingress_gateway.go index d8b5842864d5..79ceac6bc8df 100644 --- a/agent/proxycfg/ingress_gateway.go +++ b/agent/proxycfg/ingress_gateway.go @@ -196,10 +196,31 @@ func (s *handlerIngressGateway) watchIngressLeafCert(ctx context.Context, snap * return nil } +// connectTLSServingEnabled returns true if Connect TLS is enabled at either +// gateway level or for at least one of the specific listeners. +func connectTLSServingEnabled(snap *ConfigSnapshot) bool { + if snap.IngressGateway.TLSConfig.Enabled { + return true + } + + for _, l := range snap.IngressGateway.Listeners { + if l.TLS != nil && l.TLS.Enabled { + return true + } + } + return false +} + func (s *handlerIngressGateway) generateIngressDNSSANs(snap *ConfigSnapshot) []string { - // Update our leaf cert watch with wildcard entries for our DNS domains as well as any - // configured custom hostnames from the service. - if !snap.IngressGateway.TLSConfig.Enabled { + // Update our leaf cert watch with wildcard entries for our DNS domains as + // well as any configured custom hostnames from the service. Note that in the + // case that only a subset of listeners are TLS-enabled, we still load DNS + // SANs for all upstreams. We could limit it to only those that are reachable + // from the enabled listeners but that adds a lot of complication and they are + // already wildcards anyway. It's simpler to have one certificate for the + // whole proxy that works for any possible upstream we might need than try to + // be more selective when we are already using wildcard DNS names! + if !connectTLSServingEnabled(snap) { return nil } diff --git a/agent/proxycfg/state_test.go b/agent/proxycfg/state_test.go index 8ee1017e37ad..be22dbec87ac 100644 --- a/agent/proxycfg/state_test.go +++ b/agent/proxycfg/state_test.go @@ -349,15 +349,33 @@ func genVerifyConfigEntryWatch(expectedKind, expectedName, expectedDatacenter st } } -func ingressConfigWatchEvent(tlsEnabled bool) cache.UpdateEvent { +func ingressConfigWatchEvent(gwTLS bool, mixedTLS bool) cache.UpdateEvent { + e := &structs.IngressGatewayConfigEntry{ + TLS: structs.GatewayTLSConfig{ + Enabled: gwTLS, + }, + } + + if mixedTLS { + // Add two listeners one with and one without connect TLS enabled + e.Listeners = []structs.IngressListener{ + { + Port: 8080, + Protocol: "tcp", + TLS: &structs.GatewayTLSConfig{Enabled: true}, + }, + { + Port: 9090, + Protocol: "tcp", + TLS: nil, + }, + } + } + return cache.UpdateEvent{ CorrelationID: gatewayConfigWatchID, Result: &structs.ConfigEntryResponse{ - Entry: &structs.IngressGatewayConfigEntry{ - TLS: structs.GatewayTLSConfig{ - Enabled: tlsEnabled, - }, - }, + Entry: e, }, Err: nil, } @@ -938,7 +956,7 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, { events: []cache.UpdateEvent{ - ingressConfigWatchEvent(false), + ingressConfigWatchEvent(false, false), }, verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { require.False(t, snap.Valid(), "gateway without hosts set is not valid") @@ -1088,7 +1106,7 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, events: []cache.UpdateEvent{ rootWatchEvent(), - ingressConfigWatchEvent(true), + ingressConfigWatchEvent(true, false), { CorrelationID: gatewayServicesWatchID, Result: &structs.IndexedGatewayServices{ @@ -1146,6 +1164,94 @@ func TestState_WatchesAndUpdates(t *testing.T) { }, }, }, + "ingress-gateway-with-mixed-tls": { + ns: structs.NodeService{ + Kind: structs.ServiceKindIngressGateway, + ID: "ingress-gateway", + Service: "ingress-gateway", + Address: "10.0.1.1", + }, + sourceDC: "dc1", + stages: []verificationStage{ + { + requiredWatches: map[string]verifyWatchRequest{ + rootsWatchID: genVerifyRootsWatch("dc1"), + gatewayConfigWatchID: genVerifyConfigEntryWatch(structs.IngressGateway, "ingress-gateway", "dc1"), + gatewayServicesWatchID: genVerifyGatewayServiceWatch("ingress-gateway", "dc1"), + }, + events: []cache.UpdateEvent{ + rootWatchEvent(), + ingressConfigWatchEvent(false, true), + { + CorrelationID: gatewayServicesWatchID, + Result: &structs.IndexedGatewayServices{ + Services: structs.GatewayServices{ + { + Gateway: structs.NewServiceName("ingress-gateway", nil), + Service: structs.NewServiceName("api", nil), + Hosts: []string{"test.example.com"}, + Port: 9999, + }, + }, + }, + Err: nil, + }, + { + CorrelationID: leafWatchID, + Result: issuedCert, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid()) + require.True(t, snap.IngressGateway.GatewayConfigLoaded) + // GW level TLS should be disabled + require.False(t, snap.IngressGateway.TLSConfig.Enabled) + // Mixed listener TLS + l, ok := snap.IngressGateway.Listeners[IngressListenerKey{"tcp", 8080}] + require.True(t, ok) + require.NotNil(t, l.TLS) + require.True(t, l.TLS.Enabled) + l, ok = snap.IngressGateway.Listeners[IngressListenerKey{"tcp", 9090}] + require.True(t, ok) + require.Nil(t, l.TLS) + + require.True(t, snap.IngressGateway.HostsSet) + require.Len(t, snap.IngressGateway.Hosts, 1) + require.Len(t, snap.IngressGateway.Upstreams, 1) + require.Len(t, snap.IngressGateway.WatchedDiscoveryChains, 1) + require.Contains(t, snap.IngressGateway.WatchedDiscoveryChains, api.String()) + }, + }, + { + requiredWatches: map[string]verifyWatchRequest{ + // This is the real point of this test - ensure we still generate + // the right DNS SANs for the whole gateway even when only a subset + // of listeners have TLS enabled. + leafWatchID: genVerifyLeafWatchWithDNSSANs("ingress-gateway", "dc1", []string{ + "test.example.com", + "*.ingress.consul.", + "*.ingress.dc1.consul.", + "*.ingress.alt.consul.", + "*.ingress.dc1.alt.consul.", + }), + }, + events: []cache.UpdateEvent{ + { + CorrelationID: gatewayServicesWatchID, + Result: &structs.IndexedGatewayServices{}, + Err: nil, + }, + }, + verifySnapshot: func(t testing.TB, snap *ConfigSnapshot) { + require.True(t, snap.Valid()) + require.Len(t, snap.IngressGateway.Upstreams, 0) + require.Len(t, snap.IngressGateway.WatchedDiscoveryChains, 0) + require.NotContains(t, snap.IngressGateway.WatchedDiscoveryChains, "api") + }, + }, + }, + }, "terminating-gateway-initial": { ns: structs.NodeService{ Kind: structs.ServiceKindTerminatingGateway, diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index c81d9f417a8f..5abf7f125ead 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -511,49 +511,6 @@ func (s *ResourceGenerator) listenersFromSnapshotGateway(cfgSnap *proxycfg.Confi return resources, err } -func resolveListenerSDSConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerKey proxycfg.IngressListenerKey) (*structs.GatewayTLSSDSConfig, error) { - var mergedCfg structs.GatewayTLSSDSConfig - - gwSDS := cfgSnap.IngressGateway.TLSConfig.SDS - if gwSDS != nil { - mergedCfg.ClusterName = gwSDS.ClusterName - mergedCfg.CertResource = gwSDS.CertResource - } - - listenerCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey] - if !ok { - return nil, fmt.Errorf("no listener config found for listener on port %d", listenerKey.Port) - } - - if listenerCfg.TLS != nil && listenerCfg.TLS.SDS != nil { - if listenerCfg.TLS.SDS.ClusterName != "" { - mergedCfg.ClusterName = listenerCfg.TLS.SDS.ClusterName - } - if listenerCfg.TLS.SDS.CertResource != "" { - mergedCfg.CertResource = listenerCfg.TLS.SDS.CertResource - } - } - - // Validate. Either merged should have both fields empty or both set. Other - // cases shouldn't be possible as we validate them at input but be robust to - // bugs later. - switch { - case mergedCfg.ClusterName == "" && mergedCfg.CertResource == "": - return nil, nil - - case mergedCfg.ClusterName != "" && mergedCfg.CertResource != "": - return &mergedCfg, nil - - case mergedCfg.ClusterName == "" && mergedCfg.CertResource != "": - return nil, fmt.Errorf("missing SDS cluster name for listener on port %d", listenerKey.Port) - - case mergedCfg.ClusterName != "" && mergedCfg.CertResource == "": - return nil, fmt.Errorf("missing SDS cert resource for listener on port %d", listenerKey.Port) - } - - return &mergedCfg, nil -} - // makeListener returns a listener with name and bind details set. Filters must // be added before it's useful. // diff --git a/agent/xds/listeners_ingress.go b/agent/xds/listeners_ingress.go index df9010a3b8d4..4f7add5587c4 100644 --- a/agent/xds/listeners_ingress.go +++ b/agent/xds/listeners_ingress.go @@ -19,6 +19,15 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap for listenerKey, upstreams := range cfgSnap.IngressGateway.Upstreams { var tlsContext *envoy_tls_v3.DownstreamTlsContext + listenerCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey] + if !ok { + return nil, fmt.Errorf("no listener config found for listener on port %d", listenerKey.Port) + } + // Enable connect TLS if it is enabled at the Gateway or specific listener + // level. + connectTLSEnabled := cfgSnap.IngressGateway.TLSConfig.Enabled || + (listenerCfg.TLS != nil && listenerCfg.TLS.Enabled) + sdsCfg, err := resolveListenerSDSConfig(cfgSnap, listenerKey) if err != nil { return nil, err @@ -30,7 +39,7 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap CommonTlsContext: makeCommonTLSContextFromSDS(*sdsCfg), RequireClientCertificate: &wrappers.BoolValue{Value: false}, } - } else if cfgSnap.IngressGateway.TLSConfig.Enabled { + } else if connectTLSEnabled { tlsContext = &envoy_tls_v3.DownstreamTlsContext{ CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.Leaf()), RequireClientCertificate: &wrappers.BoolValue{Value: false}, @@ -118,6 +127,49 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap return resources, nil } +func resolveListenerSDSConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerKey proxycfg.IngressListenerKey) (*structs.GatewayTLSSDSConfig, error) { + var mergedCfg structs.GatewayTLSSDSConfig + + gwSDS := cfgSnap.IngressGateway.TLSConfig.SDS + if gwSDS != nil { + mergedCfg.ClusterName = gwSDS.ClusterName + mergedCfg.CertResource = gwSDS.CertResource + } + + listenerCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey] + if !ok { + return nil, fmt.Errorf("no listener config found for listener on port %d", listenerKey.Port) + } + + if listenerCfg.TLS != nil && listenerCfg.TLS.SDS != nil { + if listenerCfg.TLS.SDS.ClusterName != "" { + mergedCfg.ClusterName = listenerCfg.TLS.SDS.ClusterName + } + if listenerCfg.TLS.SDS.CertResource != "" { + mergedCfg.CertResource = listenerCfg.TLS.SDS.CertResource + } + } + + // Validate. Either merged should have both fields empty or both set. Other + // cases shouldn't be possible as we validate them at input but be robust to + // bugs later. + switch { + case mergedCfg.ClusterName == "" && mergedCfg.CertResource == "": + return nil, nil + + case mergedCfg.ClusterName != "" && mergedCfg.CertResource != "": + return &mergedCfg, nil + + case mergedCfg.ClusterName == "" && mergedCfg.CertResource != "": + return nil, fmt.Errorf("missing SDS cluster name for listener on port %d", listenerKey.Port) + + case mergedCfg.ClusterName != "" && mergedCfg.CertResource == "": + return nil, fmt.Errorf("missing SDS cert resource for listener on port %d", listenerKey.Port) + } + + return &mergedCfg, nil +} + func routeNameForUpstream(l structs.IngressListener, s structs.IngressService) string { key := proxycfg.IngressListenerKeyFromListener(l) diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index a3664db844df..7dfc1b6c91d0 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -503,6 +503,59 @@ func TestListenersFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotIngressWithTLSListener, setup: nil, }, + { + name: "ingress-with-tls-mixed-listeners", + // Use SDS helper even though we aren't testing SDS since it already sets + // up most things we need. + create: proxycfg.TestConfigSnapshotIngressWithGatewaySDS, + setup: func(snap *proxycfg.ConfigSnapshot) { + // Undo gateway-level SDS + snap.IngressGateway.TLSConfig.SDS = nil + + // No Gateway-level built-in TLS + snap.IngressGateway.TLSConfig.Enabled = false + + // One listener has built-in TLS, one doesn't + snap.IngressGateway.Upstreams = map[proxycfg.IngressListenerKey]structs.Upstreams{ + {Protocol: "http", Port: 8080}: { + { + DestinationName: "s1", + LocalBindPort: 8080, + }, + }, + {Protocol: "http", Port: 9090}: { + { + DestinationName: "s2", + LocalBindPort: 9090, + }, + }, + } + snap.IngressGateway.Listeners = map[proxycfg.IngressListenerKey]structs.IngressListener{ + {Protocol: "http", Port: 8080}: { + Port: 8080, + Services: []structs.IngressService{ + { + Name: "s1", + }, + }, + TLS: &structs.GatewayTLSConfig{ + // built-in TLS enabled + Enabled: true, + }, + }, + {Protocol: "http", Port: 9090}: { + Port: 9090, + Services: []structs.IngressService{ + { + Name: "s2", + }, + }, + // No TLS enabled + TLS: nil, + }, + } + }, + }, { name: "ingress-with-sds-listener-gw-level", create: proxycfg.TestConfigSnapshotIngressWithGatewaySDS, diff --git a/agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.envoy-1-18-x.golden b/agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.envoy-1-18-x.golden new file mode 100644 index 000000000000..e5ba23e8933c --- /dev/null +++ b/agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.envoy-1-18-x.golden @@ -0,0 +1,120 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "http:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "ingress_upstream_8080", + "rds": { + "configSource": { + "ads": { + + }, + "resourceApiVersion": "V3" + }, + "routeConfigName": "8080" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.router" + } + ], + "tracing": { + "randomSampling": { + + } + } + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": false + } + } + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.config.listener.v3.Listener", + "name": "http:1.2.3.4:9090", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 9090 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "statPrefix": "ingress_upstream_9090", + "rds": { + "configSource": { + "ads": { + + }, + "resourceApiVersion": "V3" + }, + "routeConfigName": "9090" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.router" + } + ], + "tracing": { + "randomSampling": { + + } + } + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.config.listener.v3.Listener", + "nonce": "00000001" +} \ No newline at end of file diff --git a/agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.v2compat.envoy-1-16-x.golden b/agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.v2compat.envoy-1-16-x.golden new file mode 100644 index 000000000000..29bf2b301f49 --- /dev/null +++ b/agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.v2compat.envoy-1-16-x.golden @@ -0,0 +1,120 @@ +{ + "versionInfo": "00000001", + "resources": [ + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "http:1.2.3.4:8080", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 8080 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager", + "statPrefix": "ingress_upstream_8080", + "rds": { + "configSource": { + "ads": { + + }, + "resourceApiVersion": "V2" + }, + "routeConfigName": "8080" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.router" + } + ], + "tracing": { + "randomSampling": { + + } + } + } + } + ], + "transportSocket": { + "name": "tls", + "typedConfig": { + "@type": "type.googleapis.com/envoy.api.v2.auth.DownstreamTlsContext", + "commonTlsContext": { + "tlsParams": { + + }, + "tlsCertificates": [ + { + "certificateChain": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICjDCCAjKgAwIBAgIIC5llxGV1gB8wCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowDjEMMAoG\nA1UEAxMDd2ViMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEADPv1RHVNRfa2VKR\nAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Favq5E0ivpNtv1QnFhxtPd7d5k4e+T7\nSkW1TaOCAXIwggFuMA4GA1UdDwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcD\nAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBoBgNVHQ4EYQRfN2Q6MDc6ODc6M2E6\nNDA6MTk6NDc6YzM6NWE6YzA6YmE6NjI6ZGY6YWY6NGI6ZDQ6MDU6MjU6NzY6M2Q6\nNWE6OGQ6MTY6OGQ6Njc6NWU6MmU6YTA6MzQ6N2Q6ZGM6ZmYwagYDVR0jBGMwYYBf\nZDE6MTE6MTE6YWM6MmE6YmE6OTc6YjI6M2Y6YWM6N2I6YmQ6ZGE6YmU6YjE6OGE6\nZmM6OWE6YmE6YjU6YmM6ODM6ZTc6NWU6NDE6NmY6ZjI6NzM6OTU6NTg6MGM6ZGIw\nWQYDVR0RBFIwUIZOc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9ucy9kZWZhdWx0L2RjL2RjMS9zdmMvd2ViMAoGCCqG\nSM49BAMCA0gAMEUCIGC3TTvvjj76KMrguVyFf4tjOqaSCRie3nmHMRNNRav7AiEA\npY0heYeK9A6iOLrzqxSerkXXQyj5e9bE4VgUnxgPU6g=\n-----END CERTIFICATE-----\n" + }, + "privateKey": { + "inlineString": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIMoTkpRggp3fqZzFKh82yS4LjtJI+XY+qX/7DefHFrtdoAoGCCqGSM49\nAwEHoUQDQgAEADPv1RHVNRfa2VKRAB16b6rZnEt7tuhaxCFpQXPj7M2omb0B9Fav\nq5E0ivpNtv1QnFhxtPd7d5k4e+T7SkW1TQ==\n-----END EC PRIVATE KEY-----\n" + } + } + ], + "validationContext": { + "trustedCa": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + } + }, + "requireClientCertificate": false + } + } + } + ], + "trafficDirection": "OUTBOUND" + }, + { + "@type": "type.googleapis.com/envoy.api.v2.Listener", + "name": "http:1.2.3.4:9090", + "address": { + "socketAddress": { + "address": "1.2.3.4", + "portValue": 9090 + } + }, + "filterChains": [ + { + "filters": [ + { + "name": "envoy.filters.network.http_connection_manager", + "typedConfig": { + "@type": "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager", + "statPrefix": "ingress_upstream_9090", + "rds": { + "configSource": { + "ads": { + + }, + "resourceApiVersion": "V2" + }, + "routeConfigName": "9090" + }, + "httpFilters": [ + { + "name": "envoy.filters.http.router" + } + ], + "tracing": { + "randomSampling": { + + } + } + } + } + ] + } + ], + "trafficDirection": "OUTBOUND" + } + ], + "typeUrl": "type.googleapis.com/envoy.api.v2.Listener", + "nonce": "00000001" +} \ No newline at end of file From d779a4fc2c95610bccedee23cc8d5edacb477b46 Mon Sep 17 00:00:00 2001 From: Paul Banks Date: Tue, 28 Sep 2021 13:42:59 +0100 Subject: [PATCH 2/4] Add Changelog --- .changelog/11163.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/11163.txt diff --git a/.changelog/11163.txt b/.changelog/11163.txt new file mode 100644 index 000000000000..f47fe685ab9d --- /dev/null +++ b/.changelog/11163.txt @@ -0,0 +1,3 @@ +```release-note:improvement +connect: ingress gateways may now enable built-in TLS for a subset of listeners. +``` From 6faf85bccd52ea871ea31a0b4bcf174027b7ad9b Mon Sep 17 00:00:00 2001 From: Paul Banks Date: Fri, 8 Oct 2021 12:33:20 +0100 Subject: [PATCH 3/4] Refactor `resolveListenerSDSConfig` to pass in whole config --- agent/xds/listeners_ingress.go | 13 ++++--------- agent/xds/listeners_test.go | 9 ++++----- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/agent/xds/listeners_ingress.go b/agent/xds/listeners_ingress.go index 4f7add5587c4..5a494fe99205 100644 --- a/agent/xds/listeners_ingress.go +++ b/agent/xds/listeners_ingress.go @@ -28,7 +28,7 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap connectTLSEnabled := cfgSnap.IngressGateway.TLSConfig.Enabled || (listenerCfg.TLS != nil && listenerCfg.TLS.Enabled) - sdsCfg, err := resolveListenerSDSConfig(cfgSnap, listenerKey) + sdsCfg, err := resolveListenerSDSConfig(cfgSnap, listenerCfg) if err != nil { return nil, err } @@ -127,7 +127,7 @@ func (s *ResourceGenerator) makeIngressGatewayListeners(address string, cfgSnap return resources, nil } -func resolveListenerSDSConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerKey proxycfg.IngressListenerKey) (*structs.GatewayTLSSDSConfig, error) { +func resolveListenerSDSConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerCfg structs.IngressListener) (*structs.GatewayTLSSDSConfig, error) { var mergedCfg structs.GatewayTLSSDSConfig gwSDS := cfgSnap.IngressGateway.TLSConfig.SDS @@ -136,11 +136,6 @@ func resolveListenerSDSConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerKey prox mergedCfg.CertResource = gwSDS.CertResource } - listenerCfg, ok := cfgSnap.IngressGateway.Listeners[listenerKey] - if !ok { - return nil, fmt.Errorf("no listener config found for listener on port %d", listenerKey.Port) - } - if listenerCfg.TLS != nil && listenerCfg.TLS.SDS != nil { if listenerCfg.TLS.SDS.ClusterName != "" { mergedCfg.ClusterName = listenerCfg.TLS.SDS.ClusterName @@ -161,10 +156,10 @@ func resolveListenerSDSConfig(cfgSnap *proxycfg.ConfigSnapshot, listenerKey prox return &mergedCfg, nil case mergedCfg.ClusterName == "" && mergedCfg.CertResource != "": - return nil, fmt.Errorf("missing SDS cluster name for listener on port %d", listenerKey.Port) + return nil, fmt.Errorf("missing SDS cluster name for listener on port %d", listenerCfg.Port) case mergedCfg.ClusterName != "" && mergedCfg.CertResource == "": - return nil, fmt.Errorf("missing SDS cert resource for listener on port %d", listenerKey.Port) + return nil, fmt.Errorf("missing SDS cert resource for listener on port %d", listenerCfg.Port) } return &mergedCfg, nil diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 7dfc1b6c91d0..dfbf12acb38e 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -1172,7 +1172,7 @@ func TestResolveListenerSDSConfig(t *testing.T) { snap := proxycfg.TestConfigSnapshotIngressWithGatewaySDS(t) // Override TLS configs snap.IngressGateway.TLSConfig.SDS = tc.gwSDS - var key proxycfg.IngressListenerKey + var listenerCfg structs.IngressListener for k, lisCfg := range snap.IngressGateway.Listeners { if tc.lisSDS == nil { lisCfg.TLS = nil @@ -1183,12 +1183,11 @@ func TestResolveListenerSDSConfig(t *testing.T) { } // Override listener cfg in map snap.IngressGateway.Listeners[k] = lisCfg - // Save the last key doesn't matter which as we set same listener config - // for all. - key = k + // Save the last cfg doesn't matter which as we set same for all. + listenerCfg = lisCfg } - got, err := resolveListenerSDSConfig(snap, key) + got, err := resolveListenerSDSConfig(snap, listenerCfg) if tc.wantErr != "" { require.Error(t, err) require.Contains(t, err.Error(), tc.wantErr) From c891f30c2400e610d1446c4bdb6760323b04d161 Mon Sep 17 00:00:00 2001 From: Paul Banks Date: Tue, 19 Oct 2021 21:37:58 +0100 Subject: [PATCH 4/4] Rebase and rebuild golden files for Envoy version bump --- ...olden => ingress-with-tls-mixed-listeners.envoy-1-19-x.golden} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename agent/xds/testdata/listeners/{ingress-with-tls-mixed-listeners.envoy-1-18-x.golden => ingress-with-tls-mixed-listeners.envoy-1-19-x.golden} (100%) diff --git a/agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.envoy-1-18-x.golden b/agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.envoy-1-19-x.golden similarity index 100% rename from agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.envoy-1-18-x.golden rename to agent/xds/testdata/listeners/ingress-with-tls-mixed-listeners.envoy-1-19-x.golden