From d1b1d1735e1540c71d52ff3ca1e7fbcd82000864 Mon Sep 17 00:00:00 2001 From: Derek Menteer Date: Wed, 11 Jan 2023 11:30:15 -0600 Subject: [PATCH] Add Peer field to service-defaults upstream overrides. --- agent/configentry/merge_service_config.go | 17 ++++++--- .../configentry/merge_service_config_test.go | 38 +++++-------------- agent/configentry/resolve.go | 29 ++++++++------ agent/configentry/resolve_test.go | 17 +++++---- agent/consul/config_endpoint_test.go | 27 +++++++------ agent/consul/state/config_entry.go | 6 ++- agent/service_manager.go | 2 +- agent/service_manager_test.go | 12 ++++-- agent/structs/config_entry.go | 32 +++++----------- agent/structs/connect_proxy_config_oss.go | 7 ++-- agent/structs/structs.go | 4 ++ agent/xds/routes.go | 2 +- 12 files changed, 98 insertions(+), 95 deletions(-) diff --git a/agent/configentry/merge_service_config.go b/agent/configentry/merge_service_config.go index 17e68ce1d2fcb..ad6f41117ba8e 100644 --- a/agent/configentry/merge_service_config.go +++ b/agent/configentry/merge_service_config.go @@ -37,7 +37,11 @@ func MergeNodeServiceWithCentralConfig( // so we can learn about their configs. for _, us := range ns.Proxy.Upstreams { if us.DestinationType == "" || us.DestinationType == structs.UpstreamDestTypeService { - sid := us.DestinationID() + // Peer services do not have service-defaults config entries to fetch. + if us.DestinationPeer != "" { + continue + } + sid := us.DestinationID().ServiceName.ToServiceID() sid.EnterpriseMeta.Merge(&ns.EnterpriseMeta) upstreams = append(upstreams, sid) } @@ -149,7 +153,7 @@ func MergeServiceConfig(defaults *structs.ServiceConfigResponse, service *struct } // remoteUpstreams contains synthetic Upstreams generated from central config (service-defaults.UpstreamConfigs). - remoteUpstreams := make(map[structs.ServiceID]structs.Upstream) + remoteUpstreams := make(map[structs.PeeredServiceName]structs.Upstream) for _, us := range defaults.UpstreamIDConfigs { parsed, err := structs.ParseUpstreamConfigNoDefaults(us.Config) @@ -158,9 +162,10 @@ func MergeServiceConfig(defaults *structs.ServiceConfigResponse, service *struct } remoteUpstreams[us.Upstream] = structs.Upstream{ - DestinationNamespace: us.Upstream.NamespaceOrDefault(), - DestinationPartition: us.Upstream.PartitionOrDefault(), - DestinationName: us.Upstream.ID, + DestinationNamespace: us.Upstream.ServiceName.NamespaceOrDefault(), + DestinationPartition: us.Upstream.ServiceName.PartitionOrDefault(), + DestinationName: us.Upstream.ServiceName.Name, + DestinationPeer: us.Upstream.Peer, Config: us.Config, MeshGateway: parsed.MeshGateway, CentrallyConfigured: true, @@ -170,7 +175,7 @@ func MergeServiceConfig(defaults *structs.ServiceConfigResponse, service *struct // localUpstreams stores the upstreams seen from the local registration so that we can merge in the synthetic entries. // In transparent proxy mode ns.Proxy.Upstreams will likely be empty because users do not need to define upstreams explicitly. // So to store upstream-specific flags from central config, we add entries to ns.Proxy.Upstreams with those values. - localUpstreams := make(map[structs.ServiceID]struct{}) + localUpstreams := make(map[structs.PeeredServiceName]struct{}) // Merge upstream defaults into the local registration for i := range ns.Proxy.Upstreams { diff --git a/agent/configentry/merge_service_config_test.go b/agent/configentry/merge_service_config_test.go index c6a6d3ae03fcb..7d0651df5eb4c 100644 --- a/agent/configentry/merge_service_config_test.go +++ b/agent/configentry/merge_service_config_test.go @@ -275,6 +275,9 @@ func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { defaults *structs.ServiceConfigResponse service *structs.NodeService } + zapUpstreamId := structs.PeeredServiceName{ + ServiceName: structs.NewServiceName("zap", structs.DefaultEnterpriseMetaInDefaultPartition()), + } tests := []struct { name string args args @@ -286,10 +289,7 @@ func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { defaults: &structs.ServiceConfigResponse{ UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ { - Upstream: structs.ServiceID{ - ID: "zap", - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, + Upstream: zapUpstreamId, Config: map[string]interface{}{ "passive_health_check": map[string]interface{}{ "Interval": int64(10), @@ -357,10 +357,7 @@ func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { defaults: &structs.ServiceConfigResponse{ UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ { - Upstream: structs.ServiceID{ - ID: "zap", - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, + Upstream: zapUpstreamId, Config: map[string]interface{}{ "protocol": "grpc", }, @@ -432,10 +429,7 @@ func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { defaults: &structs.ServiceConfigResponse{ UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ { - Upstream: structs.ServiceID{ - ID: "zap", - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, + Upstream: zapUpstreamId, Config: map[string]interface{}{ "protocol": "grpc", }, @@ -490,10 +484,7 @@ func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { defaults: &structs.ServiceConfigResponse{ UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ { - Upstream: structs.ServiceID{ - ID: "zap", - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, + Upstream: zapUpstreamId, Config: map[string]interface{}{ "mesh_gateway": map[string]interface{}{ "Mode": "local", @@ -550,10 +541,7 @@ func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { defaults: &structs.ServiceConfigResponse{ UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ { - Upstream: structs.ServiceID{ - ID: "zap", - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, + Upstream: zapUpstreamId, Config: map[string]interface{}{ "mesh_gateway": map[string]interface{}{ "Mode": "local", @@ -613,10 +601,7 @@ func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { defaults: &structs.ServiceConfigResponse{ UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ { - Upstream: structs.ServiceID{ - ID: "zap", - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, + Upstream: zapUpstreamId, Config: map[string]interface{}{ "protocol": "http", }, @@ -661,10 +646,7 @@ func Test_MergeServiceConfig_UpstreamOverrides(t *testing.T) { defaults: &structs.ServiceConfigResponse{ UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ { - Upstream: structs.ServiceID{ - ID: "zap", - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, + Upstream: zapUpstreamId, Config: map[string]interface{}{ "protocol": "http", }, diff --git a/agent/configentry/resolve.go b/agent/configentry/resolve.go index ee108ee70a2fc..60b2b36748eef 100644 --- a/agent/configentry/resolve.go +++ b/agent/configentry/resolve.go @@ -113,7 +113,7 @@ func ComputeResolvedServiceConfig( // Upstreams can come from: // - Explicitly from proxy registrations, and therefore as an argument to this RPC endpoint // - Implicitly from centralized upstream config in service-defaults - seenUpstreams := map[structs.ServiceID]struct{}{} + seenUpstreams := map[structs.PeeredServiceName]struct{}{} var ( noUpstreamArgs = len(upstreamIDs) == 0 && len(args.Upstreams) == 0 @@ -132,17 +132,20 @@ func ComputeResolvedServiceConfig( // First store all upstreams that were provided in the request for _, sid := range upstreamIDs { - if _, ok := seenUpstreams[sid]; !ok { - seenUpstreams[sid] = struct{}{} + psn := structs.PeeredServiceName{ + ServiceName: structs.NewServiceName(sid.ID, &sid.EnterpriseMeta), + } + if _, ok := seenUpstreams[psn]; !ok { + seenUpstreams[psn] = struct{}{} } } // Then store upstreams inferred from service-defaults and mapify the overrides. var ( - upstreamOverrides = make(map[structs.ServiceID]*structs.UpstreamConfig) + upstreamOverrides = make(map[structs.PeeredServiceName]*structs.UpstreamConfig) upstreamDefaults *structs.UpstreamConfig // resolvedConfigs stores the opaque config map for each upstream and is keyed on the upstream's ID. - resolvedConfigs = make(map[structs.ServiceID]map[string]interface{}) + resolvedConfigs = make(map[structs.PeeredServiceName]map[string]interface{}) ) if serviceConf != nil && serviceConf.UpstreamConfig != nil { for i, override := range serviceConf.UpstreamConfig.Overrides { @@ -156,8 +159,9 @@ func ComputeResolvedServiceConfig( ) continue // skip this impossible condition } - seenUpstreams[override.ServiceID()] = struct{}{} - upstreamOverrides[override.ServiceID()] = override + psn := override.PeeredServiceName() + seenUpstreams[psn] = struct{}{} + upstreamOverrides[psn] = override } if serviceConf.UpstreamConfig.Defaults != nil { upstreamDefaults = serviceConf.UpstreamConfig.Defaults @@ -175,7 +179,9 @@ func ComputeResolvedServiceConfig( cfgMap["mesh_gateway"] = args.MeshGateway } - wildcard := structs.NewServiceID(structs.WildcardSpecifier, args.WithWildcardNamespace()) + wildcard := structs.PeeredServiceName{ + ServiceName: structs.NewServiceName(structs.WildcardSpecifier, args.WithWildcardNamespace()), + } resolvedConfigs[wildcard] = cfgMap } } @@ -190,9 +196,7 @@ func ComputeResolvedServiceConfig( // (how the downstream wants to address it) protocol := proxyConfGlobalProtocol - upstreamSvcDefaults := entries.GetServiceDefaults( - structs.NewServiceID(upstream.ID, &upstream.EnterpriseMeta), - ) + upstreamSvcDefaults := entries.GetServiceDefaults(upstream.ServiceName.ToServiceID()) if upstreamSvcDefaults != nil { if upstreamSvcDefaults.Protocol != "" { protocol = upstreamSvcDefaults.Protocol @@ -249,12 +253,13 @@ func ComputeResolvedServiceConfig( return &thisReply, nil } + // TODO can we remove these legacy upstreams? if legacyUpstreams { // For legacy upstreams we return a map that is only keyed on the string ID, since they precede namespaces thisReply.UpstreamConfigs = make(map[string]map[string]interface{}) for us, conf := range resolvedConfigs { - thisReply.UpstreamConfigs[us.ID] = conf + thisReply.UpstreamConfigs[us.ServiceName.Name] = conf } } else { diff --git a/agent/configentry/resolve_test.go b/agent/configentry/resolve_test.go index 151cf06368edf..e50e6eb911dc2 100644 --- a/agent/configentry/resolve_test.go +++ b/agent/configentry/resolve_test.go @@ -21,12 +21,15 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) { ID: "sid", EnterpriseMeta: *acl.DefaultEnterpriseMeta(), } - uid := structs.ServiceID{ - ID: "upstream1", - EnterpriseMeta: *acl.DefaultEnterpriseMeta(), + uid := structs.PeeredServiceName{ + ServiceName: structs.NewServiceName("upstream1", acl.DefaultEnterpriseMeta()), + } + + uids := []structs.ServiceID{uid.ServiceName.ToServiceID()} + + wildcard := structs.PeeredServiceName{ + ServiceName: structs.NewServiceName(structs.WildcardSpecifier, acl.WildcardEnterpriseMeta()), } - uids := []structs.ServiceID{uid} - wildcard := structs.NewServiceID(structs.WildcardSpecifier, acl.WildcardEnterpriseMeta()) localMeshGW := structs.MeshGatewayConfig{Mode: structs.MeshGatewayModeLocal} remoteMeshGW := structs.MeshGatewayConfig{Mode: structs.MeshGatewayModeRemote} @@ -361,7 +364,7 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) { }, Overrides: []*structs.UpstreamConfig{ { - Name: uid.ID, + Name: uid.ServiceName.Name, MeshGateway: remoteMeshGW, // applied 3rd }, }, @@ -473,7 +476,7 @@ func Test_ComputeResolvedServiceConfig(t *testing.T) { require.NoError(t, err) // This is needed because map iteration is random and determines the order of some outputs. sort.Slice(got.UpstreamIDConfigs, func(i, j int) bool { - return got.UpstreamIDConfigs[i].Upstream.ID < got.UpstreamIDConfigs[j].Upstream.ID + return got.UpstreamIDConfigs[i].Upstream.ServiceName.Name < got.UpstreamIDConfigs[j].Upstream.ServiceName.Name }) require.Equal(t, tt.want, got) }) diff --git a/agent/consul/config_endpoint_test.go b/agent/consul/config_endpoint_test.go index fa06cf9b9f1f1..d22a991b77f51 100644 --- a/agent/consul/config_endpoint_test.go +++ b/agent/consul/config_endpoint_test.go @@ -1227,9 +1227,15 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { } t.Parallel() - mysql := structs.NewServiceID("mysql", structs.DefaultEnterpriseMetaInDefaultPartition()) - cache := structs.NewServiceID("cache", structs.DefaultEnterpriseMetaInDefaultPartition()) - wildcard := structs.NewServiceID(structs.WildcardSpecifier, structs.WildcardEnterpriseMetaInDefaultPartition()) + cache := structs.PeeredServiceName{ + ServiceName: structs.NewServiceName("cache", structs.DefaultEnterpriseMetaInDefaultPartition()), + } + mysql := structs.PeeredServiceName{ + ServiceName: structs.NewServiceName("mysql", structs.DefaultEnterpriseMetaInDefaultPartition()), + } + wildcard := structs.PeeredServiceName{ + ServiceName: structs.NewServiceName(structs.WildcardSpecifier, structs.WildcardEnterpriseMetaInDefaultPartition()), + } tt := []struct { name string @@ -1306,7 +1312,7 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { Name: "api", Datacenter: "dc1", UpstreamIDs: []structs.ServiceID{ - cache, + cache.ServiceName.ToServiceID(), }, }, expect: structs.ServiceConfigResponse{ @@ -1321,10 +1327,7 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { }, }, { - Upstream: structs.ServiceID{ - ID: "mysql", - EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(), - }, + Upstream: mysql, Config: map[string]interface{}{ "protocol": "http", }, @@ -1352,7 +1355,7 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { Mode: structs.MeshGatewayModeNone, }, UpstreamIDs: []structs.ServiceID{ - mysql, + mysql.ServiceName.ToServiceID(), }, }, expect: structs.ServiceConfigResponse{ @@ -1421,7 +1424,7 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams(t *testing.T) { Mode: structs.MeshGatewayModeNone, }, UpstreamIDs: []structs.ServiceID{ - mysql, + mysql.ServiceName.ToServiceID(), }, }, expect: structs.ServiceConfigResponse{ @@ -1856,7 +1859,9 @@ func TestConfigEntry_ResolveServiceConfig_Upstreams_Blocking(t *testing.T) { }, UpstreamIDConfigs: []structs.OpaqueUpstreamConfig{ { - Upstream: structs.NewServiceID("bar", nil), + Upstream: structs.PeeredServiceName{ + ServiceName: structs.NewServiceName("bar", nil), + }, Config: map[string]interface{}{ "protocol": "http", }, diff --git a/agent/consul/state/config_entry.go b/agent/consul/state/config_entry.go index 97b7e3f281c42..13a9dc923a8b6 100644 --- a/agent/consul/state/config_entry.go +++ b/agent/consul/state/config_entry.go @@ -1154,7 +1154,11 @@ func (s *Store) ReadResolvedServiceConfigEntries( if override.Name == "" { continue // skip this impossible condition } - seenUpstreams[override.ServiceID()] = struct{}{} + if override.Peer != "" { + continue // Peer services do not have service-defaults config entries to fetch. + } + sid := override.PeeredServiceName().ServiceName.ToServiceID() + seenUpstreams[sid] = struct{}{} } } diff --git a/agent/service_manager.go b/agent/service_manager.go index 290102cb1291a..0ff0bd0c7a104 100644 --- a/agent/service_manager.go +++ b/agent/service_manager.go @@ -326,7 +326,7 @@ func makeConfigRequest(bd BaseDeps, addReq AddServiceRequest) *structs.ServiceCo // learn about their configs. for _, us := range ns.Proxy.Upstreams { if us.DestinationType == "" || us.DestinationType == structs.UpstreamDestTypeService { - sid := us.DestinationID() + sid := us.DestinationID().ServiceName.ToServiceID() sid.EnterpriseMeta.Merge(&ns.EnterpriseMeta) upstreams = append(upstreams, sid) } diff --git a/agent/service_manager_test.go b/agent/service_manager_test.go index c346268303f0e..6036e520a6386 100644 --- a/agent/service_manager_test.go +++ b/agent/service_manager_test.go @@ -420,7 +420,9 @@ func TestServiceManager_PersistService_API(t *testing.T) { }, UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ structs.OpaqueUpstreamConfig{ - Upstream: structs.NewServiceID("redis", nil), + Upstream: structs.PeeredServiceName{ + ServiceName: structs.NewServiceName("redis", nil), + }, Config: map[string]interface{}{ "protocol": "tcp", }, @@ -468,7 +470,9 @@ func TestServiceManager_PersistService_API(t *testing.T) { }, UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ structs.OpaqueUpstreamConfig{ - Upstream: structs.NewServiceID("redis", nil), + Upstream: structs.PeeredServiceName{ + ServiceName: structs.NewServiceName("redis", nil), + }, Config: map[string]interface{}{ "protocol": "tcp", }, @@ -647,7 +651,9 @@ func TestServiceManager_PersistService_ConfigFiles(t *testing.T) { }, UpstreamIDConfigs: structs.OpaqueUpstreamConfigs{ structs.OpaqueUpstreamConfig{ - Upstream: structs.NewServiceID("redis", nil), + Upstream: structs.PeeredServiceName{ + ServiceName: structs.NewServiceName("redis", nil), + }, Config: map[string]interface{}{ "protocol": "tcp", }, diff --git a/agent/structs/config_entry.go b/agent/structs/config_entry.go index 65615103e91c6..17ed42cc12d40 100644 --- a/agent/structs/config_entry.go +++ b/agent/structs/config_entry.go @@ -168,7 +168,7 @@ func (e *ServiceConfigEntry) Normalize() error { for _, override := range e.UpstreamConfig.Overrides { err := override.NormalizeWithName(&e.EnterpriseMeta) if err != nil { - validationErr = multierror.Append(validationErr, fmt.Errorf("error in upstream override for %s: %v", override.ServiceName(), err)) + validationErr = multierror.Append(validationErr, fmt.Errorf("error in upstream override for %s: %v", override.PeeredServiceName(), err)) } } @@ -207,7 +207,7 @@ func (e *ServiceConfigEntry) Validate() error { for _, override := range e.UpstreamConfig.Overrides { err := override.ValidateWithName() if err != nil { - validationErr = multierror.Append(validationErr, fmt.Errorf("error in upstream override for %s: %v", override.ServiceName(), err)) + validationErr = multierror.Append(validationErr, fmt.Errorf("error in upstream override for %s: %v", override.PeeredServiceName(), err)) } } @@ -838,6 +838,8 @@ type UpstreamConfig struct { Name string `json:",omitempty"` // EnterpriseMeta is only accepted within a service-defaults config entry. acl.EnterpriseMeta `hcl:",squash" mapstructure:",squash"` + // Peer is only accepted within a service-defaults config entry. + Peer string // EnvoyListenerJSON is a complete override ("escape hatch") for the upstream's // listener. @@ -889,18 +891,14 @@ func (cfg UpstreamConfig) Clone() UpstreamConfig { return cfg2 } -func (cfg *UpstreamConfig) ServiceID() ServiceID { +func (cfg *UpstreamConfig) PeeredServiceName() PeeredServiceName { if cfg.Name == "" { - return ServiceID{} + return PeeredServiceName{} } - return NewServiceID(cfg.Name, &cfg.EnterpriseMeta) -} - -func (cfg *UpstreamConfig) ServiceName() ServiceName { - if cfg.Name == "" { - return ServiceName{} + return PeeredServiceName{ + Peer: cfg.Peer, + ServiceName: NewServiceName(cfg.Name, &cfg.EnterpriseMeta), } - return NewServiceName(cfg.Name, &cfg.EnterpriseMeta) } func (cfg UpstreamConfig) MergeInto(dst map[string]interface{}) { @@ -1140,22 +1138,12 @@ func (ul UpstreamLimits) Validate() error { } type OpaqueUpstreamConfig struct { - Upstream ServiceID + Upstream PeeredServiceName Config map[string]interface{} } type OpaqueUpstreamConfigs []OpaqueUpstreamConfig -func (configs OpaqueUpstreamConfigs) GetUpstreamConfig(sid ServiceID) (config map[string]interface{}, found bool) { - for _, usconf := range configs { - if usconf.Upstream.Matches(sid) { - return usconf.Config, true - } - } - - return nil, false -} - type ServiceConfigResponse struct { ProxyConfig map[string]interface{} UpstreamConfigs map[string]map[string]interface{} diff --git a/agent/structs/connect_proxy_config_oss.go b/agent/structs/connect_proxy_config_oss.go index 9e53a8fa2fbff..320d7c4700e7c 100644 --- a/agent/structs/connect_proxy_config_oss.go +++ b/agent/structs/connect_proxy_config_oss.go @@ -11,9 +11,10 @@ func (us *Upstream) GetEnterpriseMeta() *acl.EnterpriseMeta { return DefaultEnterpriseMetaInDefaultPartition() } -func (us *Upstream) DestinationID() ServiceID { - return ServiceID{ - ID: us.DestinationName, +func (us *Upstream) DestinationID() PeeredServiceName { + return PeeredServiceName{ + Peer: us.DestinationPeer, + ServiceName: NewServiceName(us.DestinationName, DefaultEnterpriseMetaInDefaultPartition()), } } diff --git a/agent/structs/structs.go b/agent/structs/structs.go index 4915c21bc6528..a12ddb70e422f 100644 --- a/agent/structs/structs.go +++ b/agent/structs/structs.go @@ -2246,6 +2246,10 @@ type PeeredServiceName struct { Peer string } +func (psn PeeredServiceName) String() string { + return fmt.Sprintf("%v:%v", psn.ServiceName.String(), psn.Peer) +} + type ServiceName struct { Name string acl.EnterpriseMeta `mapstructure:",squash"` diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 2931e2d7707f7..e77519f88c3d0 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -435,7 +435,7 @@ func findIngressServiceMatchingUpstream(l structs.IngressListener, u structs.Ups // only one IngressService for each unique name although originally that // wasn't checked as it didn't matter. Assume there is only one now // though! - wantSID := u.DestinationID() + wantSID := u.DestinationID().ServiceName.ToServiceID() var foundSameNSWildcard *structs.IngressService for _, s := range l.Services { sid := structs.NewServiceID(s.Name, &s.EnterpriseMeta)