[NET-6581] perf: Fetch services once rather than per-node on deregister #3322
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| ```release-note:improvement | ||
| control-plane: reduce Consul Catalog API requests required for endpoints reconcile in large clusters | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -933,70 +933,68 @@ func getHealthCheckStatusReason(healthCheckStatus, podName, podNamespace string) | |
| // has addresses, it will only deregister instances not in the map. | ||
| func (r *Controller) deregisterService(apiClient *api.Client, k8sSvcName, k8sSvcNamespace string, endpointsAddressesMap map[string]bool) error { | ||
| // Get services matching metadata from Consul | ||
| serviceInstances, err := r.serviceInstances(apiClient, k8sSvcName, k8sSvcNamespace) | ||
| if err != nil { | ||
| r.Log.Error(err, "failed to get service instances", "name", k8sSvcName) | ||
| return err | ||
| } | ||
|
|
||
| var errs error | ||
| for _, svc := range serviceInstances { | ||
| // We need to get services matching "k8s-service-name" and "k8s-namespace" metadata. | ||
| // If we selectively deregister, only deregister if the address is not in the map. Otherwise, deregister | ||
| // every service instance. | ||
| var serviceDeregistered bool | ||
| if endpointsAddressesMap != nil { | ||
| if _, ok := endpointsAddressesMap[svc.ServiceAddress]; !ok { | ||
| // If the service address is not in the Endpoints addresses, deregister it. | ||
| r.Log.Info("deregistering service from consul", "svc", svc.ServiceID) | ||
| _, err := apiClient.Catalog().Deregister(&api.CatalogDeregistration{ | ||
| Node: svc.Node, | ||
| ServiceID: svc.ServiceID, | ||
| Namespace: svc.Namespace, | ||
| }, nil) | ||
| if err != nil { | ||
| // Do not exit right away as there might be other services that need to be deregistered. | ||
| r.Log.Error(err, "failed to deregister service instance", "id", svc.ServiceID) | ||
| errs = multierror.Append(errs, err) | ||
| } else { | ||
| serviceDeregistered = true | ||
| } | ||
| } | ||
| } else { | ||
| r.Log.Info("deregistering service from consul", "svc", svc.ServiceID) | ||
| _, err := apiClient.Catalog().Deregister(&api.CatalogDeregistration{ | ||
| Node: svc.Node, | ||
| ServiceID: svc.ServiceID, | ||
| Namespace: svc.Namespace, | ||
| }, nil) | ||
| if err != nil { | ||
| // Do not exit right away as there might be other services that need to be deregistered. | ||
| r.Log.Error(err, "failed to deregister service instance", "id", svc.ServiceID) | ||
| errs = multierror.Append(errs, err) | ||
| } else { | ||
| serviceDeregistered = true | ||
| } | ||
| } | ||
|
|
||
| if r.AuthMethod != "" && serviceDeregistered { | ||
| r.Log.Info("reconciling ACL tokens for service", "svc", svc.ServiceName) | ||
| err := r.deleteACLTokensForServiceInstance(apiClient, svc, k8sSvcNamespace, svc.ServiceMeta[constants.MetaKeyPodName], svc.ServiceMeta[constants.MetaKeyPodUID]) | ||
| if err != nil { | ||
| r.Log.Error(err, "failed to reconcile ACL tokens for service", "svc", svc.ServiceName) | ||
| errs = multierror.Append(errs, err) | ||
| } | ||
| } | ||
|
|
||
| if serviceDeregistered { | ||
| err = r.deregisterNode(apiClient, svc.Node) | ||
| if err != nil { | ||
| r.Log.Error(err, "failed to deregister node", "svc", svc.ServiceName) | ||
| errs = multierror.Append(errs, err) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return errs | ||
|
|
||
|
|
@@ -1036,7 +1034,7 @@ func (r *Controller) deregisterNode(apiClient *api.Client, nodeName string) erro | |
| // deleteACLTokensForServiceInstance finds the ACL tokens that belongs to the service instance and deletes it from Consul. | ||
| // It will only check for ACL tokens that have been created with the auth method this controller | ||
| // has been configured with and will only delete tokens for the provided podName and podUID. | ||
| func (r *Controller) deleteACLTokensForServiceInstance(apiClient *api.Client, svc *api.CatalogService, k8sNS, podName, podUID string) error { | ||
| // Skip if podName is empty. | ||
| if podName == "" { | ||
| return nil | ||
|
|
@@ -1049,7 +1047,7 @@ func (r *Controller) deleteACLTokensForServiceInstance(apiClient *api.Client, sv | |
| // matches as well. | ||
| tokens, _, err := apiClient.ACL().TokenListFiltered( | ||
| api.ACLTokenFilterOptions{ | ||
| ServiceName: svc.ServiceName, | ||
| }, | ||
| &api.QueryOptions{ | ||
| Namespace: svc.Namespace, | ||
|
|
@@ -1064,7 +1062,7 @@ func (r *Controller) deleteACLTokensForServiceInstance(apiClient *api.Client, sv | |
| // * have a single service identity whose service name is the same as 'svc.Service' | ||
| if token.AuthMethod == r.AuthMethod && | ||
| len(token.ServiceIdentities) == 1 && | ||
| token.ServiceIdentities[0].ServiceName == svc.ServiceName { | ||
| tokenMeta, err := getTokenMetaFromDescription(token.Description) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to parse token metadata: %s", err) | ||
|
|
@@ -1162,49 +1160,100 @@ func getTokenMetaFromDescription(description string) (map[string]string, error) | |
| return tokenMeta, nil | ||
| } | ||
|
|
||
| func (r *Controller) serviceInstances(apiClient *api.Client, k8sServiceName, k8sServiceNamespace string) ([]*api.CatalogService, error) { | ||
| var ( | ||
| instances []*api.CatalogService | ||
| errs error | ||
| ) | ||
|
|
||
| // Get the names of services that have the provided k8sServiceName and k8sServiceNamespace in their metadata. | ||
| // This is necessary so that we can then list the service instances for each Consul service name, which may | ||
| // not match the K8s service name. | ||
| services, err := r.servicesForK8SServiceNameAndNamespace(apiClient, k8sServiceName, k8sServiceNamespace) | ||
| if err != nil { | ||
| r.Log.Error(err, "failed to get catalog services", "name", k8sServiceName) | ||
| return nil, err | ||
| } | ||
|
|
||
| // Query consul catalog for a list of service instances matching the given service names. | ||
| filter := fmt.Sprintf(`NodeMeta[%q] == %q and ServiceMeta[%q] == %q and ServiceMeta[%q] == %q and ServiceMeta[%q] == %q`, | ||
zalimeni marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| metaKeySyntheticNode, "true", | ||
| metaKeyKubeServiceName, k8sServiceName, | ||
| constants.MetaKeyKubeNS, k8sServiceNamespace, | ||
| metaKeyManagedBy, constants.ManagedByValue) | ||
| for _, service := range services { | ||
| var is []*api.CatalogService | ||
| // Always query the default NS. This ensures that we include mesh gateways, which are always registered to the default NS. | ||
| // It also ensures that during migrations that enable namespaces, we deregister old service instances in the default NS. | ||
| // An alternative approach to this dual query would be a service instance list using the wildcard NS, which would also | ||
| // include instances in Consul namespaces that are no longer in use (e.g. configuration change); this capability does | ||
| // not currently exist in Consul's catalog API (as of 1.17) and would need to first be added. | ||
| // | ||
| // This request uses the service index of the services table (does not perform a full table scan), then decorates each | ||
| // result with a single node fetched by ID index from the nodes table. | ||
| is, _, err = apiClient.Catalog().Service(service, "", &api.QueryOptions{Filter: filter}) | ||
| if err != nil { | ||
| errs = multierror.Append(errs, err) | ||
| } else { | ||
| instances = append(instances, is...) | ||
| } | ||
| // If namespaces are enabled a non-default NS is targeted, also query by target Consul NS. | ||
| if r.EnableConsulNamespaces { | ||
| nonDefaultNamespace := namespaces.NonDefaultConsulNamespace(r.consulNamespace(k8sServiceNamespace)) | ||
| if nonDefaultNamespace != "" { | ||
hashi-derek marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| is, _, err = apiClient.Catalog().Service(service, "", &api.QueryOptions{Filter: filter, Namespace: nonDefaultNamespace}) | ||
| if err != nil { | ||
| errs = multierror.Append(errs, err) | ||
| } else { | ||
| instances = append(instances, is...) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return instances, errs | ||
| } | ||
|
|
||
| // servicesForK8SServiceNameAndNamespace calls Consul's Services to get the list | ||
| // of services that have the provided k8sServiceName and k8sServiceNamespace in their metadata. | ||
| func (r *Controller) servicesForK8SServiceNameAndNamespace(apiClient *api.Client, k8sServiceName, k8sServiceNamespace string) ([]string, error) { | ||
| var ( | ||
| services map[string][]string | ||
| err error | ||
| ) | ||
| filter := fmt.Sprintf(`ServiceMeta[%q] == %q and ServiceMeta[%q] == %q and ServiceMeta[%q] == %q`, | ||
|
Comment on lines
-1200
to
+1224
There was a problem hiding this comment. Same filter, different model field. I checked agent/consul/catalog_endpoint.go and If we find the new queries are not performant enough, we have the option of adding an index for |
||
| metaKeyKubeServiceName, k8sServiceName, constants.MetaKeyKubeNS, k8sServiceNamespace, metaKeyManagedBy, constants.ManagedByValue) | ||
| // Always query the default NS. This ensures that we cover CE->Ent upgrades where services were previously | ||
| // in the default NS, as well as mesh gateways, which are always registered to the default NS. | ||
| // | ||
| // This request performs a NS-bound scan of the services table. If needed in the future, its performance | ||
| // could be improved by adding an index on ServiceMeta to Consul's state store. | ||
| services, _, err = apiClient.Catalog().Services(&api.QueryOptions{Filter: filter}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // If namespaces are enabled a non-default NS is targeted, also query by target Consul NS. | ||
| if r.EnableConsulNamespaces { | ||
| nonDefaultNamespace := namespaces.NonDefaultConsulNamespace(r.consulNamespace(k8sServiceNamespace)) | ||
| if nonDefaultNamespace != "" { | ||
| ss, _, err := apiClient.Catalog().Services(&api.QueryOptions{Filter: filter, Namespace: nonDefaultNamespace}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // Add to existing map to deduplicate. | ||
| for s := range ss { | ||
| services[s] = nil // We don't use the tags, so just set to nil | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Return just the service name keys (we don't need the tags) | ||
| // https://developer.hashicorp.com/consul/api-docs/catalog#list-services | ||
| var serviceNames []string | ||
| for s := range services { | ||
| serviceNames = append(serviceNames, s) | ||
| } | ||
| return serviceNames, err | ||
| } | ||
|
|
||
| // processPreparedQueryUpstream processes an upstream in the format: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Most of the above code is unchanged. The GH diff is easier to read with ?w=1.
The changes are:
CatalogNodeServiceList.Node.Node->ServiceNode.NodeNodeService.ID->ServiceNode.ServiceIDNodeService.Service->ServiceNode.ServiceNameNodeService.Meta->ServiceNode.ServiceMetaNodeService.Address->ServiceNode.ServiceAddressHere's a visual of the diff of the two models read locally from a multi-node
kindcluster (the sidecar proxy is similar and carries identicalNodeMetaandServiceMeta, which is what we filter on):