-
Notifications
You must be signed in to change notification settings - Fork 4.6k
internal/xds: move the LDS and RDS watchers to dependency manager #8651
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
fbfb4b1
66270ee
d28cb9e
d80ee7a
3502dbf
b696279
ed60964
fff6013
58478e6
c279455
1457ea2
45742dc
c0951c3
b528518
19adb95
4eab78a
ffa4926
d623feb
ee48dfa
22e7fdc
80f2e99
d6e1b25
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| /* | ||
| * | ||
| * Copyright 2025 gRPC authors. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package xdsresource | ||
|
|
||
| import "google.golang.org/grpc/resolver" | ||
|
|
||
| // XDSConfig holds the complete gRPC client-side xDS configuration containing | ||
| // all necessary resources. | ||
| type XDSConfig struct { | ||
arjan-bal marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| // Listener holds the listener configuration. It is guaranteed to be | ||
| // non-nil. | ||
| Listener *ListenerUpdate | ||
|
|
||
| // RouteConfig holds the route configuration. It will be populated even if | ||
| // the route configuration was inlined into the Listener resource. It is | ||
| // guaranteed to be non-nil. | ||
| RouteConfig *RouteConfigUpdate | ||
|
|
||
| // VirtualHost is selected from the route configuration whose domain field | ||
| // offers the best match against the provided dataplane authority. It is | ||
| // guaranteed to be non-nil. | ||
| VirtualHost *VirtualHost | ||
|
|
||
| // Clusters is a map from cluster name to its configuration. | ||
| Clusters map[string]*ClusterResult | ||
| } | ||
|
|
||
| // ClusterResult contains a cluster's configuration when a valid resource is | ||
| // received from the management server. It contains an error when: | ||
| // - an invalid resource is received from the management server and | ||
| // a valid resource was not already present or | ||
| // - the cluster resource does not exist on the management server | ||
| type ClusterResult struct { | ||
| Config ClusterConfig | ||
| Err error | ||
| } | ||
|
|
||
| // ClusterConfig contains configuration for a single cluster. | ||
| type ClusterConfig struct { | ||
| // Cluster configuration for the cluster. This field is always set to a | ||
| // non-nil value. | ||
| Cluster *ClusterUpdate | ||
| // EndpointConfig contains endpoint configuration for a leaf cluster. This | ||
| // field is only set for EDS and LOGICAL_DNS clusters. | ||
| EndpointConfig *EndpointConfig | ||
| // AggregateConfig contains configuration for an aggregate cluster. This | ||
| // field is only set for AGGREGATE clusters. | ||
| AggregateConfig *AggregateConfig | ||
| } | ||
|
|
||
| // AggregateConfig holds the configuration for an aggregate cluster. | ||
| type AggregateConfig struct { | ||
| // LeafClusters contains a prioritized list of names of the leaf clusters | ||
| // for the cluster. | ||
| LeafClusters []string | ||
arjan-bal marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| // EndpointConfig contains configuration corresponding to the endpoints in a | ||
| // cluster. Only one of EDSUpdate or DNSEndpoints will be populated based on the | ||
| // cluster type. | ||
| type EndpointConfig struct { | ||
| // Endpoint configurartion for the EDS clusters. | ||
| EDSUpdate *EndpointsUpdate | ||
| // Endpoint configuration for the LOGICAL_DNS clusters. | ||
| DNSEndpoints *DNSUpdate | ||
| // ResolutionNote stores error encountered while obtaining endpoints data for the cluster. It may contain a nil value when a valid endpoint datais received. It contains an error when: | ||
| // - an invalid resource is received from the management server or | ||
| // - the endpoint resource does not exist on the management server | ||
| ResolutionNote error | ||
| } | ||
|
|
||
| // DNSUpdate represents the result of a DNS resolution, containing a list of | ||
| // discovered endpoints. This is only populated for the LOGICAL_DNS clusters. | ||
| type DNSUpdate struct { | ||
| // Endpoints is the complete list of endpoints returned by the DNS resolver. | ||
| Endpoints []resolver.Endpoint | ||
| } | ||
|
|
||
| // xdsConfigkey is the type used as the key to store XDSConfig in the Attributes | ||
| // field of resolver.State. | ||
| type xdsConfigkey struct{} | ||
|
|
||
| // SetXDSConfig returns a copy of state in which the Attributes field is updated | ||
| // with the XDSConfig. | ||
| func SetXDSConfig(state resolver.State, config *XDSConfig) resolver.State { | ||
| state.Attributes = state.Attributes.WithValue(xdsConfigkey{}, config) | ||
| return state | ||
| } | ||
|
|
||
| // XDSConfigFromResolverState returns XDSConfig stored as an attribute in the | ||
| // resolver state. | ||
| func XDSConfigFromResolverState(state resolver.State) *XDSConfig { | ||
| state.Attributes.Value(xdsConfigkey{}) | ||
| if v := state.Attributes.Value(xdsConfigkey{}); v != nil { | ||
| return v.(*XDSConfig) | ||
| } | ||
| return nil | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| /* | ||
| * | ||
| * Copyright 2025 gRPC authors. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package xdsdepmgr | ||
|
|
||
| import ( | ||
| "sync/atomic" | ||
|
|
||
| "google.golang.org/grpc/internal/xds/xdsclient/xdsresource" | ||
| ) | ||
|
|
||
| type listenerWatcher struct { | ||
| resourceName string | ||
| cancel func() | ||
| depMgr *DependencyManager | ||
| stopped atomic.Bool | ||
| } | ||
|
|
||
| func newListenerWatcher(resourceName string, depMgr *DependencyManager) *listenerWatcher { | ||
| lw := &listenerWatcher{resourceName: resourceName, depMgr: depMgr} | ||
| lw.cancel = xdsresource.WatchListener(depMgr.xdsClient, resourceName, lw) | ||
| return lw | ||
| } | ||
|
|
||
| func (l *listenerWatcher) ResourceChanged(update *xdsresource.ListenerUpdate, onDone func()) { | ||
| if l.stopped.Load() { | ||
| onDone() | ||
| return | ||
| } | ||
| l.depMgr.onListenerResourceUpdate(update, onDone) | ||
| } | ||
|
|
||
| func (l *listenerWatcher) ResourceError(err error, onDone func()) { | ||
| if l.stopped.Load() { | ||
| onDone() | ||
| return | ||
| } | ||
| l.depMgr.onListenerResourceError(err, onDone) | ||
| } | ||
|
|
||
| func (l *listenerWatcher) AmbientError(err error, onDone func()) { | ||
| if l.stopped.Load() { | ||
| onDone() | ||
| return | ||
| } | ||
| l.depMgr.onListenerResourceAmbientError(err, onDone) | ||
| } | ||
|
|
||
| func (l *listenerWatcher) stop() { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is there a guarantee that stop is not called concurrently with
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The only time
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Now, again coming back to the point of not needing the serializer. This means that access to this field needs to be guarded by a mutex or make it an atomic.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. Changed to an atomic. Also after the change it is not guaranteed to be called concurrently anymore since we removed the serializer. So I have removed the comment too.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The code is checking
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does the xds client guarantee that it will not call watchers after they have unsubscribed? If it does, do we really need an
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. My understanding is that the main intention behind the code in dependency manager that started the conversation about this change was we were discarding the update from the cancelled watcher (by checking if it is the current name we got in the listener) because it could be the case where the old RDS watcher already got an update before we could cancel the watch completely but since we have a new route name in LDS, we do not want to use that update anymore and just discard it. This change will ensure that as soon as we cancel it, even if we get an update before we get a chance to unwatch the resource, it will not be used.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
This is the docstring from the
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think an atomic is sufficient here. We need to think about "what guarantee does the Usually, the guarantee is that once Does this guarantee hold with an atomic? Consider the following:
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. One solution is to use a mutex that needs to be held while making calls into the I haven't looked into the implementation of the XDSClient to see how the race mentioned in the comment is triggered. Maybe it's possible to prevent the race in the |
||
| l.stopped.Store(true) | ||
| l.cancel() | ||
| if l.depMgr.logger.V(2) { | ||
| l.depMgr.logger.Infof("Canceling watch on Listener resource %q", l.resourceName) | ||
| } | ||
| } | ||
|
|
||
| type routeConfigWatcher struct { | ||
| resourceName string | ||
| cancel func() | ||
| depMgr *DependencyManager | ||
| stopped atomic.Bool | ||
| } | ||
|
|
||
| func newRouteConfigWatcher(resourceName string, depMgr *DependencyManager) *routeConfigWatcher { | ||
| rw := &routeConfigWatcher{resourceName: resourceName, depMgr: depMgr} | ||
| rw.cancel = xdsresource.WatchRouteConfig(depMgr.xdsClient, resourceName, rw) | ||
| return rw | ||
| } | ||
|
|
||
| func (r *routeConfigWatcher) ResourceChanged(u *xdsresource.RouteConfigUpdate, onDone func()) { | ||
| if r.stopped.Load() { | ||
| onDone() | ||
| return | ||
| } | ||
| r.depMgr.onRouteConfigResourceUpdate(r.resourceName, u, onDone) | ||
| } | ||
|
|
||
| func (r *routeConfigWatcher) ResourceError(err error, onDone func()) { | ||
| if r.stopped.Load() { | ||
| onDone() | ||
| return | ||
| } | ||
| r.depMgr.onRouteConfigResourceError(r.resourceName, err, onDone) | ||
| } | ||
|
|
||
| func (r *routeConfigWatcher) AmbientError(err error, onDone func()) { | ||
| if r.stopped.Load() { | ||
| onDone() | ||
| return | ||
| } | ||
| r.depMgr.onRouteConfigResourceAmbientError(r.resourceName, err, onDone) | ||
| } | ||
|
|
||
| func (r *routeConfigWatcher) stop() { | ||
| r.stopped.Store(true) | ||
| r.cancel() | ||
| if r.depMgr.logger.V(2) { | ||
| r.depMgr.logger.Infof("Canceling watch on RouteConfiguration resource %q", r.resourceName) | ||
| } | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.