diff --git a/README.md b/README.md index 1f0a2387f..1cfaf1d7d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,12 @@ in [data-plane-api](https://github.com/envoyproxy/data-plane-api). It started li [go-control-plane](https://github.com/envoyproxy/go-control-plane), but building an idiomatic Java implementation is prioritized over exact interface parity with the Go implementation. +Both v2 and v3 resources as well as transport versions are supported. Migrating +to v3 is recommended as Envoy will drop v2 support at EOY 2020 (see +[API_VERSIONING.md](https://github.com/envoyproxy/envoy/blob/4c6206865061591155d18b55972b4d626e1703dd/api/API_VERSIONING.md)) + +See the (v2-to-v3 migration guide)[V2_TO_V3_GUIDE.md] for an exmplanation of migration paths. + ### Requirements 1. Java 8+ diff --git a/V2_TO_V3_GUIDE.md b/V2_TO_V3_GUIDE.md new file mode 100644 index 000000000..e7b92240a --- /dev/null +++ b/V2_TO_V3_GUIDE.md @@ -0,0 +1,41 @@ +# Migrating from xDS v2 to v3 + +To faciliate migrating from the v2 xDS APIs to v3, this repo supports both the +v2 and v3 gRPC transports, with each transport supporting type URL rewriting of +DiscoveryResponse to whatever version the client requests with +api_resource_version. + +The migration requires care - for example, using v3-only fields too soon or trying to use +deprecated v2 fields too late can cause Envoy to reject or improperly apply config. + +### Recommended Sequence + +This section assumes you have sufficient control over Envoy sidecar versions that you do +not need to run v2 and v3 simultaneously for a long migration period. + +1. Make sure your oldest Envoy client supports final v2 message versions. +2. + 1. Ensure your control plane is not using any deprecated v2 fields. + Deprecated v2 fields will cause errors when they are translated to v3 + (because deprecated v2 fields are dropped in v3). + 2. Configure a V3DiscoveryServer alongside the V2DiscoveryServer in your + control plane. You can (and should) use the same (v2) Cache implementation + in both servers. +3. Deploy all Envoy clients to switch to both the v3 transport_api_version and + resource_api_version in each respective xDS configs. As this happens, the V3DiscoveryServer + will be translating your v2 resources to v3 automatically, and the V2DiscoveryServer will + stop being used. +4. + 1. Rewrite your control plane code to use v3 resources, which means using + V3SimpleCache (if you use SimpleCache). You may now start using v3-only + message fields if you choose. + 2. Drop the V2DiscoveryServer. + +### Alternative + +Another possible path to the one above is to switch to generating v3 in the +control plane first (e.g. by using V3SimpleCache) and then deploying Envoy +clients to use v3 transport and resource versions. + +This approach requires care to not use new V3-only fields until the client side +upgrade is complete (or at least understand the consequences of doing so). diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/Cache.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/Cache.java index 39a1196dc..34529c1db 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/Cache.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/Cache.java @@ -1,6 +1,5 @@ package io.envoyproxy.controlplane.cache; -import io.envoyproxy.envoy.api.v2.core.Node; import java.util.Collection; import javax.annotation.concurrent.ThreadSafe; @@ -11,13 +10,13 @@ public interface Cache extends ConfigWatcher { /** - * Returns all known {@link Node} groups. + * Returns all known groups. * */ Collection groups(); /** - * Returns the current {@link StatusInfo} for the given {@link Node} group. + * Returns the current {@link StatusInfo} for the given group. * * @param group the node group whose status is being fetched */ diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/ConfigWatcher.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/ConfigWatcher.java index f4bff03ee..fc94fe8d3 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/ConfigWatcher.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/ConfigWatcher.java @@ -1,6 +1,5 @@ package io.envoyproxy.controlplane.cache; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; import java.util.Set; import java.util.function.Consumer; import javax.annotation.concurrent.ThreadSafe; @@ -25,7 +24,7 @@ public interface ConfigWatcher { */ Watch createWatch( boolean ads, - DiscoveryRequest request, + XdsRequest request, Set knownResourceNames, Consumer responseConsumer, boolean hasClusterChanged); diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/NodeGroup.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/NodeGroup.java index 45c9b462f..ea8959812 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/NodeGroup.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/NodeGroup.java @@ -15,4 +15,11 @@ public interface NodeGroup { * @param node identifier for the envoy instance that is requesting config */ T hash(Node node); + + /** + * Returns a consistent identifier of the given {@link io.envoyproxy.envoy.config.core.v3.Node}. + * + * @param node identifier for the envoy instance that is requesting config + */ + T hash(io.envoyproxy.envoy.config.core.v3.Node node); } diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/Resources.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/Resources.java index cad6dfe9f..98ba25bd0 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/Resources.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/Resources.java @@ -1,6 +1,13 @@ package io.envoyproxy.controlplane.cache; import static com.google.common.base.Strings.isNullOrEmpty; +import static io.envoyproxy.controlplane.cache.Resources.ApiVersion.V2; +import static io.envoyproxy.controlplane.cache.Resources.ApiVersion.V3; +import static io.envoyproxy.controlplane.cache.Resources.ResourceType.CLUSTER; +import static io.envoyproxy.controlplane.cache.Resources.ResourceType.ENDPOINT; +import static io.envoyproxy.controlplane.cache.Resources.ResourceType.LISTENER; +import static io.envoyproxy.controlplane.cache.Resources.ResourceType.ROUTE; +import static io.envoyproxy.controlplane.cache.Resources.ResourceType.SECRET; import static io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager.RouteSpecifierCase.RDS; import com.google.common.base.Preconditions; @@ -30,33 +37,113 @@ public class Resources { + /** + * Version-agnostic representation of a resource. This is useful when the version qualifier + * isn't needed. + */ + public enum ResourceType { + CLUSTER, + ENDPOINT, + LISTENER, + ROUTE, + SECRET + } + + public enum ApiVersion { + V2, + V3 + } + private static final Logger LOGGER = LoggerFactory.getLogger(Resources.class); static final String FILTER_ENVOY_ROUTER = "envoy.router"; static final String FILTER_HTTP_CONNECTION_MANAGER = "envoy.http_connection_manager"; - private static final String TYPE_URL_PREFIX = "type.googleapis.com/envoy.api.v2."; - - public static final String CLUSTER_TYPE_URL = TYPE_URL_PREFIX + "Cluster"; - public static final String ENDPOINT_TYPE_URL = TYPE_URL_PREFIX + "ClusterLoadAssignment"; - public static final String LISTENER_TYPE_URL = TYPE_URL_PREFIX + "Listener"; - public static final String ROUTE_TYPE_URL = TYPE_URL_PREFIX + "RouteConfiguration"; - public static final String SECRET_TYPE_URL = TYPE_URL_PREFIX + "auth.Secret"; - - public static final List TYPE_URLS = ImmutableList.of( - CLUSTER_TYPE_URL, - ENDPOINT_TYPE_URL, - LISTENER_TYPE_URL, - ROUTE_TYPE_URL, - SECRET_TYPE_URL); - - public static final Map> RESOURCE_TYPE_BY_URL = ImmutableMap.of( - CLUSTER_TYPE_URL, Cluster.class, - ENDPOINT_TYPE_URL, ClusterLoadAssignment.class, - LISTENER_TYPE_URL, Listener.class, - ROUTE_TYPE_URL, RouteConfiguration.class, - SECRET_TYPE_URL, Secret.class - ); + public static class V2 { + private static final String TYPE_URL_PREFIX = "type.googleapis.com/envoy.api.v2."; + public static final String SECRET_TYPE_URL = TYPE_URL_PREFIX + "auth.Secret"; + public static final String ROUTE_TYPE_URL = TYPE_URL_PREFIX + "RouteConfiguration"; + public static final String LISTENER_TYPE_URL = TYPE_URL_PREFIX + "Listener"; + public static final String ENDPOINT_TYPE_URL = TYPE_URL_PREFIX + "ClusterLoadAssignment"; + public static final String CLUSTER_TYPE_URL = TYPE_URL_PREFIX + "Cluster"; + + public static final List TYPE_URLS = ImmutableList.of( + CLUSTER_TYPE_URL, + ENDPOINT_TYPE_URL, + LISTENER_TYPE_URL, + ROUTE_TYPE_URL, + SECRET_TYPE_URL); + } + + public static class V3 { + + public static final String CLUSTER_TYPE_URL = "type.googleapis.com/envoy.config.cluster.v3" + + ".Cluster"; + public static final String ENDPOINT_TYPE_URL = "type.googleapis.com/envoy.config.endpoint.v3" + + ".ClusterLoadAssignment"; + public static final String LISTENER_TYPE_URL = "type.googleapis.com/envoy.config.listener.v3" + + ".Listener"; + public static final String ROUTE_TYPE_URL = "type.googleapis.com/envoy.config.route.v3" + + ".RouteConfiguration"; + public static final String SECRET_TYPE_URL = "type.googleapis.com/envoy.extensions" + + ".transport_sockets.tls.v3.Secret"; + + public static final List TYPE_URLS = ImmutableList.of( + CLUSTER_TYPE_URL, + ENDPOINT_TYPE_URL, + LISTENER_TYPE_URL, + ROUTE_TYPE_URL, + SECRET_TYPE_URL); + } + + public static final List RESOURCE_TYPES_IN_ORDER = ImmutableList.of( + CLUSTER, + ENDPOINT, + LISTENER, + ROUTE, + SECRET); + + public static final Map V3_TYPE_URLS_TO_V2 = ImmutableMap.of( + Resources.V3.CLUSTER_TYPE_URL, Resources.V2.CLUSTER_TYPE_URL, + Resources.V3.ENDPOINT_TYPE_URL, Resources.V2.ENDPOINT_TYPE_URL, + Resources.V3.LISTENER_TYPE_URL, Resources.V2.LISTENER_TYPE_URL, + Resources.V3.ROUTE_TYPE_URL, Resources.V2.ROUTE_TYPE_URL, + Resources.V3.SECRET_TYPE_URL, Resources.V2.SECRET_TYPE_URL); + + public static final Map V2_TYPE_URLS_TO_V3 = ImmutableMap.of( + Resources.V2.CLUSTER_TYPE_URL, Resources.V3.CLUSTER_TYPE_URL, + Resources.V2.ENDPOINT_TYPE_URL, Resources.V3.ENDPOINT_TYPE_URL, + Resources.V2.LISTENER_TYPE_URL, Resources.V3.LISTENER_TYPE_URL, + Resources.V2.ROUTE_TYPE_URL, Resources.V3.ROUTE_TYPE_URL, + Resources.V2.SECRET_TYPE_URL, Resources.V3.SECRET_TYPE_URL); + + public static final Map TYPE_URLS_TO_RESOURCE_TYPE = + new ImmutableMap.Builder() + .put(Resources.V3.CLUSTER_TYPE_URL, CLUSTER) + .put(Resources.V2.CLUSTER_TYPE_URL, CLUSTER) + .put(Resources.V3.ENDPOINT_TYPE_URL, ENDPOINT) + .put(Resources.V2.ENDPOINT_TYPE_URL, ENDPOINT) + .put(Resources.V3.LISTENER_TYPE_URL, LISTENER) + .put(Resources.V2.LISTENER_TYPE_URL, LISTENER) + .put(Resources.V3.ROUTE_TYPE_URL, ROUTE) + .put(Resources.V2.ROUTE_TYPE_URL, ROUTE) + .put(Resources.V3.SECRET_TYPE_URL, SECRET) + .put(Resources.V2.SECRET_TYPE_URL, SECRET) + .build(); + + public static final Map> RESOURCE_TYPE_BY_URL = + new ImmutableMap.Builder>() + .put(Resources.V2.CLUSTER_TYPE_URL, Cluster.class) + .put(Resources.V2.ENDPOINT_TYPE_URL, ClusterLoadAssignment.class) + .put(Resources.V2.LISTENER_TYPE_URL, Listener.class) + .put(Resources.V2.ROUTE_TYPE_URL, RouteConfiguration.class) + .put(Resources.V2.SECRET_TYPE_URL, Secret.class) + .put(Resources.V3.CLUSTER_TYPE_URL, io.envoyproxy.envoy.config.cluster.v3.Cluster.class) + .put(Resources.V3.ENDPOINT_TYPE_URL, io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.class) + .put(Resources.V3.LISTENER_TYPE_URL, io.envoyproxy.envoy.config.listener.v3.Listener.class) + .put(Resources.V3.ROUTE_TYPE_URL, io.envoyproxy.envoy.config.route.v3.RouteConfiguration.class) + .put(Resources.V3.SECRET_TYPE_URL, io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret.class) + .build(); /** * Returns the name of the given resource message. @@ -84,6 +171,26 @@ public static String getResourceName(Message resource) { return ((Secret) resource).getName(); } + if (resource instanceof io.envoyproxy.envoy.config.cluster.v3.Cluster) { + return ((io.envoyproxy.envoy.config.cluster.v3.Cluster) resource).getName(); + } + + if (resource instanceof io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment) { + return ((io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment) resource).getClusterName(); + } + + if (resource instanceof io.envoyproxy.envoy.config.listener.v3.Listener) { + return ((io.envoyproxy.envoy.config.listener.v3.Listener) resource).getName(); + } + + if (resource instanceof io.envoyproxy.envoy.config.route.v3.RouteConfiguration) { + return ((io.envoyproxy.envoy.config.route.v3.RouteConfiguration) resource).getName(); + } + + if (resource instanceof io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret) { + return ((io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret) resource).getName(); + } + return ""; } @@ -133,6 +240,17 @@ public static Set getResourceReferences(Collection re refs.add(c.getName()); } } + } else if (r instanceof io.envoyproxy.envoy.config.cluster.v3.Cluster) { + io.envoyproxy.envoy.config.cluster.v3.Cluster c = (io.envoyproxy.envoy.config.cluster.v3.Cluster) r; + + // For EDS clusters, use the cluster name or the service name override. + if (c.getType() == io.envoyproxy.envoy.config.cluster.v3.Cluster.DiscoveryType.EDS) { + if (!isNullOrEmpty(c.getEdsClusterConfig().getServiceName())) { + refs.add(c.getEdsClusterConfig().getServiceName()); + } else { + refs.add(c.getName()); + } + } } else if (r instanceof Listener) { Listener l = (Listener) r; @@ -144,17 +262,56 @@ public static Set getResourceReferences(Collection re } try { - HttpConnectionManager.Builder config = HttpConnectionManager.newBuilder(); - - // TODO: Filter#getConfig() is deprecated, migrate to use Filter#getTypedConfig(). - structAsMessage(filter.getConfig(), config); + HttpConnectionManager config; + + if (filter.hasTypedConfig()) { + config = filter.getTypedConfig().unpack(HttpConnectionManager.class); + } else { + HttpConnectionManager.Builder builder = HttpConnectionManager.newBuilder(); + structAsMessage(filter.getConfig(), builder); + config = builder.build(); + } if (config.getRouteSpecifierCase() == RDS && !isNullOrEmpty(config.getRds().getRouteConfigName())) { refs.add(config.getRds().getRouteConfigName()); } } catch (InvalidProtocolBufferException e) { LOGGER.error( - "Failed to convert HTTP connection manager config struct into protobuf message for listener {}", + "Failed to convert v2 HTTP connection manager config struct into protobuf " + + "message for listener {}", + getResourceName(l), + e); + } + } + } + } else if (r instanceof io.envoyproxy.envoy.config.listener.v3.Listener) { + + io.envoyproxy.envoy.config.listener.v3.Listener l = + (io.envoyproxy.envoy.config.listener.v3.Listener) r; + + // Extract the route configuration names from the HTTP connection manager. + for (io.envoyproxy.envoy.config.listener.v3.FilterChain chain : l.getFilterChainsList()) { + for (io.envoyproxy.envoy.config.listener.v3.Filter filter : chain.getFiltersList()) { + if (!filter.getName().equals(FILTER_HTTP_CONNECTION_MANAGER)) { + continue; + } + + try { + io.envoyproxy.envoy.extensions.filters.network + .http_connection_manager.v3.HttpConnectionManager config = filter + .getTypedConfig().unpack( + io.envoyproxy.envoy.extensions.filters.network + .http_connection_manager.v3.HttpConnectionManager.class); + + if (config.getRouteSpecifierCase() == io.envoyproxy.envoy.extensions.filters.network + .http_connection_manager.v3.HttpConnectionManager.RouteSpecifierCase.RDS + && !isNullOrEmpty(config.getRds().getRouteConfigName())) { + refs.add(config.getRds().getRouteConfigName()); + } + } catch (InvalidProtocolBufferException e) { + LOGGER.error( + "Failed to convert v3 HTTP connection manager config struct into protobuf " + + "message for listener {}", getResourceName(l), e); } @@ -166,6 +323,19 @@ public static Set getResourceReferences(Collection re return refs.build(); } + /** + * Returns the API version (v2 or v3) for a given type URL. + */ + public static ApiVersion getResourceApiVersion(String typeUrl) { + if (Resources.V2.TYPE_URLS.contains(typeUrl)) { + return V2; + } else if (Resources.V3.TYPE_URLS.contains(typeUrl)) { + return V3; + } + + throw new RuntimeException(String.format("Unsupported API version for type URL %s", typeUrl)); + } + private static void structAsMessage(Struct struct, Message.Builder messageBuilder) throws InvalidProtocolBufferException { diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/Response.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/Response.java index 537485b11..9bf161fc1 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/Response.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/Response.java @@ -2,7 +2,6 @@ import com.google.auto.value.AutoValue; import com.google.protobuf.Message; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; import java.util.Collection; /** @@ -11,14 +10,15 @@ @AutoValue public abstract class Response { - public static Response create(DiscoveryRequest request, Collection resources, String version) { + public static Response create(XdsRequest request, Collection resources, + String version) { return new AutoValue_Response(request, resources, version); } /** * Returns the original request associated with the response. */ - public abstract DiscoveryRequest request(); + public abstract XdsRequest request(); /** * Returns the resources to include in the response. diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/SimpleCache.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/SimpleCache.java index e7b255743..141cf5858 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/SimpleCache.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/SimpleCache.java @@ -1,10 +1,13 @@ package io.envoyproxy.controlplane.cache; +import static io.envoyproxy.controlplane.cache.Resources.RESOURCE_TYPES_IN_ORDER; + import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.protobuf.Message; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; +import io.envoyproxy.controlplane.cache.Resources.ResourceType; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -31,7 +34,7 @@ * *

The snapshot can be partial, e.g. only include RDS or EDS resources. */ -public class SimpleCache implements SnapshotCache { +public abstract class SimpleCache implements SnapshotCache { private static final Logger LOGGER = LoggerFactory.getLogger(SimpleCache.class); @@ -42,8 +45,9 @@ public class SimpleCache implements SnapshotCache { private final Lock writeLock = lock.writeLock(); @GuardedBy("lock") - private final Map snapshots = new HashMap<>(); - private final ConcurrentMap>> statuses = new ConcurrentHashMap<>(); + private final Map snapshots = new HashMap<>(); + private final ConcurrentMap>> statuses = + new ConcurrentHashMap<>(); private AtomicLong watchCount = new AtomicLong(); @@ -52,7 +56,7 @@ public class SimpleCache implements SnapshotCache { * * @param groups maps an envoy host to a node group */ - public SimpleCache(NodeGroup groups) { + protected SimpleCache(NodeGroup groups) { this.groups = groups; } @@ -64,7 +68,7 @@ public boolean clearSnapshot(T group) { // we take a writeLock to prevent watches from being created writeLock.lock(); try { - Map> status = statuses.get(group); + Map> status = statuses.get(group); // If we don't know about this group, do nothing. if (status != null && status.values().stream().mapToLong(CacheStatusInfo::numWatches).sum() > 0) { @@ -84,7 +88,7 @@ public boolean clearSnapshot(T group) { public Watch createWatch( boolean ads, - DiscoveryRequest request, + XdsRequest request, Set knownResourceNames, Consumer responseConsumer) { return createWatch(ads, request, knownResourceNames, responseConsumer, false); @@ -96,22 +100,31 @@ public Watch createWatch( @Override public Watch createWatch( boolean ads, - DiscoveryRequest request, + XdsRequest request, Set knownResourceNames, Consumer responseConsumer, boolean hasClusterChanged) { + ResourceType requestResourceType = request.getResourceType(); + Preconditions.checkNotNull(requestResourceType, "unsupported type URL %s", + request.getTypeUrl()); + T group; + if (request.v3Request() != null) { + group = groups.hash(request.v3Request().getNode()); + } else { + group = groups.hash(request.v2Request().getNode()); + } - T group = groups.hash(request.getNode()); // even though we're modifying, we take a readLock to allow multiple watches to be created in parallel since it // doesn't conflict readLock.lock(); try { CacheStatusInfo status = statuses.computeIfAbsent(group, g -> new ConcurrentHashMap<>()) - .computeIfAbsent(request.getTypeUrl(), s -> new CacheStatusInfo<>(group)); + .computeIfAbsent(requestResourceType, s -> new CacheStatusInfo<>(group)); status.setLastWatchRequestTime(System.currentTimeMillis()); - Snapshot snapshot = snapshots.get(group); - String version = snapshot == null ? "" : snapshot.version(request.getTypeUrl(), request.getResourceNamesList()); + U snapshot = snapshots.get(group); + String version = snapshot == null ? "" : snapshot.version(requestResourceType, + request.getResourceNamesList()); Watch watch = new Watch(ads, request, responseConsumer); @@ -124,7 +137,7 @@ public Watch createWatch( // If any of the newly requested resources are in the snapshot respond immediately. If not we'll fall back to // version comparisons. - if (snapshot.resources(request.getTypeUrl()) + if (snapshot.resources(requestResourceType) .keySet() .stream() .anyMatch(newResourceHints::contains)) { @@ -132,7 +145,8 @@ public Watch createWatch( return watch; } - } else if (hasClusterChanged && request.getTypeUrl().equals(Resources.ENDPOINT_TYPE_URL)) { + } else if (hasClusterChanged + && (requestResourceType.equals(ResourceType.ENDPOINT))) { respond(watch, snapshot, group); return watch; @@ -189,7 +203,7 @@ public Watch createWatch( * {@inheritDoc} */ @Override - public Snapshot getSnapshot(T group) { + public U getSnapshot(T group) { readLock.lock(); try { @@ -211,9 +225,9 @@ public Collection groups() { * {@inheritDoc} */ @Override - public synchronized void setSnapshot(T group, Snapshot snapshot) { + public synchronized void setSnapshot(T group, U snapshot) { // we take a writeLock to prevent watches from being created while we update the snapshot - ConcurrentMap> status; + ConcurrentMap> status; writeLock.lock(); try { // Update the existing snapshot entry. @@ -227,7 +241,8 @@ public synchronized void setSnapshot(T group, Snapshot snapshot) { return; } - // Responses should be in specific order and TYPE_URLS has a list of resources in the right order. + // Responses should be in specific order and typeUrls has a list of resources in the right + // order. respondWithSpecificOrder(group, snapshot, status); } @@ -239,7 +254,7 @@ public StatusInfo statusInfo(T group) { readLock.lock(); try { - ConcurrentMap> statusMap = statuses.get(group); + ConcurrentMap> statusMap = statuses.get(group); if (statusMap == null || statusMap.isEmpty()) { return null; } @@ -251,19 +266,21 @@ public StatusInfo statusInfo(T group) { } @VisibleForTesting - protected void respondWithSpecificOrder(T group, Snapshot snapshot, - ConcurrentMap> statusMap) { - for (String typeUrl : Resources.TYPE_URLS) { - CacheStatusInfo status = statusMap.get(typeUrl); + protected void respondWithSpecificOrder(T group, + U snapshot, + ConcurrentMap> statusMap) { + for (ResourceType resourceType : RESOURCE_TYPES_IN_ORDER) { + CacheStatusInfo status = statusMap.get(resourceType); if (status == null) { continue; } status.watchesRemoveIf((id, watch) -> { - if (!watch.request().getTypeUrl().equals(typeUrl)) { + if (!watch.request().getResourceType().equals(resourceType)) { return false; } - String version = snapshot.version(watch.request().getTypeUrl(), watch.request().getResourceNamesList()); + String version = snapshot.version(watch.request().getResourceType(), + watch.request().getResourceNamesList()); if (!watch.request().getVersionInfo().equals(version)) { if (LOGGER.isDebugEnabled()) { @@ -285,7 +302,8 @@ protected void respondWithSpecificOrder(T group, Snapshot snapshot, } } - private Response createResponse(DiscoveryRequest request, Map resources, String version) { + private Response createResponse(XdsRequest request, Map resources, + String version) { Collection filtered = request.getResourceNamesList().isEmpty() ? resources.values() : request.getResourceNamesList().stream() @@ -296,8 +314,8 @@ private Response createResponse(DiscoveryRequest request, Map snapshotResources = snapshot.resources(watch.request().getTypeUrl()); + private boolean respond(Watch watch, U snapshot, T group) { + Map snapshotResources = snapshot.resources(watch.request().getResourceType()); if (!watch.request().getResourceNamesList().isEmpty() && watch.ads()) { Collection missingNames = watch.request().getResourceNamesList().stream() @@ -309,7 +327,7 @@ private boolean respond(Watch watch, Snapshot snapshot, T group) { "not responding in ADS mode for {} from node {} at version {} for request [{}] since [{}] not in snapshot", watch.request().getTypeUrl(), group, - snapshot.version(watch.request().getTypeUrl(), watch.request().getResourceNamesList()), + snapshot.version(watch.request().getResourceType(), watch.request().getResourceNamesList()), String.join(", ", watch.request().getResourceNamesList()), String.join(", ", missingNames)); @@ -317,7 +335,8 @@ private boolean respond(Watch watch, Snapshot snapshot, T group) { } } - String version = snapshot.version(watch.request().getTypeUrl(), watch.request().getResourceNamesList()); + String version = snapshot.version(watch.request().getResourceType(), + watch.request().getResourceNamesList()); LOGGER.debug("responding for {} from node {} at version {} with version {}", watch.request().getTypeUrl(), diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/Snapshot.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/Snapshot.java index 350396333..41ec42b7b 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/Snapshot.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/Snapshot.java @@ -1,240 +1,16 @@ package io.envoyproxy.controlplane.cache; -import static io.envoyproxy.controlplane.cache.Resources.CLUSTER_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.ENDPOINT_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.LISTENER_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.ROUTE_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.SECRET_TYPE_URL; - -import com.google.auto.value.AutoValue; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; import com.google.protobuf.Message; -import io.envoyproxy.envoy.api.v2.Cluster; -import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; -import io.envoyproxy.envoy.api.v2.Listener; -import io.envoyproxy.envoy.api.v2.RouteConfiguration; -import io.envoyproxy.envoy.api.v2.auth.Secret; -import java.util.Collections; +import io.envoyproxy.controlplane.cache.Resources.ResourceType; import java.util.List; import java.util.Map; import java.util.Set; -/** - * {@code Snapshot} is a data class that contains an internally consistent snapshot of xDS resources. Snapshots should - * have distinct versions per node group. - */ -@AutoValue public abstract class Snapshot { - /** - * Returns a new {@link Snapshot} instance that is versioned uniformly across all resources. - * - * @param clusters the cluster resources in this snapshot - * @param endpoints the endpoint resources in this snapshot - * @param listeners the listener resources in this snapshot - * @param routes the route resources in this snapshot - * @param version the version associated with all resources in this snapshot - */ - public static Snapshot create( - Iterable clusters, - Iterable endpoints, - Iterable listeners, - Iterable routes, - Iterable secrets, - String version) { - - return new AutoValue_Snapshot( - SnapshotResources.create(clusters, version), - SnapshotResources.create(endpoints, version), - SnapshotResources.create(listeners, version), - SnapshotResources.create(routes, version), - SnapshotResources.create(secrets, version)); - } - - /** - * Returns a new {@link Snapshot} instance that has separate versions for each resource type. - * - * @param clusters the cluster resources in this snapshot - * @param clustersVersion the version of the cluster resources - * @param endpoints the endpoint resources in this snapshot - * @param endpointsVersion the version of the endpoint resources - * @param listeners the listener resources in this snapshot - * @param listenersVersion the version of the listener resources - * @param routes the route resources in this snapshot - * @param routesVersion the version of the route resources - */ - public static Snapshot create( - Iterable clusters, - String clustersVersion, - Iterable endpoints, - String endpointsVersion, - Iterable listeners, - String listenersVersion, - Iterable routes, - String routesVersion, - Iterable secrets, - String secretsVersion) { - - // TODO(snowp): add a builder alternative - return new AutoValue_Snapshot( - SnapshotResources.create(clusters, clustersVersion), - SnapshotResources.create(endpoints, endpointsVersion), - SnapshotResources.create(listeners, listenersVersion), - SnapshotResources.create(routes, routesVersion), - SnapshotResources.create(secrets, secretsVersion)); - } - - /** - * Returns a new {@link Snapshot} instance that has separate versions for each resource type. - * - * @param clusters the cluster resources in this snapshot - * @param clusterVersionResolver version resolver of the clusters in this snapshot - * @param endpoints the endpoint resources in this snapshot - * @param endpointVersionResolver version resolver of the endpoints in this snapshot - * @param listeners the listener resources in this snapshot - * @param listenerVersionResolver version resolver of listeners in this snapshot - * @param routes the route resources in this snapshot - * @param routeVersionResolver version resolver of the routes in this snapshot - * @param secrets the secret resources in this snapshot - * @param secretVersionResolver version resolver of the secrets in this snapshot - */ - public static Snapshot create( - Iterable clusters, - ResourceVersionResolver clusterVersionResolver, - Iterable endpoints, - ResourceVersionResolver endpointVersionResolver, - Iterable listeners, - ResourceVersionResolver listenerVersionResolver, - Iterable routes, - ResourceVersionResolver routeVersionResolver, - Iterable secrets, - ResourceVersionResolver secretVersionResolver) { - - return new AutoValue_Snapshot( - SnapshotResources.create(clusters, clusterVersionResolver), - SnapshotResources.create(endpoints, endpointVersionResolver), - SnapshotResources.create(listeners, listenerVersionResolver), - SnapshotResources.create(routes, routeVersionResolver), - SnapshotResources.create(secrets, secretVersionResolver)); - } - - /** - * Creates an empty snapshot with the given version. - * - * @param version the version of the snapshot resources - */ - public static Snapshot createEmpty(String version) { - return create(Collections.emptySet(), Collections.emptySet(), - Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), version); - } - - /** - * Returns all cluster items in the CDS payload. - */ - public abstract SnapshotResources clusters(); + public abstract String version(ResourceType resourceType, List resourceNames); - /** - * Returns all endpoint items in the EDS payload. - */ - public abstract SnapshotResources endpoints(); - - /** - * Returns all listener items in the LDS payload. - */ - public abstract SnapshotResources listeners(); - - /** - * Returns all route items in the RDS payload. - */ - public abstract SnapshotResources routes(); - - /** - * Returns all secret items in the SDS payload. - */ - public abstract SnapshotResources secrets(); - - /** - * Asserts that all dependent resources are included in the snapshot. All EDS resources are listed by name in CDS - * resources, and all RDS resources are listed by name in LDS resources. - * - *

Note that clusters and listeners are requested without name references, so Envoy will accept the snapshot list - * of clusters as-is, even if it does not match all references found in xDS. - * - * @throws SnapshotConsistencyException if the snapshot is not consistent - */ - public void ensureConsistent() throws SnapshotConsistencyException { - Set clusterEndpointRefs = Resources.getResourceReferences(clusters().resources().values()); - - ensureAllResourceNamesExist(CLUSTER_TYPE_URL, ENDPOINT_TYPE_URL, clusterEndpointRefs, endpoints().resources()); - - Set listenerRouteRefs = Resources.getResourceReferences(listeners().resources().values()); - - ensureAllResourceNamesExist(LISTENER_TYPE_URL, ROUTE_TYPE_URL, listenerRouteRefs, routes().resources()); - } - - /** - * Returns the resources with the given type. - * - * @param typeUrl the URL for the requested resource type - */ - public Map resources(String typeUrl) { - if (Strings.isNullOrEmpty(typeUrl)) { - return ImmutableMap.of(); - } - - switch (typeUrl) { - case CLUSTER_TYPE_URL: - return clusters().resources(); - case ENDPOINT_TYPE_URL: - return endpoints().resources(); - case LISTENER_TYPE_URL: - return listeners().resources(); - case ROUTE_TYPE_URL: - return routes().resources(); - case SECRET_TYPE_URL: - return secrets().resources(); - default: - return ImmutableMap.of(); - } - } - - /** - * Returns the version in this snapshot for the given resource type. - * - * @param typeUrl the URL for the requested resource type - */ - public String version(String typeUrl) { - return version(typeUrl, Collections.emptyList()); - } - - /** - * Returns the version in this snapshot for the given resource type. - * - * @param typeUrl the URL for the requested resource type - * @param resourceNames list of requested resource names, - * used to calculate a version for the given resources - */ - public String version(String typeUrl, List resourceNames) { - if (Strings.isNullOrEmpty(typeUrl)) { - return ""; - } - - switch (typeUrl) { - case CLUSTER_TYPE_URL: - return clusters().version(resourceNames); - case ENDPOINT_TYPE_URL: - return endpoints().version(resourceNames); - case LISTENER_TYPE_URL: - return listeners().version(resourceNames); - case ROUTE_TYPE_URL: - return routes().version(resourceNames); - case SECRET_TYPE_URL: - return secrets().version(resourceNames); - default: - return ""; - } - } + public abstract Map resources(ResourceType resourceType); /** * Asserts that all of the given resource names have corresponding values in the given resources collection. @@ -245,7 +21,7 @@ public String version(String typeUrl, List resourceNames) { * @param resources the collection of resources whose names are being checked * @throws SnapshotConsistencyException if a name is given that does not exist in the resources collection */ - private static void ensureAllResourceNamesExist( + protected static void ensureAllResourceNamesExist( String parentTypeUrl, String dependencyTypeUrl, Set resourceNames, diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/SnapshotCache.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/SnapshotCache.java index 83e8b9b82..c182c6192 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/SnapshotCache.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/SnapshotCache.java @@ -1,6 +1,6 @@ package io.envoyproxy.controlplane.cache; -public interface SnapshotCache extends Cache { +public interface SnapshotCache extends Cache { /** * Clears the most recently set {@link Snapshot} and associated metadata for the given node group. @@ -22,7 +22,7 @@ public interface SnapshotCache extends Cache { * @param group group identifier * @return latest snapshot */ - Snapshot getSnapshot(T group); + U getSnapshot(T group); /** * Set the {@link Snapshot} for the given node group. Snapshots should have distinct versions and be internally @@ -31,5 +31,5 @@ public interface SnapshotCache extends Cache { * @param group group identifier * @param snapshot a versioned collection of node config data */ - void setSnapshot(T group, Snapshot snapshot); + void setSnapshot(T group, U snapshot); } diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/StatusInfo.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/StatusInfo.java index eee2a8724..0651288a3 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/StatusInfo.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/StatusInfo.java @@ -1,6 +1,6 @@ package io.envoyproxy.controlplane.cache; -import io.envoyproxy.envoy.api.v2.core.Node; +import io.envoyproxy.envoy.config.core.v3.Node; /** * {@code StatusInfo} tracks the state for remote envoy nodes. @@ -13,7 +13,8 @@ public interface StatusInfo { long lastWatchRequestTime(); /** - * Returns the node grouping represented by this status, generated via {@link NodeGroup#hash(Node)}. + * Returns the node grouping represented by this status, generated via + * {@link NodeGroup#hash(Node)} or {@link NodeGroup#hash(io.envoyproxy.envoy.api.v2.core.Node)}. */ T nodeGroup(); diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/TestResources.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/TestResources.java index 1f8deb0d8..883aeb1c7 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/TestResources.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/TestResources.java @@ -1,6 +1,7 @@ package io.envoyproxy.controlplane.cache; import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.Any; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.MessageOrBuilder; import com.google.protobuf.Struct; @@ -33,6 +34,7 @@ import io.envoyproxy.envoy.api.v2.route.RouteAction; import io.envoyproxy.envoy.api.v2.route.RouteMatch; import io.envoyproxy.envoy.api.v2.route.VirtualHost; +import io.envoyproxy.envoy.config.core.v3.ApiVersion; import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager; import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager.CodecType; import io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2.HttpFilter; @@ -81,11 +83,65 @@ public static Cluster createCluster(String clusterName, String address, int port .setName(clusterName) .setConnectTimeout(Durations.fromSeconds(5)) .setType(DiscoveryType.STRICT_DNS) - .addHosts(Address.newBuilder() - .setSocketAddress(SocketAddress.newBuilder() - .setAddress(address) - .setPortValue(port) - .setProtocolValue(Protocol.TCP_VALUE))) + .setLoadAssignment(ClusterLoadAssignment.newBuilder() + .setClusterName(clusterName) + .addEndpoints(LocalityLbEndpoints.newBuilder() + .addLbEndpoints(LbEndpoint.newBuilder() + .setEndpoint(Endpoint.newBuilder() + .setAddress(Address.newBuilder() + .setSocketAddress(SocketAddress.newBuilder() + .setAddress(address) + .setPortValue(port) + .setProtocolValue(Protocol.TCP_VALUE))))))) + .build(); + } + + /** + * Returns a new test v3 cluster using EDS. + * + * @param clusterName name of the new cluster + */ + public static io.envoyproxy.envoy.config.cluster.v3.Cluster createClusterV3(String clusterName) { + io.envoyproxy.envoy.config.core.v3.ConfigSource edsSource = + io.envoyproxy.envoy.config.core.v3.ConfigSource.newBuilder() + .setAds(io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource.getDefaultInstance()) + .build(); + + return io.envoyproxy.envoy.config.cluster.v3.Cluster.newBuilder() + .setName(clusterName) + .setConnectTimeout(Durations.fromSeconds(5)) + .setEdsClusterConfig(io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig.newBuilder() + .setEdsConfig(edsSource) + .setServiceName(clusterName)) + .setType(io.envoyproxy.envoy.config.cluster.v3.Cluster.DiscoveryType.EDS) + .build(); + } + + /** + * Returns a new test v3 cluster not using EDS. + * + * @param clusterName name of the new cluster + * @param address address to use for the cluster endpoint + * @param port port to use for the cluster endpoint + */ + public static io.envoyproxy.envoy.config.cluster.v3.Cluster createClusterV3( + String clusterName, String address, int port) { + return io.envoyproxy.envoy.config.cluster.v3.Cluster.newBuilder() + .setName(clusterName) + .setConnectTimeout(Durations.fromSeconds(5)) + .setType(io.envoyproxy.envoy.config.cluster.v3.Cluster.DiscoveryType.STRICT_DNS) + .setLoadAssignment(io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.newBuilder() + .setClusterName(clusterName) + .addEndpoints(io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints.newBuilder() + .addLbEndpoints(io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint.newBuilder() + .setEndpoint(io.envoyproxy.envoy.config.endpoint.v3.Endpoint.newBuilder() + .setAddress(io.envoyproxy.envoy.config.core.v3.Address.newBuilder() + .setSocketAddress(io.envoyproxy.envoy.config.core.v3.SocketAddress.newBuilder() + .setAddress(address) + .setPortValue(port) + .setProtocolValue(Protocol.TCP_VALUE))))) + ) + ) .build(); } @@ -121,21 +177,62 @@ public static ClusterLoadAssignment createEndpoint(String clusterName, String ad } /** - * Returns a new test listener. + * Returns a new test v3 endpoint for the given cluster. * - * @param ads should RDS for the listener be configured to use XDS? + * @param clusterName name of the test cluster that is associated with this endpoint + * @param port port to use for the endpoint + */ + public static io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment createEndpointV3( + String clusterName, int port) { + return createEndpointV3(clusterName, LOCALHOST, port); + } + + /** + * Returns a new test v3 endpoint for the given cluster. + * + * @param clusterName name of the test cluster that is associated with this endpoint + * @param address ip address to use for the endpoint + * @param port port to use for the endpoint + */ + public static io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment createEndpointV3( + String clusterName, String address, int port) { + return io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.newBuilder() + .setClusterName(clusterName) + .addEndpoints(io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints.newBuilder() + .addLbEndpoints(io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint.newBuilder() + .setEndpoint(io.envoyproxy.envoy.config.endpoint.v3.Endpoint.newBuilder() + .setAddress(io.envoyproxy.envoy.config.core.v3.Address.newBuilder() + .setSocketAddress(io.envoyproxy.envoy.config.core.v3.SocketAddress.newBuilder() + .setAddress(address) + .setPortValue(port) + .setProtocol(io.envoyproxy.envoy.config.core.v3.SocketAddress.Protocol.TCP)))))) + .build(); + } + + /** + * Returns a new test listener. + * @param ads should RDS for the listener be configured to use XDS? + * @param rdsTransportVersion the transport_api_version that should be set for RDS + * @param rdsResourceVersion the resource_api_version that should be set for RDS * @param listenerName name of the new listener * @param port port to use for the listener * @param routeName name of the test route that is associated with this listener */ - public static Listener createListener(boolean ads, String listenerName, int port, String routeName) { + public static Listener createListener(boolean ads, + io.envoyproxy.envoy.api.v2.core.ApiVersion rdsTransportVersion, + io.envoyproxy.envoy.api.v2.core.ApiVersion rdsResourceVersion, String listenerName, + int port, String routeName) { + ConfigSource.Builder configSourceBuilder = ConfigSource.newBuilder() + .setResourceApiVersion(rdsResourceVersion); + ConfigSource rdsSource = ads - ? ConfigSource.newBuilder() - .setAds(AggregatedConfigSource.getDefaultInstance()) - .build() - : ConfigSource.newBuilder() + ? configSourceBuilder + .setAds(AggregatedConfigSource.getDefaultInstance()) + .build() + : configSourceBuilder .setApiConfigSource(ApiConfigSource.newBuilder() .setApiType(ApiType.GRPC) + .setTransportApiVersion(rdsTransportVersion) .addGrpcServices(GrpcService.newBuilder() .setEnvoyGrpc(EnvoyGrpc.newBuilder() .setClusterName(XDS_CLUSTER)))) @@ -161,7 +258,67 @@ public static Listener createListener(boolean ads, String listenerName, int port .addFilterChains(FilterChain.newBuilder() .addFilters(Filter.newBuilder() .setName(Resources.FILTER_HTTP_CONNECTION_MANAGER) - .setConfig(messageAsStruct(manager)))) + .setTypedConfig(Any.pack(manager)))) + .build(); + } + + /** + * Returns a new test v3 listener. + * @param ads should RDS for the listener be configured to use XDS? + * @param rdsTransportVersion the transport_api_version that should be set for RDS config on the + * listener + * @param rdsResourceVersion the resource_api_version that should be set for RDS config on the + * listener + * @param listenerName name of the new listener + * @param port port to use for the listener + * @param routeName name of the test route that is associated with this listener + */ + public static io.envoyproxy.envoy.config.listener.v3.Listener createListenerV3(boolean ads, + ApiVersion rdsTransportVersion, + ApiVersion rdsResourceVersion, String listenerName, + int port, String routeName) { + io.envoyproxy.envoy.config.core.v3.ConfigSource.Builder configSourceBuilder = + io.envoyproxy.envoy.config.core.v3.ConfigSource.newBuilder() + .setResourceApiVersion(rdsResourceVersion); + io.envoyproxy.envoy.config.core.v3.ConfigSource rdsSource = ads + ? configSourceBuilder + .setAds(io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource.getDefaultInstance()) + .build() + : configSourceBuilder + .setApiConfigSource(io.envoyproxy.envoy.config.core.v3.ApiConfigSource.newBuilder() + .setTransportApiVersion(rdsTransportVersion) + .setApiType(io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.GRPC) + .addGrpcServices(io.envoyproxy.envoy.config.core.v3.GrpcService.newBuilder() + .setEnvoyGrpc(io.envoyproxy.envoy.config.core.v3.GrpcService.EnvoyGrpc.newBuilder() + .setClusterName(XDS_CLUSTER)))) + .build(); + + io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + manager = io.envoyproxy.envoy.extensions.filters.network + .http_connection_manager.v3.HttpConnectionManager.newBuilder() + .setCodecType( + io.envoyproxy.envoy.extensions.filters.network + .http_connection_manager.v3.HttpConnectionManager.CodecType.AUTO) + .setStatPrefix("http") + .setRds(io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds.newBuilder() + .setConfigSource(rdsSource) + .setRouteConfigName(routeName)) + .addHttpFilters( + io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter.newBuilder() + .setName(Resources.FILTER_ENVOY_ROUTER)) + .build(); + + return io.envoyproxy.envoy.config.listener.v3.Listener.newBuilder() + .setName(listenerName) + .setAddress(io.envoyproxy.envoy.config.core.v3.Address.newBuilder() + .setSocketAddress(io.envoyproxy.envoy.config.core.v3.SocketAddress.newBuilder() + .setAddress(ANY_ADDRESS) + .setPortValue(port) + .setProtocol(io.envoyproxy.envoy.config.core.v3.SocketAddress.Protocol.TCP))) + .addFilterChains(io.envoyproxy.envoy.config.listener.v3.FilterChain.newBuilder() + .addFilters(io.envoyproxy.envoy.config.listener.v3.Filter.newBuilder() + .setName(Resources.FILTER_HTTP_CONNECTION_MANAGER) + .setTypedConfig(Any.pack(manager)))) .build(); } @@ -185,6 +342,27 @@ public static RouteConfiguration createRoute(String routeName, String clusterNam .build(); } + /** + * Returns a new test v3 route. + * + * @param routeName name of the new route + * @param clusterName name of the test cluster that is associated with this route + */ + public static io.envoyproxy.envoy.config.route.v3.RouteConfiguration createRouteV3( + String routeName, String clusterName) { + return io.envoyproxy.envoy.config.route.v3.RouteConfiguration.newBuilder() + .setName(routeName) + .addVirtualHosts(io.envoyproxy.envoy.config.route.v3.VirtualHost.newBuilder() + .setName("all") + .addDomains("*") + .addRoutes(io.envoyproxy.envoy.config.route.v3.Route.newBuilder() + .setMatch(io.envoyproxy.envoy.config.route.v3.RouteMatch.newBuilder() + .setPrefix("/")) + .setRoute(io.envoyproxy.envoy.config.route.v3.RouteAction.newBuilder() + .setCluster(clusterName)))) + .build(); + } + /** * Returns a new test secret. * @@ -199,6 +377,20 @@ public static Secret createSecret(String secretName) { .build(); } + /** + * Returns a new test v3 secret. + * + * @param secretName name of the new secret + */ + public static io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret createSecretV3(String secretName) { + return io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret.newBuilder() + .setName(secretName) + .setTlsCertificate(io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.TlsCertificate.newBuilder() + .setPrivateKey(io.envoyproxy.envoy.config.core.v3.DataSource.newBuilder() + .setInlineString("secret!"))) + .build(); + } + private static Struct messageAsStruct(MessageOrBuilder message) { try { String json = JsonFormat.printer() diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/Watch.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/Watch.java index f3970938f..76605fac3 100644 --- a/cache/src/main/java/io/envoyproxy/controlplane/cache/Watch.java +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/Watch.java @@ -1,6 +1,5 @@ package io.envoyproxy.controlplane.cache; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.function.Consumer; @@ -12,7 +11,7 @@ public class Watch { private static final AtomicIntegerFieldUpdater isCancelledUpdater = AtomicIntegerFieldUpdater.newUpdater(Watch.class, "isCancelled"); private final boolean ads; - private final DiscoveryRequest request; + private final XdsRequest request; private final Consumer responseConsumer; private volatile int isCancelled = 0; private Runnable stop; @@ -24,7 +23,7 @@ public class Watch { * @param request the original request for the watch * @param responseConsumer handler for outgoing response messages */ - public Watch(boolean ads, DiscoveryRequest request, Consumer responseConsumer) { + public Watch(boolean ads, XdsRequest request, Consumer responseConsumer) { this.ads = ads; this.request = request; this.responseConsumer = responseConsumer; @@ -59,7 +58,7 @@ public boolean isCancelled() { /** * Returns the original request for the watch. */ - public DiscoveryRequest request() { + public XdsRequest request() { return request; } diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/XdsRequest.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/XdsRequest.java new file mode 100644 index 000000000..dc7f7daa9 --- /dev/null +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/XdsRequest.java @@ -0,0 +1,102 @@ +package io.envoyproxy.controlplane.cache; + +import static io.envoyproxy.controlplane.cache.Resources.TYPE_URLS_TO_RESOURCE_TYPE; + +import com.google.auto.value.AutoValue; +import com.google.protobuf.ProtocolStringList; +import io.envoyproxy.controlplane.cache.Resources.ResourceType; +import io.envoyproxy.envoy.api.v2.DiscoveryRequest; +import javax.annotation.Nullable; + +/** + * XdsRequest wraps a v2 or v3 DiscoveryRequest of and provides common methods as a + * workaround to the proto messages not implementing a common interface that can be used to + * abstract away xDS version. XdsRequest is passed around the codebase through common code, + * however the callers that need the raw request from it have knowledge of whether it is a v2 or + * a v3 request. + */ +@AutoValue +public abstract class XdsRequest { + public static XdsRequest create(DiscoveryRequest discoveryRequest) { + return new AutoValue_XdsRequest(discoveryRequest, null); + } + + public static XdsRequest create(io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest discoveryRequest) { + return new AutoValue_XdsRequest(null, discoveryRequest); + } + + /** + * Returns the underlying v2 request, or null if this was a v3 request. Callers should have + * knowledge of whether the request was v2 or not. + * @return v2 DiscoveryRequest or null + */ + @Nullable public abstract DiscoveryRequest v2Request(); + + /** + * Returns he underlying v3 request, or null if this was a v2 request. Callers should have + * knowledge of whether the request was v3 or not. + * @return v3 DiscoveryRequest or null + */ + @Nullable public abstract io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest v3Request(); + + /** + * Returns the type URL of the v2 or v3 request. + */ + public String getTypeUrl() { + if (v2Request() != null) { + return v2Request().getTypeUrl(); + } + return v3Request().getTypeUrl(); + } + + /** + * Returns the ResourceType of the underlying request. This is useful for accepting requests + * for both v2 and v3 resource types and having a key to normalize on the logical resource. + */ + public ResourceType getResourceType() { + if (v2Request() != null) { + return TYPE_URLS_TO_RESOURCE_TYPE.get(v2Request().getTypeUrl()); + } + return TYPE_URLS_TO_RESOURCE_TYPE.get(v3Request().getTypeUrl()); + } + + /** + * Returns the resource names from the underlying DiscoveryRequest. + */ + public ProtocolStringList getResourceNamesList() { + if (v2Request() != null) { + return v2Request().getResourceNamesList(); + } + return v3Request().getResourceNamesList(); + } + + /** + * Returns the version_info from the underlying DiscoveryRequest. + */ + public String getVersionInfo() { + if (v2Request() != null) { + return v2Request().getVersionInfo(); + } + return v3Request().getVersionInfo(); + } + + /** + * Returns the response nonse from the underlying DiscoveryRequest. + */ + public String getResponseNonce() { + if (v2Request() != null) { + return v2Request().getResponseNonce(); + } + return v3Request().getResponseNonce(); + } + + /** + * Returns the error_detail from the underlying v2 or v3 request. + */ + public boolean hasErrorDetail() { + if (v2Request() != null) { + return v2Request().hasErrorDetail(); + } + return v3Request().hasErrorDetail(); + } +} diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/v2/SimpleCache.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/v2/SimpleCache.java new file mode 100644 index 000000000..7df3e0f76 --- /dev/null +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/v2/SimpleCache.java @@ -0,0 +1,9 @@ +package io.envoyproxy.controlplane.cache.v2; + +import io.envoyproxy.controlplane.cache.NodeGroup; + +public class SimpleCache extends io.envoyproxy.controlplane.cache.SimpleCache { + public SimpleCache(NodeGroup nodeGroup) { + super(nodeGroup); + } +} diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/v2/Snapshot.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/v2/Snapshot.java new file mode 100644 index 000000000..38e88e7e2 --- /dev/null +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/v2/Snapshot.java @@ -0,0 +1,279 @@ +package io.envoyproxy.controlplane.cache.v2; + +import static io.envoyproxy.controlplane.cache.Resources.TYPE_URLS_TO_RESOURCE_TYPE; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.ResourceVersionResolver; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Resources.ResourceType; +import io.envoyproxy.controlplane.cache.SnapshotConsistencyException; +import io.envoyproxy.controlplane.cache.SnapshotResources; +import io.envoyproxy.envoy.api.v2.Cluster; +import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; +import io.envoyproxy.envoy.api.v2.Listener; +import io.envoyproxy.envoy.api.v2.RouteConfiguration; +import io.envoyproxy.envoy.api.v2.auth.Secret; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * {@code Snapshot} is a data class that contains an internally consistent snapshot of v2 xDS + * resources. Snapshots should have distinct versions per node group. + */ +@AutoValue +public abstract class Snapshot extends io.envoyproxy.controlplane.cache.Snapshot { + + /** + * Returns a new {@link Snapshot} instance that is versioned uniformly across all resources. + * + * @param clusters the cluster resources in this snapshot + * @param endpoints the endpoint resources in this snapshot + * @param listeners the listener resources in this snapshot + * @param routes the route resources in this snapshot + * @param version the version associated with all resources in this snapshot + */ + public static Snapshot create( + Iterable clusters, + Iterable endpoints, + Iterable listeners, + Iterable routes, + Iterable secrets, + String version) { + + return new AutoValue_Snapshot( + SnapshotResources.create(clusters, version), + SnapshotResources.create(endpoints, version), + SnapshotResources.create(listeners, version), + SnapshotResources.create(routes, version), + SnapshotResources.create(secrets, version)); + } + + /** + * Returns a new {@link Snapshot} instance that has separate versions for each resource type. + * + * @param clusters the cluster resources in this snapshot + * @param clustersVersion the version of the cluster resources + * @param endpoints the endpoint resources in this snapshot + * @param endpointsVersion the version of the endpoint resources + * @param listeners the listener resources in this snapshot + * @param listenersVersion the version of the listener resources + * @param routes the route resources in this snapshot + * @param routesVersion the version of the route resources + */ + public static Snapshot create( + Iterable clusters, + String clustersVersion, + Iterable endpoints, + String endpointsVersion, + Iterable listeners, + String listenersVersion, + Iterable routes, + String routesVersion, + Iterable secrets, + String secretsVersion) { + + // TODO(snowp): add a builder alternative + return new AutoValue_Snapshot( + SnapshotResources.create(clusters, clustersVersion), + SnapshotResources.create(endpoints, endpointsVersion), + SnapshotResources.create(listeners, listenersVersion), + SnapshotResources.create(routes, routesVersion), + SnapshotResources.create(secrets, secretsVersion)); + } + + /** + * Returns a new {@link Snapshot} instance that has separate versions for each resource type. + * + * @param clusters the cluster resources in this snapshot + * @param clusterVersionResolver version resolver of the clusters in this snapshot + * @param endpoints the endpoint resources in this snapshot + * @param endpointVersionResolver version resolver of the endpoints in this snapshot + * @param listeners the listener resources in this snapshot + * @param listenerVersionResolver version resolver of listeners in this snapshot + * @param routes the route resources in this snapshot + * @param routeVersionResolver version resolver of the routes in this snapshot + * @param secrets the secret resources in this snapshot + * @param secretVersionResolver version resolver of the secrets in this snapshot + */ + public static Snapshot create( + Iterable clusters, + ResourceVersionResolver clusterVersionResolver, + Iterable endpoints, + ResourceVersionResolver endpointVersionResolver, + Iterable listeners, + ResourceVersionResolver listenerVersionResolver, + Iterable routes, + ResourceVersionResolver routeVersionResolver, + Iterable secrets, + ResourceVersionResolver secretVersionResolver) { + + return new AutoValue_Snapshot( + SnapshotResources.create(clusters, clusterVersionResolver), + SnapshotResources.create(endpoints, endpointVersionResolver), + SnapshotResources.create(listeners, listenerVersionResolver), + SnapshotResources.create(routes, routeVersionResolver), + SnapshotResources.create(secrets, secretVersionResolver)); + } + + /** + * Creates an empty snapshot with the given version. + * + * @param version the version of the snapshot resources + */ + public static Snapshot createEmpty(String version) { + return create(Collections.emptySet(), Collections.emptySet(), + Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), version); + } + + /** + * Returns all cluster items in the CDS payload. + */ + public abstract SnapshotResources clusters(); + + + /** + * Returns all endpoint items in the EDS payload. + */ + public abstract SnapshotResources endpoints(); + + /** + * Returns all listener items in the LDS payload. + */ + public abstract SnapshotResources listeners(); + + /** + * Returns all route items in the RDS payload. + */ + public abstract SnapshotResources routes(); + + /** + * Returns all secret items in the SDS payload. + */ + public abstract SnapshotResources secrets(); + + /** + * Asserts that all dependent resources are included in the snapshot. All EDS resources are listed by name in CDS + * resources, and all RDS resources are listed by name in LDS resources. + * + *

Note that clusters and listeners are requested without name references, so Envoy will accept the snapshot list + * of clusters as-is, even if it does not match all references found in xDS. + * + * @throws SnapshotConsistencyException if the snapshot is not consistent + */ + public void ensureConsistent() throws SnapshotConsistencyException { + Set clusterEndpointRefs = Resources.getResourceReferences(clusters().resources().values()); + + ensureAllResourceNamesExist(Resources.V2.CLUSTER_TYPE_URL, Resources.V2.ENDPOINT_TYPE_URL, + clusterEndpointRefs, endpoints().resources()); + + Set listenerRouteRefs = Resources.getResourceReferences(listeners().resources().values()); + + ensureAllResourceNamesExist(Resources.V2.LISTENER_TYPE_URL, Resources.V2.ROUTE_TYPE_URL, + listenerRouteRefs, routes().resources()); + } + + /** + * Returns the resources with the given type. + * + * @param typeUrl the type URL of the requested resource type + */ + public Map resources(String typeUrl) { + if (Strings.isNullOrEmpty(typeUrl)) { + return ImmutableMap.of(); + } + + ResourceType resourceType = TYPE_URLS_TO_RESOURCE_TYPE.get(typeUrl); + if (resourceType == null) { + return ImmutableMap.of(); + } + + return resources(resourceType); + } + + /** + * Returns the resources with the given type. + * + * @param resourceType the requested resource type + */ + public Map resources(ResourceType resourceType) { + switch (resourceType) { + case CLUSTER: + return clusters().resources(); + case ENDPOINT: + return endpoints().resources(); + case LISTENER: + return listeners().resources(); + case ROUTE: + return routes().resources(); + case SECRET: + return secrets().resources(); + default: + return ImmutableMap.of(); + } + } + + /** + * Returns the version in this snapshot for the given resource type. + * + * @param typeUrl the type URL of the requested resource type + */ + public String version(String typeUrl) { + return version(typeUrl, Collections.emptyList()); + } + + /** + * Returns the version in this snapshot for the given resource type. + * + * @param typeUrl the type URL of the requested resource type + * @param resourceNames list of requested resource names, + * used to calculate a version for the given resources + */ + public String version(String typeUrl, List resourceNames) { + if (Strings.isNullOrEmpty(typeUrl)) { + return ""; + } + + ResourceType resourceType = TYPE_URLS_TO_RESOURCE_TYPE.get(typeUrl); + if (resourceType == null) { + return ""; + } + + return version(resourceType, resourceNames); + } + + public String version(ResourceType resourceType) { + return version(resourceType, Collections.emptyList()); + } + + /** + * Returns the version in this snapshot for the given resource type. + * + * @param resourceType the the requested resource type + * @param resourceNames list of requested resource names, + * used to calculate a version for the given resources + */ + @Override + public String version(ResourceType resourceType, List resourceNames) { + switch (resourceType) { + case CLUSTER: + return clusters().version(resourceNames); + case ENDPOINT: + return endpoints().version(resourceNames); + case LISTENER: + return listeners().version(resourceNames); + case ROUTE: + return routes().version(resourceNames); + case SECRET: + return secrets().version(resourceNames); + default: + return ""; + } + } + + +} diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/v3/SimpleCache.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/v3/SimpleCache.java new file mode 100644 index 000000000..8779d151e --- /dev/null +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/v3/SimpleCache.java @@ -0,0 +1,9 @@ +package io.envoyproxy.controlplane.cache.v3; + +import io.envoyproxy.controlplane.cache.NodeGroup; + +public class SimpleCache extends io.envoyproxy.controlplane.cache.SimpleCache { + public SimpleCache(NodeGroup nodeGroup) { + super(nodeGroup); + } +} diff --git a/cache/src/main/java/io/envoyproxy/controlplane/cache/v3/Snapshot.java b/cache/src/main/java/io/envoyproxy/controlplane/cache/v3/Snapshot.java new file mode 100644 index 000000000..0c495d47b --- /dev/null +++ b/cache/src/main/java/io/envoyproxy/controlplane/cache/v3/Snapshot.java @@ -0,0 +1,233 @@ +package io.envoyproxy.controlplane.cache.v3; + +import static io.envoyproxy.controlplane.cache.Resources.TYPE_URLS_TO_RESOURCE_TYPE; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Resources.ResourceType; +import io.envoyproxy.controlplane.cache.SnapshotConsistencyException; +import io.envoyproxy.controlplane.cache.SnapshotResources; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * {@code Snapshot} is a data class that contains an internally consistent snapshot of v3 xDS + * resources. Snapshots should have distinct versions per node group. + */ +@AutoValue +public abstract class Snapshot extends io.envoyproxy.controlplane.cache.Snapshot { + /** + * Returns a new {@link io.envoyproxy.controlplane.cache.v2.Snapshot} instance that is versioned + * uniformly across all resources. + * + * @param clusters the cluster resources in this snapshot + * @param endpoints the endpoint resources in this snapshot + * @param listeners the listener resources in this snapshot + * @param routes the route resources in this snapshot + * @param version the version associated with all resources in this snapshot + */ + public static Snapshot create( + Iterable clusters, + Iterable endpoints, + Iterable listeners, + Iterable routes, + Iterable secrets, + String version) { + + return new AutoValue_Snapshot( + SnapshotResources.create(clusters, version), + SnapshotResources.create(endpoints, version), + SnapshotResources.create(listeners, version), + SnapshotResources.create(routes, version), + SnapshotResources.create(secrets, version)); + } + + /** + * Returns a new {@link io.envoyproxy.controlplane.cache.v2.Snapshot} instance that has separate + * versions for each resource type. + * + * @param clusters the cluster resources in this snapshot + * @param clustersVersion the version of the cluster resources + * @param endpoints the endpoint resources in this snapshot + * @param endpointsVersion the version of the endpoint resources + * @param listeners the listener resources in this snapshot + * @param listenersVersion the version of the listener resources + * @param routes the route resources in this snapshot + * @param routesVersion the version of the route resources + */ + public static Snapshot create( + Iterable clusters, + String clustersVersion, + Iterable endpoints, + String endpointsVersion, + Iterable listeners, + String listenersVersion, + Iterable routes, + String routesVersion, + Iterable secrets, + String secretsVersion) { + + // TODO(snowp): add a builder alternative + return new AutoValue_Snapshot( + SnapshotResources.create(clusters, clustersVersion), + SnapshotResources.create(endpoints, endpointsVersion), + SnapshotResources.create(listeners, listenersVersion), + SnapshotResources.create(routes, routesVersion), + SnapshotResources.create(secrets, secretsVersion)); + } + + /** + * Returns all v3 cluster items in the CDS payload. + */ + public abstract SnapshotResources clusters(); + + /** + * Returns all v3 endpoint items in the EDS payload. + */ + public abstract SnapshotResources endpoints(); + + /** + * Returns all listener items in the LDS payload. + */ + public abstract SnapshotResources listeners(); + + /** + * Returns all route items in the RDS payload. + */ + public abstract SnapshotResources routes(); + + /** + * Returns all secret items in the SDS payload. + */ + public abstract SnapshotResources secrets(); + + /** + * Asserts that all dependent resources are included in the snapshot. All EDS resources are listed by name in CDS + * resources, and all RDS resources are listed by name in LDS resources. + * + *

Note that clusters and listeners are requested without name references, so Envoy will accept the snapshot list + * of clusters as-is, even if it does not match all references found in xDS. + * + * @throws SnapshotConsistencyException if the snapshot is not consistent + */ + public void ensureConsistent() throws SnapshotConsistencyException { + Set clusterEndpointRefs = + Resources.getResourceReferences(clusters().resources().values()); + + ensureAllResourceNamesExist(Resources.V3.CLUSTER_TYPE_URL, Resources.V3.ENDPOINT_TYPE_URL, + clusterEndpointRefs, endpoints().resources()); + + Set listenerRouteRefs = + Resources.getResourceReferences(listeners().resources().values()); + + ensureAllResourceNamesExist(Resources.V3.LISTENER_TYPE_URL, Resources.V3.ROUTE_TYPE_URL, + listenerRouteRefs, routes().resources()); + } + + /** + * Returns the resources with the given type. + * + * @param typeUrl the type URL of the requested resource type + */ + public Map resources(String typeUrl) { + if (Strings.isNullOrEmpty(typeUrl)) { + return ImmutableMap.of(); + } + + ResourceType resourceType = TYPE_URLS_TO_RESOURCE_TYPE.get(typeUrl); + if (resourceType == null) { + return ImmutableMap.of(); + } + + return resources(resourceType); + } + + /** + * Returns the resources with the given type. + * + * @param resourceType the requested resource type + */ + public Map resources(ResourceType resourceType) { + switch (resourceType) { + case CLUSTER: + return clusters().resources(); + case ENDPOINT: + return endpoints().resources(); + case LISTENER: + return listeners().resources(); + case ROUTE: + return routes().resources(); + case SECRET: + return secrets().resources(); + default: + return ImmutableMap.of(); + } + } + + /** + * Returns the version in this snapshot for the given resource type. + * + * @param typeUrl the type URL of the requested resource type + */ + public String version(String typeUrl) { + return version(typeUrl, Collections.emptyList()); + } + + /** + * Returns the version in this snapshot for the given resource type. + * + * @param typeUrl the type URL of the requested resource type + * @param resourceNames list of requested resource names, + * used to calculate a version for the given resources + */ + public String version(String typeUrl, List resourceNames) { + if (Strings.isNullOrEmpty(typeUrl)) { + return ""; + } + + ResourceType resourceType = TYPE_URLS_TO_RESOURCE_TYPE.get(typeUrl); + if (resourceType == null) { + return ""; + } + return version(resourceType, resourceNames); + } + + public String version(ResourceType resourceType) { + return version(resourceType, Collections.emptyList()); + } + + /** + * Returns the version in this snapshot for the given resource type. + * + * @param resourceType the the requested resource type + * @param resourceNames list of requested resource names, + * used to calculate a version for the given resources + */ + @Override + public String version(ResourceType resourceType, List resourceNames) { + switch (resourceType) { + case CLUSTER: + return clusters().version(resourceNames); + case ENDPOINT: + return endpoints().version(resourceNames); + case LISTENER: + return listeners().version(resourceNames); + case ROUTE: + return routes().version(resourceNames); + case SECRET: + return secrets().version(resourceNames); + default: + return ""; + } + } +} diff --git a/cache/src/test/java/io/envoyproxy/controlplane/cache/CacheStatusInfoTest.java b/cache/src/test/java/io/envoyproxy/controlplane/cache/CacheStatusInfoTest.java index 55b9641a2..b40f53add 100644 --- a/cache/src/test/java/io/envoyproxy/controlplane/cache/CacheStatusInfoTest.java +++ b/cache/src/test/java/io/envoyproxy/controlplane/cache/CacheStatusInfoTest.java @@ -50,12 +50,14 @@ public void numWatchesReturnsExpectedSize() { assertThat(info.numWatches()).isZero(); - info.setWatch(watchId1, new Watch(ads, DiscoveryRequest.getDefaultInstance(), r -> { })); + info.setWatch(watchId1, new Watch(ads, + XdsRequest.create(DiscoveryRequest.getDefaultInstance()), r -> { })); assertThat(info.numWatches()).isEqualTo(1); assertThat(info.watchIds()).containsExactlyInAnyOrder(watchId1); - info.setWatch(watchId2, new Watch(ads, DiscoveryRequest.getDefaultInstance(), r -> { })); + info.setWatch(watchId2, new Watch(ads, + XdsRequest.create(DiscoveryRequest.getDefaultInstance()), r -> { })); assertThat(info.numWatches()).isEqualTo(2); assertThat(info.watchIds()).containsExactlyInAnyOrder(watchId1, watchId2); @@ -74,8 +76,10 @@ public void watchesRemoveIfRemovesExpectedWatches() { CacheStatusInfo info = new CacheStatusInfo<>(Node.getDefaultInstance()); - info.setWatch(watchId1, new Watch(ads, DiscoveryRequest.getDefaultInstance(), r -> { })); - info.setWatch(watchId2, new Watch(ads, DiscoveryRequest.getDefaultInstance(), r -> { })); + info.setWatch(watchId1, new Watch(ads, + XdsRequest.create(DiscoveryRequest.getDefaultInstance()), r -> { })); + info.setWatch(watchId2, new Watch(ads, + XdsRequest.create(DiscoveryRequest.getDefaultInstance()), r -> { })); assertThat(info.numWatches()).isEqualTo(2); assertThat(info.watchIds()).containsExactlyInAnyOrder(watchId1, watchId2); @@ -96,7 +100,8 @@ public void testConcurrentSetWatchAndRemove() { Collection watchIds = LongStream.range(0, watchCount).boxed().collect(Collectors.toList()); watchIds.parallelStream().forEach(watchId -> { - Watch watch = new Watch(ads, DiscoveryRequest.getDefaultInstance(), r -> { }); + Watch watch = new Watch(ads, XdsRequest.create(DiscoveryRequest.getDefaultInstance()), + r -> { }); info.setWatch(watchId, watch); }); diff --git a/cache/src/test/java/io/envoyproxy/controlplane/cache/ResourcesTest.java b/cache/src/test/java/io/envoyproxy/controlplane/cache/ResourcesTest.java index d80d25e7d..730a8f938 100644 --- a/cache/src/test/java/io/envoyproxy/controlplane/cache/ResourcesTest.java +++ b/cache/src/test/java/io/envoyproxy/controlplane/cache/ResourcesTest.java @@ -1,5 +1,6 @@ package io.envoyproxy.controlplane.cache; +import static io.envoyproxy.envoy.config.core.v3.ApiVersion.V3; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -15,6 +16,7 @@ import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.Listener; import io.envoyproxy.envoy.api.v2.RouteConfiguration; +import io.envoyproxy.envoy.api.v2.auth.Secret; import java.util.Collection; import java.util.Map; import java.util.Set; @@ -24,17 +26,38 @@ public class ResourcesTest { private static final boolean ADS = ThreadLocalRandom.current().nextBoolean(); - private static final String CLUSTER_NAME = "cluster0"; - private static final String LISTENER_NAME = "listener0"; - private static final String ROUTE_NAME = "route0"; + private static final String CLUSTER_NAME = "v2cluster"; + private static final String LISTENER_NAME = "v2listener"; + private static final String ROUTE_NAME = "v2route"; + private static final String SECRET_NAME = "v2secret"; + private static final String V3_CLUSTER_NAME = "v3cluster"; + private static final String V3_LISTENER_NAME = "v3listener"; + private static final String V3_ROUTE_NAME = "v3route"; + private static final String V3_SECRET_NAME = "v3secret"; private static final int ENDPOINT_PORT = ThreadLocalRandom.current().nextInt(10000, 20000); private static final int LISTENER_PORT = ThreadLocalRandom.current().nextInt(20000, 30000); private static final Cluster CLUSTER = TestResources.createCluster(CLUSTER_NAME); private static final ClusterLoadAssignment ENDPOINT = TestResources.createEndpoint(CLUSTER_NAME, ENDPOINT_PORT); - private static final Listener LISTENER = TestResources.createListener(ADS, LISTENER_NAME, LISTENER_PORT, ROUTE_NAME); + private static final Listener LISTENER = TestResources.createListener(ADS, + io.envoyproxy.envoy.api.v2.core.ApiVersion.V2, io.envoyproxy.envoy.api.v2.core.ApiVersion.V2, + LISTENER_NAME, LISTENER_PORT, + ROUTE_NAME); private static final RouteConfiguration ROUTE = TestResources.createRoute(ROUTE_NAME, CLUSTER_NAME); + private static final Secret SECRET = TestResources.createSecret(SECRET_NAME); + + private static final io.envoyproxy.envoy.config.cluster.v3.Cluster V3_CLUSTER = + TestResources.createClusterV3(V3_CLUSTER_NAME); + private static final io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment V3_ENDPOINT = + TestResources.createEndpointV3(V3_CLUSTER_NAME, ENDPOINT_PORT); + private static final io.envoyproxy.envoy.config.listener.v3.Listener + V3_LISTENER = TestResources.createListenerV3(ADS, V3, V3, V3_LISTENER_NAME, + LISTENER_PORT, V3_ROUTE_NAME); + private static final io.envoyproxy.envoy.config.route.v3.RouteConfiguration V3_ROUTE = + TestResources.createRouteV3(V3_ROUTE_NAME, V3_CLUSTER_NAME); + private static final io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret V3_SECRET = + TestResources.createSecretV3(V3_SECRET_NAME); @Test public void getResourceNameReturnsExpectedNameForValidResourceMessage() { @@ -42,7 +65,21 @@ public void getResourceNameReturnsExpectedNameForValidResourceMessage() { CLUSTER, CLUSTER_NAME, ENDPOINT, CLUSTER_NAME, LISTENER, LISTENER_NAME, - ROUTE, ROUTE_NAME); + ROUTE, ROUTE_NAME, + SECRET, SECRET_NAME); + + cases.forEach((resource, expectedName) -> + assertThat(Resources.getResourceName(resource)).isEqualTo(expectedName)); + } + + @Test + public void getResourceNameReturnsExpectedNameForValidResourceMessageV3() { + Map cases = ImmutableMap.of( + V3_CLUSTER, V3_CLUSTER_NAME, + V3_ENDPOINT, V3_CLUSTER_NAME, + V3_LISTENER, V3_LISTENER_NAME, + V3_ROUTE, V3_ROUTE_NAME, + V3_SECRET, V3_SECRET_NAME); cases.forEach((resource, expectedName) -> assertThat(Resources.getResourceName(resource)).isEqualTo(expectedName)); @@ -85,4 +122,30 @@ public void getResourceReferencesReturnsExpectedReferencesForValidResourceMessag cases.forEach((resources, refs) -> assertThat(Resources.getResourceReferences(resources)).containsExactlyElementsOf(refs)); } + + @Test + public void getResourceReferencesReturnsExpectedReferencesForValidV3ResourceMessages() { + String clusterServiceName = "clusterWithServiceName0"; + io.envoyproxy.envoy.config.cluster.v3.Cluster clusterWithServiceName = io.envoyproxy.envoy.config.cluster.v3.Cluster + .newBuilder() + .setName(V3_CLUSTER_NAME) + .setEdsClusterConfig( + io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig.newBuilder() + .setServiceName(clusterServiceName)) + .setType(io.envoyproxy.envoy.config.cluster.v3.Cluster.DiscoveryType.EDS) + .build(); + + Map, Set> cases = ImmutableMap., Set>builder() + .put(ImmutableList.of(V3_CLUSTER), ImmutableSet.of(V3_CLUSTER_NAME)) + .put(ImmutableList.of(clusterWithServiceName), ImmutableSet.of(clusterServiceName)) + .put(ImmutableList.of(V3_ENDPOINT), ImmutableSet.of()) + .put(ImmutableList.of(V3_LISTENER), ImmutableSet.of(V3_ROUTE_NAME)) + .put(ImmutableList.of(V3_ROUTE), ImmutableSet.of()) + .put(ImmutableList.of(V3_CLUSTER, V3_ENDPOINT, V3_LISTENER, V3_ROUTE), + ImmutableSet.of(V3_CLUSTER_NAME, V3_ROUTE_NAME)) + .build(); + + cases.forEach((resources, refs) -> + assertThat(Resources.getResourceReferences(resources)).containsExactlyElementsOf(refs)); + } } diff --git a/cache/src/test/java/io/envoyproxy/controlplane/cache/WatchTest.java b/cache/src/test/java/io/envoyproxy/controlplane/cache/WatchTest.java index e1d9a8a4b..408e4b5df 100644 --- a/cache/src/test/java/io/envoyproxy/controlplane/cache/WatchTest.java +++ b/cache/src/test/java/io/envoyproxy/controlplane/cache/WatchTest.java @@ -19,7 +19,8 @@ public class WatchTest { public void adsReturnsGivenValue() { final boolean ads = ThreadLocalRandom.current().nextBoolean(); - Watch watch = new Watch(ads, DiscoveryRequest.getDefaultInstance(), r -> { }); + Watch watch = new Watch(ads, XdsRequest.create(DiscoveryRequest.getDefaultInstance()), + r -> { }); assertThat(watch.ads()).isEqualTo(ads); } @@ -28,7 +29,7 @@ public void adsReturnsGivenValue() { public void isCancelledTrueAfterCancel() { final boolean ads = ThreadLocalRandom.current().nextBoolean(); - Watch watch = new Watch(ads, DiscoveryRequest.getDefaultInstance(), r -> { }); + Watch watch = new Watch(ads, XdsRequest.create(DiscoveryRequest.getDefaultInstance()), r -> { }); assertThat(watch.isCancelled()).isFalse(); @@ -43,7 +44,7 @@ public void cancelWithStopCallsStop() { AtomicInteger stopCount = new AtomicInteger(); - Watch watch = new Watch(ads, DiscoveryRequest.getDefaultInstance(), r -> { }); + Watch watch = new Watch(ads, XdsRequest.create(DiscoveryRequest.getDefaultInstance()), r -> { }); watch.setStop(stopCount::getAndIncrement); @@ -62,23 +63,23 @@ public void responseHandlerExecutedForResponsesUntilCancelled() { final boolean ads = ThreadLocalRandom.current().nextBoolean(); Response response1 = Response.create( - DiscoveryRequest.newBuilder().build(), + XdsRequest.create(DiscoveryRequest.getDefaultInstance()), ImmutableList.of(), UUID.randomUUID().toString()); Response response2 = Response.create( - DiscoveryRequest.newBuilder().build(), + XdsRequest.create(DiscoveryRequest.getDefaultInstance()), ImmutableList.of(), UUID.randomUUID().toString()); Response response3 = Response.create( - DiscoveryRequest.newBuilder().build(), + XdsRequest.create(DiscoveryRequest.getDefaultInstance()), ImmutableList.of(), UUID.randomUUID().toString()); List responses = new LinkedList<>(); - Watch watch = new Watch(ads, DiscoveryRequest.getDefaultInstance(), responses::add); + Watch watch = new Watch(ads, XdsRequest.create(DiscoveryRequest.getDefaultInstance()), responses::add); try { watch.respond(response1); diff --git a/cache/src/test/java/io/envoyproxy/controlplane/cache/SimpleCacheTest.java b/cache/src/test/java/io/envoyproxy/controlplane/cache/v2/SimpleCacheTest.java similarity index 80% rename from cache/src/test/java/io/envoyproxy/controlplane/cache/SimpleCacheTest.java rename to cache/src/test/java/io/envoyproxy/controlplane/cache/v2/SimpleCacheTest.java index 06cdac3ac..b24cdec51 100644 --- a/cache/src/test/java/io/envoyproxy/controlplane/cache/SimpleCacheTest.java +++ b/cache/src/test/java/io/envoyproxy/controlplane/cache/v2/SimpleCacheTest.java @@ -1,11 +1,18 @@ -package io.envoyproxy.controlplane.cache; +package io.envoyproxy.controlplane.cache.v2; -import static io.envoyproxy.controlplane.cache.Resources.ROUTE_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V2.CLUSTER_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V2.ROUTE_TYPE_URL; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Response; +import io.envoyproxy.controlplane.cache.StatusInfo; +import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.envoy.api.v2.Cluster; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; @@ -21,6 +28,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import java.util.stream.Collectors; +import org.assertj.core.api.Assertions; import org.junit.Test; public class SimpleCacheTest { @@ -40,7 +48,7 @@ public class SimpleCacheTest { ImmutableList.of(ClusterLoadAssignment.getDefaultInstance()), ImmutableList.of(Listener.newBuilder().setName(LISTENER_NAME).build()), ImmutableList.of(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build()), - ImmutableList.of(Secret.newBuilder().setName(ROUTE_NAME).build()), + ImmutableList.of(Secret.newBuilder().setName(SECRET_NAME).build()), VERSION1); private static final Snapshot SNAPSHOT2 = Snapshot.create( @@ -48,7 +56,7 @@ public class SimpleCacheTest { ImmutableList.of(ClusterLoadAssignment.getDefaultInstance()), ImmutableList.of(Listener.newBuilder().setName(LISTENER_NAME).build()), ImmutableList.of(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build()), - ImmutableList.of(Secret.newBuilder().setName(ROUTE_NAME).build()), + ImmutableList.of(Secret.newBuilder().setName(SECRET_NAME).build()), VERSION2); private static final Snapshot MULTIPLE_RESOURCES_SNAPSHOT2 = Snapshot.create( @@ -58,7 +66,7 @@ public class SimpleCacheTest { ClusterLoadAssignment.newBuilder().setClusterName(SECONDARY_CLUSTER_NAME).build()), ImmutableList.of(Listener.newBuilder().setName(LISTENER_NAME).build()), ImmutableList.of(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build()), - ImmutableList.of(Secret.newBuilder().setName(ROUTE_NAME).build()), + ImmutableList.of(Secret.newBuilder().setName(SECRET_NAME).build()), VERSION2); @Test @@ -71,11 +79,11 @@ public void invalidNamesListShouldReturnWatcherWithNoResponseInAdsMode() { Watch watch = cache.createWatch( true, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) - .setTypeUrl(Resources.ENDPOINT_TYPE_URL) + .setTypeUrl(Resources.V2.ENDPOINT_TYPE_URL) .addResourceNames("none") - .build(), + .build()), Collections.emptySet(), responseTracker); @@ -92,16 +100,16 @@ public void invalidNamesListShouldReturnWatcherWithResponseInXdsMode() { Watch watch = cache.createWatch( false, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) - .setTypeUrl(Resources.ENDPOINT_TYPE_URL) + .setTypeUrl(Resources.V2.ENDPOINT_TYPE_URL) .addResourceNames("none") - .build(), + .build()), Collections.emptySet(), responseTracker); assertThat(watch.isCancelled()).isFalse(); - assertThat(responseTracker.responses).isNotEmpty(); + Assertions.assertThat(responseTracker.responses).isNotEmpty(); } @Test @@ -110,16 +118,16 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatch() { cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { ResponseTracker responseTracker = new ResponseTracker(); Watch watch = cache.createWatch( ADS, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setTypeUrl(typeUrl) .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) - .build(), + .build()), Collections.emptySet(), responseTracker); @@ -141,19 +149,19 @@ public void shouldSendEdsWhenClusterChangedButEdsVersionDidnt() { Watch watch = cache.createWatch( ADS, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setVersionInfo(VERSION1) - .setTypeUrl(Resources.ENDPOINT_TYPE_URL) - .addAllResourceNames(SNAPSHOT1.resources(Resources.ENDPOINT_TYPE_URL).keySet()) - .build(), + .setTypeUrl(Resources.V2.ENDPOINT_TYPE_URL) + .addAllResourceNames(SNAPSHOT1.resources(Resources.V2.ENDPOINT_TYPE_URL).keySet()) + .build()), Sets.newHashSet(""), responseTracker, true); - assertThat(watch.request().getTypeUrl()).isEqualTo(Resources.ENDPOINT_TYPE_URL); + assertThat(watch.request().getTypeUrl()).isEqualTo(Resources.V2.ENDPOINT_TYPE_URL); assertThat(watch.request().getResourceNamesList()).containsExactlyElementsOf( - SNAPSHOT1.resources(Resources.ENDPOINT_TYPE_URL).keySet()); + SNAPSHOT1.resources(Resources.V2.ENDPOINT_TYPE_URL).keySet()); assertThatWatchReceivesSnapshot(new WatchAndTracker(watch, responseTracker), SNAPSHOT1); } @@ -162,7 +170,7 @@ public void shouldSendEdsWhenClusterChangedButEdsVersionDidnt() { public void successfullyWatchAllResourceTypesWithSetAfterWatch() { SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); - Map watches = Resources.TYPE_URLS.stream() + Map watches = Resources.V2.TYPE_URLS.stream() .collect(Collectors.toMap( typeUrl -> typeUrl, typeUrl -> { @@ -170,11 +178,11 @@ public void successfullyWatchAllResourceTypesWithSetAfterWatch() { Watch watch = cache.createWatch( ADS, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setTypeUrl(typeUrl) .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) - .build(), + .build()), Collections.emptySet(), responseTracker); @@ -183,7 +191,7 @@ public void successfullyWatchAllResourceTypesWithSetAfterWatch() { cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { assertThatWatchReceivesSnapshot(watches.get(typeUrl), SNAPSHOT1); } } @@ -199,7 +207,7 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithRequestVersio HashMap watches = new HashMap<>(); for (int i = 0; i < 2; ++i) { - watches.putAll(Resources.TYPE_URLS.stream() + watches.putAll(Resources.V2.TYPE_URLS.stream() .collect(Collectors.toMap( typeUrl -> typeUrl, typeUrl -> { @@ -207,12 +215,12 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithRequestVersio Watch watch = cache.createWatch( ADS, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setTypeUrl(typeUrl) .setVersionInfo(SNAPSHOT1.version(typeUrl)) .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) - .build(), + .build()), SNAPSHOT2.resources(typeUrl).keySet(), r -> { responseTracker.accept(r); @@ -225,21 +233,21 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithRequestVersio } // The request version matches the current snapshot version, so the watches shouldn't receive any responses. - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { assertThatWatchIsOpenWithNoResponses(watches.get(typeUrl)); } cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT2); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { assertThatWatchReceivesSnapshot(watches.get(typeUrl), SNAPSHOT2); } // Verify that CDS and LDS always get triggered before EDS and RDS respectively. - assertThat(responseOrderTracker.responseTypes).containsExactly(Resources.CLUSTER_TYPE_URL, - Resources.CLUSTER_TYPE_URL, Resources.ENDPOINT_TYPE_URL, Resources.ENDPOINT_TYPE_URL, - Resources.LISTENER_TYPE_URL, Resources.LISTENER_TYPE_URL, Resources.ROUTE_TYPE_URL, - Resources.ROUTE_TYPE_URL, Resources.SECRET_TYPE_URL, Resources.SECRET_TYPE_URL); + assertThat(responseOrderTracker.responseTypes).containsExactly(Resources.V2.CLUSTER_TYPE_URL, + Resources.V2.CLUSTER_TYPE_URL, Resources.V2.ENDPOINT_TYPE_URL, Resources.V2.ENDPOINT_TYPE_URL, + Resources.V2.LISTENER_TYPE_URL, Resources.V2.LISTENER_TYPE_URL, Resources.V2.ROUTE_TYPE_URL, + Resources.V2.ROUTE_TYPE_URL, Resources.V2.SECRET_TYPE_URL, Resources.V2.SECRET_TYPE_URL); } @Test @@ -253,7 +261,7 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithSameRequestVe // // Note how we're requesting the resources from MULTIPLE_RESOURCE_SNAPSHOT2 while claiming we // only know about the ones from SNAPSHOT2 - Map watches = Resources.TYPE_URLS.stream() + Map watches = Resources.V2.TYPE_URLS.stream() .collect(Collectors.toMap( typeUrl -> typeUrl, typeUrl -> { @@ -261,12 +269,12 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithSameRequestVe Watch watch = cache.createWatch( ADS, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setTypeUrl(typeUrl) .setVersionInfo(MULTIPLE_RESOURCES_SNAPSHOT2.version(typeUrl)) .addAllResourceNames(MULTIPLE_RESOURCES_SNAPSHOT2.resources(typeUrl).keySet()) - .build(), + .build()), SNAPSHOT2.resources(typeUrl).keySet(), responseTracker); @@ -275,8 +283,8 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithSameRequestVe // The snapshot version matches for all resources, but for eds and cds there are new resources present // for the same version, so we expect the watches to trigger. - assertThatWatchReceivesSnapshot(watches.remove(Resources.CLUSTER_TYPE_URL), MULTIPLE_RESOURCES_SNAPSHOT2); - assertThatWatchReceivesSnapshot(watches.remove(Resources.ENDPOINT_TYPE_URL), MULTIPLE_RESOURCES_SNAPSHOT2); + assertThatWatchReceivesSnapshot(watches.remove(Resources.V2.CLUSTER_TYPE_URL), MULTIPLE_RESOURCES_SNAPSHOT2); + assertThatWatchReceivesSnapshot(watches.remove(Resources.V2.ENDPOINT_TYPE_URL), MULTIPLE_RESOURCES_SNAPSHOT2); // Remaining watches should not trigger for (WatchAndTracker watchAndTracker : watches.values()) { @@ -297,7 +305,7 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithSameRequestVe // while we only know about the resources found in SNAPSHOT2. Since SNAPSHOT2 is the current // snapshot, we have nothing to respond with for the new resources so we should not trigger // the watch. - Map watches = Resources.TYPE_URLS.stream() + Map watches = Resources.V2.TYPE_URLS.stream() .collect(Collectors.toMap( typeUrl -> typeUrl, typeUrl -> { @@ -305,12 +313,12 @@ public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithSameRequestVe Watch watch = cache.createWatch( ADS, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setTypeUrl(typeUrl) .setVersionInfo(SNAPSHOT2.version(typeUrl)) .addAllResourceNames(MULTIPLE_RESOURCES_SNAPSHOT2.resources(typeUrl).keySet()) - .build(), + .build()), SNAPSHOT2.resources(typeUrl).keySet(), responseTracker); @@ -329,7 +337,7 @@ public void setSnapshotWithVersionMatchingRequestShouldLeaveWatchOpenWithoutAddi cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); - Map watches = Resources.TYPE_URLS.stream() + Map watches = Resources.V2.TYPE_URLS.stream() .collect(Collectors.toMap( typeUrl -> typeUrl, typeUrl -> { @@ -337,12 +345,12 @@ public void setSnapshotWithVersionMatchingRequestShouldLeaveWatchOpenWithoutAddi Watch watch = cache.createWatch( ADS, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setTypeUrl(typeUrl) .setVersionInfo(SNAPSHOT1.version(typeUrl)) .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) - .build(), + .build()), SNAPSHOT1.resources(typeUrl).keySet(), responseTracker); @@ -350,14 +358,14 @@ public void setSnapshotWithVersionMatchingRequestShouldLeaveWatchOpenWithoutAddi })); // The request version matches the current snapshot version, so the watches shouldn't receive any responses. - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { assertThatWatchIsOpenWithNoResponses(watches.get(typeUrl)); } cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); // The request version still matches the current snapshot version, so the watches shouldn't receive any responses. - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { assertThatWatchIsOpenWithNoResponses(watches.get(typeUrl)); } } @@ -366,7 +374,7 @@ public void setSnapshotWithVersionMatchingRequestShouldLeaveWatchOpenWithoutAddi public void watchesAreReleasedAfterCancel() { SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); - Map watches = Resources.TYPE_URLS.stream() + Map watches = Resources.V2.TYPE_URLS.stream() .collect(Collectors.toMap( typeUrl -> typeUrl, typeUrl -> { @@ -374,11 +382,11 @@ public void watchesAreReleasedAfterCancel() { Watch watch = cache.createWatch( ADS, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setTypeUrl(typeUrl) .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) - .build(), + .build()), Collections.emptySet(), responseTracker); @@ -405,11 +413,11 @@ public void watchIsLeftOpenIfNotRespondedImmediately() { ResponseTracker responseTracker = new ResponseTracker(); Watch watch = cache.createWatch( true, - DiscoveryRequest.newBuilder() + XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) .setTypeUrl(ROUTE_TYPE_URL) .addAllResourceNames(Collections.singleton(ROUTE_NAME)) - .build(), + .build()), Collections.singleton(ROUTE_NAME), responseTracker); @@ -442,10 +450,13 @@ public void clearSnapshotWithWatches() { cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); - final Watch watch = cache.createWatch(ADS, DiscoveryRequest.newBuilder() + // Create a watch with an arbitrary type URL and a versionInfo that matches the saved + // snapshot, so the watch doesn't immediately close. + final Watch watch = cache.createWatch(ADS, XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) - .setTypeUrl("") - .build(), + .setTypeUrl(CLUSTER_TYPE_URL) + .setVersionInfo(SNAPSHOT1.version(CLUSTER_TYPE_URL)) + .build()), Collections.emptySet(), r -> { }); @@ -468,10 +479,10 @@ public void groups() { assertThat(cache.groups()).isEmpty(); - cache.createWatch(ADS, DiscoveryRequest.newBuilder() + cache.createWatch(ADS, XdsRequest.create(DiscoveryRequest.newBuilder() .setNode(Node.getDefaultInstance()) - .setTypeUrl("") - .build(), + .setTypeUrl(CLUSTER_TYPE_URL) + .build()), Collections.emptySet(), r -> { }); @@ -480,11 +491,11 @@ public void groups() { private static void assertThatWatchIsOpenWithNoResponses(WatchAndTracker watchAndTracker) { assertThat(watchAndTracker.watch.isCancelled()).isFalse(); - assertThat(watchAndTracker.tracker.responses).isEmpty(); + Assertions.assertThat(watchAndTracker.tracker.responses).isEmpty(); } private static void assertThatWatchReceivesSnapshot(WatchAndTracker watchAndTracker, Snapshot snapshot) { - assertThat(watchAndTracker.tracker.responses).isNotEmpty(); + Assertions.assertThat(watchAndTracker.tracker.responses).isNotEmpty(); Response response = watchAndTracker.tracker.responses.getFirst(); @@ -526,6 +537,11 @@ public String hash(Node node) { return GROUP; } + + @Override + public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + throw new IllegalStateException("should not have received a v3 Node in a v2 Test"); + } } private static class WatchAndTracker { diff --git a/cache/src/test/java/io/envoyproxy/controlplane/cache/SnapshotTest.java b/cache/src/test/java/io/envoyproxy/controlplane/cache/v2/SnapshotTest.java similarity index 91% rename from cache/src/test/java/io/envoyproxy/controlplane/cache/SnapshotTest.java rename to cache/src/test/java/io/envoyproxy/controlplane/cache/v2/SnapshotTest.java index 99c5d51d3..293e63783 100644 --- a/cache/src/test/java/io/envoyproxy/controlplane/cache/SnapshotTest.java +++ b/cache/src/test/java/io/envoyproxy/controlplane/cache/v2/SnapshotTest.java @@ -1,15 +1,18 @@ -package io.envoyproxy.controlplane.cache; +package io.envoyproxy.controlplane.cache.v2; -import static io.envoyproxy.controlplane.cache.Resources.CLUSTER_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.ENDPOINT_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.LISTENER_TYPE_URL; -import static io.envoyproxy.controlplane.cache.Resources.ROUTE_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V2.CLUSTER_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V2.ENDPOINT_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V2.LISTENER_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V2.ROUTE_TYPE_URL; +import static io.envoyproxy.envoy.api.v2.core.ApiVersion.V2; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.SnapshotConsistencyException; +import io.envoyproxy.controlplane.cache.TestResources; import io.envoyproxy.envoy.api.v2.Cluster; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.Listener; @@ -33,7 +36,8 @@ public class SnapshotTest { private static final Cluster CLUSTER = TestResources.createCluster(CLUSTER_NAME); private static final ClusterLoadAssignment ENDPOINT = TestResources.createEndpoint(CLUSTER_NAME, ENDPOINT_PORT); - private static final Listener LISTENER = TestResources.createListener(ADS, LISTENER_NAME, LISTENER_PORT, ROUTE_NAME); + private static final Listener LISTENER = TestResources.createListener(ADS, V2, V2, + LISTENER_NAME, LISTENER_PORT, ROUTE_NAME); private static final RouteConfiguration ROUTE = TestResources.createRoute(ROUTE_NAME, CLUSTER_NAME); private static final Secret SECRET = TestResources.createSecret(SECRET_NAME); @@ -139,7 +143,8 @@ public void resourcesReturnsExpectedResources() { .containsEntry(ROUTE_NAME, ROUTE) .hasSize(1); - assertThat(snapshot.resources(null)).isEmpty(); + String nullString = null; + assertThat(snapshot.version(nullString)).isEmpty(); assertThat(snapshot.resources("")).isEmpty(); assertThat(snapshot.resources(UUID.randomUUID().toString())).isEmpty(); } @@ -161,13 +166,15 @@ public void versionReturnsExpectedVersion() { assertThat(snapshot.version(LISTENER_TYPE_URL)).isEqualTo(version); assertThat(snapshot.version(ROUTE_TYPE_URL)).isEqualTo(version); - assertThat(snapshot.version(null)).isEmpty(); + String nullString = null; + assertThat(snapshot.version(nullString)).isEmpty(); assertThat(snapshot.version("")).isEmpty(); assertThat(snapshot.version(UUID.randomUUID().toString())).isEmpty(); } @Test - public void ensureConsistentReturnsWithoutExceptionForConsistentSnapshot() throws SnapshotConsistencyException { + public void ensureConsistentReturnsWithoutExceptionForConsistentSnapshot() throws + SnapshotConsistencyException { Snapshot snapshot = Snapshot.create( ImmutableList.of(CLUSTER), ImmutableList.of(ENDPOINT), diff --git a/cache/src/test/java/io/envoyproxy/controlplane/cache/v3/SimpleCacheTest.java b/cache/src/test/java/io/envoyproxy/controlplane/cache/v3/SimpleCacheTest.java new file mode 100644 index 000000000..87a41d1c3 --- /dev/null +++ b/cache/src/test/java/io/envoyproxy/controlplane/cache/v3/SimpleCacheTest.java @@ -0,0 +1,560 @@ +package io.envoyproxy.controlplane.cache.v3; + +import static io.envoyproxy.controlplane.cache.Resources.V3.CLUSTER_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V3.ROUTE_TYPE_URL; +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Response; +import io.envoyproxy.controlplane.cache.StatusInfo; +import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.XdsRequest; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.core.v3.Node; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.assertj.core.api.Assertions; +import org.junit.Test; + +public class SimpleCacheTest { + + private static final boolean ADS = ThreadLocalRandom.current().nextBoolean(); + private static final String CLUSTER_NAME = "cluster0"; + private static final String SECONDARY_CLUSTER_NAME = "cluster1"; + private static final String LISTENER_NAME = "listener0"; + private static final String ROUTE_NAME = "route0"; + private static final String SECRET_NAME = "secret0"; + + private static final String VERSION1 = UUID.randomUUID().toString(); + private static final String VERSION2 = UUID.randomUUID().toString(); + + private static final Snapshot SNAPSHOT1 = Snapshot.create( + ImmutableList.of(Cluster.newBuilder().setName(CLUSTER_NAME).build()), + ImmutableList.of(ClusterLoadAssignment.getDefaultInstance()), + ImmutableList.of(Listener.newBuilder().setName(LISTENER_NAME).build()), + ImmutableList.of(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build()), + ImmutableList.of(Secret.newBuilder().setName(SECRET_NAME).build()), + VERSION1); + + private static final Snapshot SNAPSHOT2 = Snapshot.create( + ImmutableList.of(Cluster.newBuilder().setName(CLUSTER_NAME).build()), + ImmutableList.of(ClusterLoadAssignment.getDefaultInstance()), + ImmutableList.of(Listener.newBuilder().setName(LISTENER_NAME).build()), + ImmutableList.of(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build()), + ImmutableList.of(Secret.newBuilder().setName(SECRET_NAME).build()), + VERSION2); + + private static final Snapshot MULTIPLE_RESOURCES_SNAPSHOT2 = Snapshot.create( + ImmutableList.of(Cluster.newBuilder().setName(CLUSTER_NAME).build(), + Cluster.newBuilder().setName(SECONDARY_CLUSTER_NAME).build()), + ImmutableList.of(ClusterLoadAssignment.newBuilder().setClusterName(CLUSTER_NAME).build(), + ClusterLoadAssignment.newBuilder().setClusterName(SECONDARY_CLUSTER_NAME).build()), + ImmutableList.of(Listener.newBuilder().setName(LISTENER_NAME).build()), + ImmutableList.of(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build()), + ImmutableList.of(Secret.newBuilder().setName(SECRET_NAME).build()), + VERSION2); + + @Test + public void invalidNamesListShouldReturnWatcherWithNoResponseInAdsMode() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + true, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(Resources.V3.ENDPOINT_TYPE_URL) + .addResourceNames("none") + .build()), + Collections.emptySet(), + responseTracker); + + assertThatWatchIsOpenWithNoResponses(new WatchAndTracker(watch, responseTracker)); + } + + @Test + public void invalidNamesListShouldReturnWatcherWithResponseInXdsMode() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + false, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(Resources.V3.ENDPOINT_TYPE_URL) + .addResourceNames("none") + .build()), + Collections.emptySet(), + responseTracker); + + assertThat(watch.isCancelled()).isFalse(); + Assertions.assertThat(responseTracker.responses).isNotEmpty(); + } + + @Test + public void successfullyWatchAllResourceTypesWithSetBeforeWatch() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + ADS, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(typeUrl) + .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) + .build()), + Collections.emptySet(), + responseTracker); + + assertThat(watch.request().getTypeUrl()).isEqualTo(typeUrl); + assertThat(watch.request().getResourceNamesList()).containsExactlyElementsOf( + SNAPSHOT1.resources(typeUrl).keySet()); + + assertThatWatchReceivesSnapshot(new WatchAndTracker(watch, responseTracker), SNAPSHOT1); + } + } + + @Test + public void shouldSendEdsWhenClusterChangedButEdsVersionDidnt() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + ADS, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setVersionInfo(VERSION1) + .setTypeUrl(Resources.V3.ENDPOINT_TYPE_URL) + .addAllResourceNames(SNAPSHOT1.resources(Resources.V3.ENDPOINT_TYPE_URL).keySet()) + .build()), + Sets.newHashSet(""), + responseTracker, + true); + + assertThat(watch.request().getTypeUrl()).isEqualTo(Resources.V3.ENDPOINT_TYPE_URL); + assertThat(watch.request().getResourceNamesList()).containsExactlyElementsOf( + SNAPSHOT1.resources(Resources.V3.ENDPOINT_TYPE_URL).keySet()); + + assertThatWatchReceivesSnapshot(new WatchAndTracker(watch, responseTracker), SNAPSHOT1); + } + + @Test + public void successfullyWatchAllResourceTypesWithSetAfterWatch() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + Map watches = Resources.V3.TYPE_URLS.stream() + .collect(Collectors.toMap( + typeUrl -> typeUrl, + typeUrl -> { + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + ADS, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(typeUrl) + .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) + .build()), + Collections.emptySet(), + responseTracker); + + return new WatchAndTracker(watch, responseTracker); + })); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + assertThatWatchReceivesSnapshot(watches.get(typeUrl), SNAPSHOT1); + } + } + + @Test + public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithRequestVersion() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + ResponseOrderTracker responseOrderTracker = new ResponseOrderTracker(); + + HashMap watches = new HashMap<>(); + + for (int i = 0; i < 2; ++i) { + watches.putAll(Resources.V3.TYPE_URLS.stream() + .collect(Collectors.toMap( + typeUrl -> typeUrl, + typeUrl -> { + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + ADS, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(typeUrl) + .setVersionInfo(SNAPSHOT1.version(typeUrl)) + .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) + .build()), + SNAPSHOT2.resources(typeUrl).keySet(), + r -> { + responseTracker.accept(r); + responseOrderTracker.accept(r); + }); + + return new WatchAndTracker(watch, responseTracker); + })) + ); + } + + // The request version matches the current snapshot version, so the watches shouldn't receive any responses. + for (String typeUrl : Resources.V3.TYPE_URLS) { + assertThatWatchIsOpenWithNoResponses(watches.get(typeUrl)); + } + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT2); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + assertThatWatchReceivesSnapshot(watches.get(typeUrl), SNAPSHOT2); + } + + // Verify that CDS and LDS always get triggered before EDS and RDS respectively. + assertThat(responseOrderTracker.responseTypes).containsExactly(Resources.V3.CLUSTER_TYPE_URL, + Resources.V3.CLUSTER_TYPE_URL, Resources.V3.ENDPOINT_TYPE_URL, + Resources.V3.ENDPOINT_TYPE_URL, Resources.V3.LISTENER_TYPE_URL, + Resources.V3.LISTENER_TYPE_URL, ROUTE_TYPE_URL, ROUTE_TYPE_URL, + Resources.V3.SECRET_TYPE_URL, Resources.V3.SECRET_TYPE_URL); + } + + @Test + public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithSameRequestVersionNewResourceHints() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, MULTIPLE_RESOURCES_SNAPSHOT2); + + // Set a watch for the current snapshot with the same version but with resource hints present + // in the snapshot that the watch creator does not currently know about. + // + // Note how we're requesting the resources from MULTIPLE_RESOURCE_SNAPSHOT2 while claiming we + // only know about the ones from SNAPSHOT2 + Map watches = Resources.V3.TYPE_URLS.stream() + .collect(Collectors.toMap( + typeUrl -> typeUrl, + typeUrl -> { + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + ADS, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(typeUrl) + .setVersionInfo(MULTIPLE_RESOURCES_SNAPSHOT2.version(typeUrl)) + .addAllResourceNames(MULTIPLE_RESOURCES_SNAPSHOT2.resources(typeUrl).keySet()) + .build()), + SNAPSHOT2.resources(typeUrl).keySet(), + responseTracker); + + return new WatchAndTracker(watch, responseTracker); + })); + + // The snapshot version matches for all resources, but for eds and cds there are new resources present + // for the same version, so we expect the watches to trigger. + assertThatWatchReceivesSnapshot(watches.remove(Resources.V3.CLUSTER_TYPE_URL), + MULTIPLE_RESOURCES_SNAPSHOT2); + assertThatWatchReceivesSnapshot(watches.remove(Resources.V3.ENDPOINT_TYPE_URL), + MULTIPLE_RESOURCES_SNAPSHOT2); + + // Remaining watches should not trigger + for (WatchAndTracker watchAndTracker : watches.values()) { + assertThatWatchIsOpenWithNoResponses(watchAndTracker); + } + } + + @Test + public void successfullyWatchAllResourceTypesWithSetBeforeWatchWithSameRequestVersionNewResourceHintsNoChange() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT2); + + // Set a watch for the current snapshot for the same version but with new resource hints not + // present in the snapshot that the watch creator does not know about. + // + // Note that we're requesting the additional resources found in MULTIPLE_RESOURCE_SNAPSHOT2 + // while we only know about the resources found in SNAPSHOT2. Since SNAPSHOT2 is the current + // snapshot, we have nothing to respond with for the new resources so we should not trigger + // the watch. + Map watches = Resources.V3.TYPE_URLS.stream() + .collect(Collectors.toMap( + typeUrl -> typeUrl, + typeUrl -> { + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + ADS, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(typeUrl) + .setVersionInfo(SNAPSHOT2.version(typeUrl)) + .addAllResourceNames(MULTIPLE_RESOURCES_SNAPSHOT2.resources(typeUrl).keySet()) + .build()), + SNAPSHOT2.resources(typeUrl).keySet(), + responseTracker); + + return new WatchAndTracker(watch, responseTracker); + })); + + // No watches should trigger since no new information will be returned + for (WatchAndTracker watchAndTracker : watches.values()) { + assertThatWatchIsOpenWithNoResponses(watchAndTracker); + } + } + + @Test + public void setSnapshotWithVersionMatchingRequestShouldLeaveWatchOpenWithoutAdditionalResponse() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + Map watches = Resources.V3.TYPE_URLS.stream() + .collect(Collectors.toMap( + typeUrl -> typeUrl, + typeUrl -> { + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + ADS, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(typeUrl) + .setVersionInfo(SNAPSHOT1.version(typeUrl)) + .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) + .build()), + SNAPSHOT1.resources(typeUrl).keySet(), + responseTracker); + + return new WatchAndTracker(watch, responseTracker); + })); + + // The request version matches the current snapshot version, so the watches shouldn't receive any responses. + for (String typeUrl : Resources.V3.TYPE_URLS) { + assertThatWatchIsOpenWithNoResponses(watches.get(typeUrl)); + } + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + // The request version still matches the current snapshot version, so the watches shouldn't receive any responses. + for (String typeUrl : Resources.V3.TYPE_URLS) { + assertThatWatchIsOpenWithNoResponses(watches.get(typeUrl)); + } + } + + @Test + public void watchesAreReleasedAfterCancel() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + Map watches = Resources.V3.TYPE_URLS.stream() + .collect(Collectors.toMap( + typeUrl -> typeUrl, + typeUrl -> { + ResponseTracker responseTracker = new ResponseTracker(); + + Watch watch = cache.createWatch( + ADS, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(typeUrl) + .addAllResourceNames(SNAPSHOT1.resources(typeUrl).keySet()) + .build()), + Collections.emptySet(), + responseTracker); + + return new WatchAndTracker(watch, responseTracker); + })); + + StatusInfo statusInfo = cache.statusInfo(SingleNodeGroup.GROUP); + + assertThat(statusInfo.numWatches()).isEqualTo(watches.size()); + + watches.values().forEach(w -> w.watch.cancel()); + + assertThat(statusInfo.numWatches()).isZero(); + + watches.values().forEach(w -> assertThat(w.watch.isCancelled()).isTrue()); + } + + @Test + public void watchIsLeftOpenIfNotRespondedImmediately() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + cache.setSnapshot(SingleNodeGroup.GROUP, Snapshot.create( + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), VERSION1)); + + ResponseTracker responseTracker = new ResponseTracker(); + Watch watch = cache.createWatch( + true, + XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(ROUTE_TYPE_URL) + .addAllResourceNames(Collections.singleton(ROUTE_NAME)) + .build()), + Collections.singleton(ROUTE_NAME), + responseTracker); + + assertThatWatchIsOpenWithNoResponses(new WatchAndTracker(watch, responseTracker)); + } + + @Test + public void getSnapshot() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + assertThat(cache.getSnapshot(SingleNodeGroup.GROUP)).isEqualTo(SNAPSHOT1); + } + + @Test + public void clearSnapshot() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + assertThat(cache.clearSnapshot(SingleNodeGroup.GROUP)).isTrue(); + + assertThat(cache.getSnapshot(SingleNodeGroup.GROUP)).isNull(); + } + + @Test + public void clearSnapshotWithWatches() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + cache.setSnapshot(SingleNodeGroup.GROUP, SNAPSHOT1); + + // Create a watch with an arbitrary type URL and a versionInfo that matches the saved + // snapshot, so the watch doesn't immediately close. + final Watch watch = cache.createWatch(ADS, XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(CLUSTER_TYPE_URL) + .setVersionInfo(SNAPSHOT1.version(CLUSTER_TYPE_URL)) + .build()), + Collections.emptySet(), + r -> { }); + + // clearSnapshot should fail and the snapshot should be left untouched + assertThat(cache.clearSnapshot(SingleNodeGroup.GROUP)).isFalse(); + assertThat(cache.getSnapshot(SingleNodeGroup.GROUP)).isEqualTo(SNAPSHOT1); + assertThat(cache.statusInfo(SingleNodeGroup.GROUP)).isNotNull(); + + watch.cancel(); + + // now that the watch is gone we should be able to clear it + assertThat(cache.clearSnapshot(SingleNodeGroup.GROUP)).isTrue(); + assertThat(cache.getSnapshot(SingleNodeGroup.GROUP)).isNull(); + assertThat(cache.statusInfo(SingleNodeGroup.GROUP)).isNull(); + } + + @Test + public void groups() { + SimpleCache cache = new SimpleCache<>(new SingleNodeGroup()); + + assertThat(cache.groups()).isEmpty(); + + cache.createWatch(ADS, XdsRequest.create(DiscoveryRequest.newBuilder() + .setNode(Node.getDefaultInstance()) + .setTypeUrl(CLUSTER_TYPE_URL) + .build()), + Collections.emptySet(), + r -> { }); + + assertThat(cache.groups()).containsExactly(SingleNodeGroup.GROUP); + } + + private static void assertThatWatchIsOpenWithNoResponses(WatchAndTracker watchAndTracker) { + assertThat(watchAndTracker.watch.isCancelled()).isFalse(); + Assertions.assertThat(watchAndTracker.tracker.responses).isEmpty(); + } + + private static void assertThatWatchReceivesSnapshot(WatchAndTracker watchAndTracker, Snapshot snapshot) { + Assertions.assertThat(watchAndTracker.tracker.responses).isNotEmpty(); + + Response response = watchAndTracker.tracker.responses.getFirst(); + + assertThat(response).isNotNull(); + assertThat(response.version()).isEqualTo(snapshot.version(watchAndTracker.watch.request().getTypeUrl())); + assertThat(response.resources().toArray(new Message[0])) + .containsExactlyElementsOf(snapshot.resources(watchAndTracker.watch.request().getTypeUrl()).values()); + } + + private static class ResponseTracker implements Consumer { + + private final LinkedList responses = new LinkedList<>(); + + @Override + public void accept(Response response) { + responses.add(response); + } + + } + + private static class ResponseOrderTracker implements Consumer { + + private final LinkedList responseTypes = new LinkedList<>(); + + @Override public void accept(Response response) { + responseTypes.add(response.request().getTypeUrl()); + } + } + + private static class SingleNodeGroup implements NodeGroup { + + private static final String GROUP = "node"; + + @Override + public String hash(io.envoyproxy.envoy.api.v2.core.Node node) { + throw new IllegalStateException("should not have received a v2 node in a v3 test"); + } + + @Override + public String hash(Node node) { + if (node == null) { + throw new IllegalArgumentException("node"); + } + + return GROUP; + } + } + + private static class WatchAndTracker { + + final Watch watch; + final ResponseTracker tracker; + + WatchAndTracker(Watch watch, ResponseTracker tracker) { + this.watch = watch; + this.tracker = tracker; + } + } +} diff --git a/cache/src/test/java/io/envoyproxy/controlplane/cache/v3/SnapshotTest.java b/cache/src/test/java/io/envoyproxy/controlplane/cache/v3/SnapshotTest.java new file mode 100644 index 000000000..c99b0fae4 --- /dev/null +++ b/cache/src/test/java/io/envoyproxy/controlplane/cache/v3/SnapshotTest.java @@ -0,0 +1,265 @@ +package io.envoyproxy.controlplane.cache.v3; + +import static io.envoyproxy.controlplane.cache.Resources.V3.CLUSTER_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V3.ENDPOINT_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V3.LISTENER_TYPE_URL; +import static io.envoyproxy.controlplane.cache.Resources.V3.ROUTE_TYPE_URL; +import static io.envoyproxy.envoy.config.core.v3.ApiVersion.V3; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.SnapshotConsistencyException; +import io.envoyproxy.controlplane.cache.TestResources; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import org.junit.Test; + +public class SnapshotTest { + private static final boolean ADS = ThreadLocalRandom.current().nextBoolean(); + private static final String CLUSTER_NAME = "cluster0"; + private static final String LISTENER_NAME = "listener0"; + private static final String ROUTE_NAME = "route0"; + private static final String SECRET_NAME = "secret0"; + + private static final int ENDPOINT_PORT = ThreadLocalRandom.current().nextInt(10000, 20000); + private static final int LISTENER_PORT = ThreadLocalRandom.current().nextInt(20000, 30000); + + private static final Cluster CLUSTER = TestResources.createClusterV3(CLUSTER_NAME); + private static final ClusterLoadAssignment + ENDPOINT = TestResources.createEndpointV3(CLUSTER_NAME, ENDPOINT_PORT); + private static final Listener + LISTENER = TestResources.createListenerV3(ADS, V3, V3, LISTENER_NAME, LISTENER_PORT, ROUTE_NAME); + private static final RouteConfiguration ROUTE = TestResources.createRouteV3(ROUTE_NAME, + CLUSTER_NAME); + private static final Secret SECRET = TestResources.createSecretV3(SECRET_NAME); + + @Test + public void createSingleVersionSetsResourcesCorrectly() { + final String version = UUID.randomUUID().toString(); + + Snapshot snapshot = Snapshot.create( + ImmutableList.of(CLUSTER), + ImmutableList.of(ENDPOINT), + ImmutableList.of(LISTENER), + ImmutableList.of(ROUTE), + ImmutableList.of(SECRET), + version); + + assertThat(snapshot.clusters().resources()) + .containsEntry(CLUSTER_NAME, CLUSTER) + .hasSize(1); + + assertThat(snapshot.endpoints().resources()) + .containsEntry(CLUSTER_NAME, ENDPOINT) + .hasSize(1); + + assertThat(snapshot.listeners().resources()) + .containsEntry(LISTENER_NAME, LISTENER) + .hasSize(1); + + assertThat(snapshot.routes().resources()) + .containsEntry(ROUTE_NAME, ROUTE) + .hasSize(1); + + assertThat(snapshot.clusters().version()).isEqualTo(version); + assertThat(snapshot.endpoints().version()).isEqualTo(version); + assertThat(snapshot.listeners().version()).isEqualTo(version); + assertThat(snapshot.routes().version()).isEqualTo(version); + } + + @Test + public void createSeparateVersionsSetsResourcesCorrectly() { + final String clustersVersion = UUID.randomUUID().toString(); + final String endpointsVersion = UUID.randomUUID().toString(); + final String listenersVersion = UUID.randomUUID().toString(); + final String routesVersion = UUID.randomUUID().toString(); + final String secretsVersion = UUID.randomUUID().toString(); + + Snapshot snapshot = Snapshot.create( + ImmutableList.of(CLUSTER), clustersVersion, + ImmutableList.of(ENDPOINT), endpointsVersion, + ImmutableList.of(LISTENER), listenersVersion, + ImmutableList.of(ROUTE), routesVersion, + ImmutableList.of(SECRET), secretsVersion + ); + + assertThat(snapshot.clusters().resources()) + .containsEntry(CLUSTER_NAME, CLUSTER) + .hasSize(1); + + assertThat(snapshot.endpoints().resources()) + .containsEntry(CLUSTER_NAME, ENDPOINT) + .hasSize(1); + + assertThat(snapshot.listeners().resources()) + .containsEntry(LISTENER_NAME, LISTENER) + .hasSize(1); + + assertThat(snapshot.routes().resources()) + .containsEntry(ROUTE_NAME, ROUTE) + .hasSize(1); + + assertThat(snapshot.clusters().version()).isEqualTo(clustersVersion); + assertThat(snapshot.endpoints().version()).isEqualTo(endpointsVersion); + assertThat(snapshot.listeners().version()).isEqualTo(listenersVersion); + assertThat(snapshot.routes().version()).isEqualTo(routesVersion); + } + + @Test + @SuppressWarnings("unchecked") + public void resourcesReturnsExpectedResources() { + Snapshot snapshot = Snapshot.create( + ImmutableList.of(CLUSTER), + ImmutableList.of(ENDPOINT), + ImmutableList.of(LISTENER), + ImmutableList.of(ROUTE), + ImmutableList.of(SECRET), + UUID.randomUUID().toString()); + + // We have to do some lame casting to appease java's compiler, otherwise it fails to compile + // due to limitations with + // generic type constraints. + + assertThat((Map) snapshot.resources(CLUSTER_TYPE_URL)) + .containsEntry(CLUSTER_NAME, CLUSTER) + .hasSize(1); + + assertThat((Map) snapshot.resources(ENDPOINT_TYPE_URL)) + .containsEntry(CLUSTER_NAME, ENDPOINT) + .hasSize(1); + + assertThat((Map) snapshot.resources(LISTENER_TYPE_URL)) + .containsEntry(LISTENER_NAME, LISTENER) + .hasSize(1); + + assertThat((Map) snapshot.resources(ROUTE_TYPE_URL)) + .containsEntry(ROUTE_NAME, ROUTE) + .hasSize(1); + + String nullString = null; + assertThat(snapshot.resources(nullString)).isEmpty(); + assertThat(snapshot.resources("")).isEmpty(); + assertThat(snapshot.resources(UUID.randomUUID().toString())).isEmpty(); + } + + @Test + public void versionReturnsExpectedVersion() { + final String version = UUID.randomUUID().toString(); + + Snapshot snapshot = Snapshot.create( + ImmutableList.of(CLUSTER), + ImmutableList.of(ENDPOINT), + ImmutableList.of(LISTENER), + ImmutableList.of(ROUTE), + ImmutableList.of(SECRET), + version); + + assertThat(snapshot.version(CLUSTER_TYPE_URL)).isEqualTo(version); + assertThat(snapshot.version(ENDPOINT_TYPE_URL)).isEqualTo(version); + assertThat(snapshot.version(LISTENER_TYPE_URL)).isEqualTo(version); + assertThat(snapshot.version(ROUTE_TYPE_URL)).isEqualTo(version); + + String nullString = null; + assertThat(snapshot.resources(nullString)).isEmpty(); + assertThat(snapshot.version("")).isEmpty(); + assertThat(snapshot.version(UUID.randomUUID().toString())).isEmpty(); + } + + @Test + public void ensureConsistentReturnsWithoutExceptionForConsistentSnapshot() + throws SnapshotConsistencyException { + Snapshot snapshot = Snapshot.create( + ImmutableList.of(CLUSTER), + ImmutableList.of(ENDPOINT), + ImmutableList.of(LISTENER), + ImmutableList.of(ROUTE), + ImmutableList.of(SECRET), + UUID.randomUUID().toString()); + + snapshot.ensureConsistent(); + } + + @Test + public void ensureConsistentThrowsIfEndpointOrRouteRefCountMismatch() { + Snapshot snapshot1 = Snapshot.create( + ImmutableList.of(CLUSTER), + ImmutableList.of(), + ImmutableList.of(LISTENER), + ImmutableList.of(ROUTE), + ImmutableList.of(SECRET), + UUID.randomUUID().toString()); + + assertThatThrownBy(snapshot1::ensureConsistent) + .isInstanceOf(SnapshotConsistencyException.class) + .hasMessage(format( + "Mismatched %s -> %s reference and resource lengths, [%s] != 0", + CLUSTER_TYPE_URL, + ENDPOINT_TYPE_URL, + CLUSTER_NAME)); + + Snapshot snapshot2 = Snapshot.create( + ImmutableList.of(CLUSTER), + ImmutableList.of(ENDPOINT), + ImmutableList.of(LISTENER), + ImmutableList.of(), + ImmutableList.of(SECRET), + UUID.randomUUID().toString()); + + assertThatThrownBy(snapshot2::ensureConsistent) + .isInstanceOf(SnapshotConsistencyException.class) + .hasMessage(format( + "Mismatched %s -> %s reference and resource lengths, [%s] != 0", + LISTENER_TYPE_URL, + ROUTE_TYPE_URL, + ROUTE_NAME)); + } + + @Test + public void ensureConsistentThrowsIfEndpointOrRouteNamesMismatch() { + final String otherClusterName = "someothercluster0"; + final String otherRouteName = "someotherroute0"; + + Snapshot snapshot1 = Snapshot.create( + ImmutableList.of(CLUSTER), + ImmutableList.of(TestResources.createEndpointV3(otherClusterName, ENDPOINT_PORT)), + ImmutableList.of(LISTENER), + ImmutableList.of(ROUTE), + ImmutableList.of(SECRET), + UUID.randomUUID().toString()); + + assertThatThrownBy(snapshot1::ensureConsistent) + .isInstanceOf(SnapshotConsistencyException.class) + .hasMessage(format( + "%s named '%s', referenced by a %s, not listed in [%s]", + ENDPOINT_TYPE_URL, + CLUSTER_NAME, + CLUSTER_TYPE_URL, + otherClusterName)); + + Snapshot snapshot2 = Snapshot.create( + ImmutableList.of(CLUSTER), + ImmutableList.of(ENDPOINT), + ImmutableList.of(LISTENER), + ImmutableList.of(TestResources.createRouteV3(otherRouteName, CLUSTER_NAME)), + ImmutableList.of(SECRET), + UUID.randomUUID().toString()); + + assertThatThrownBy(snapshot2::ensureConsistent) + .isInstanceOf(SnapshotConsistencyException.class) + .hasMessage(format( + "%s named '%s', referenced by a %s, not listed in [%s]", + ROUTE_TYPE_URL, + ROUTE_NAME, + LISTENER_TYPE_URL, + otherRouteName)); + } +} diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/AdsDiscoveryRequestStreamObserver.java b/server/src/main/java/io/envoyproxy/controlplane/server/AdsDiscoveryRequestStreamObserver.java index a2cee89dd..f408923ba 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/AdsDiscoveryRequestStreamObserver.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/AdsDiscoveryRequestStreamObserver.java @@ -2,10 +2,9 @@ import static io.envoyproxy.controlplane.server.DiscoveryServer.ANY_TYPE_URL; +import com.google.common.base.Preconditions; import io.envoyproxy.controlplane.cache.Resources; import io.envoyproxy.controlplane.cache.Watch; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; import io.grpc.Status; import io.grpc.stub.StreamObserver; import java.util.Collections; @@ -19,24 +18,28 @@ * {@code AdsDiscoveryRequestStreamObserver} is an implementation of {@link DiscoveryRequestStreamObserver} tailored for * ADS streams, which handle multiple watches for all TYPE_URLS. */ -public class AdsDiscoveryRequestStreamObserver extends DiscoveryRequestStreamObserver { +public class AdsDiscoveryRequestStreamObserver extends DiscoveryRequestStreamObserver { private final ConcurrentMap watches; private final ConcurrentMap latestResponse; private final ConcurrentMap> ackedResources; + private final DiscoveryServer discoveryServer; - AdsDiscoveryRequestStreamObserver(StreamObserver responseObserver, - long streamId, - Executor executor, - DiscoveryServer discoveryServer) { + AdsDiscoveryRequestStreamObserver(StreamObserver responseObserver, + long streamId, + Executor executor, + DiscoveryServer discoveryServer) { super(ANY_TYPE_URL, responseObserver, streamId, executor, discoveryServer); - this.watches = new ConcurrentHashMap<>(Resources.TYPE_URLS.size()); - this.latestResponse = new ConcurrentHashMap<>(Resources.TYPE_URLS.size()); - this.ackedResources = new ConcurrentHashMap<>(Resources.TYPE_URLS.size()); + + Preconditions.checkState(Resources.V2.TYPE_URLS.size() == Resources.V3.TYPE_URLS.size()); + this.watches = new ConcurrentHashMap<>(Resources.V2.TYPE_URLS.size()); + this.latestResponse = new ConcurrentHashMap<>(Resources.V2.TYPE_URLS.size()); + this.ackedResources = new ConcurrentHashMap<>(Resources.V2.TYPE_URLS.size()); + this.discoveryServer = discoveryServer; } @Override - public void onNext(DiscoveryRequest request) { - if (request.getTypeUrl().isEmpty()) { + public void onNext(T request) { + if (discoveryServer.wrapXdsRequest(request).getTypeUrl().isEmpty()) { closeWithError( Status.UNKNOWN .withDescription(String.format("[%d] type URL is required for ADS", streamId)) @@ -66,9 +69,9 @@ LatestDiscoveryResponse latestResponse(String typeUrl) { @Override void setLatestResponse(String typeUrl, LatestDiscoveryResponse response) { latestResponse.put(typeUrl, response); - if (typeUrl.equals(Resources.CLUSTER_TYPE_URL)) { + if (typeUrl.equals(Resources.V2.CLUSTER_TYPE_URL) || typeUrl.equals(Resources.V3.CLUSTER_TYPE_URL)) { hasClusterChanged = true; - } else if (typeUrl.equals(Resources.ENDPOINT_TYPE_URL)) { + } else if (typeUrl.equals(Resources.V2.ENDPOINT_TYPE_URL) || typeUrl.equals(Resources.V3.ENDPOINT_TYPE_URL)) { hasClusterChanged = false; } } diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryRequestStreamObserver.java b/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryRequestStreamObserver.java index 060d7a87b..49df750e9 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryRequestStreamObserver.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryRequestStreamObserver.java @@ -6,9 +6,8 @@ import io.envoyproxy.controlplane.cache.Resources; import io.envoyproxy.controlplane.cache.Response; import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.controlplane.server.exception.RequestException; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; @@ -23,8 +22,9 @@ /** * {@code DiscoveryRequestStreamObserver} provides the base implementation for XDS stream handling. + * The proto message types are abstracted so it can be used with different xDS versions. */ -public abstract class DiscoveryRequestStreamObserver implements StreamObserver { +public abstract class DiscoveryRequestStreamObserver implements StreamObserver { private static final AtomicLongFieldUpdater streamNonceUpdater = AtomicLongFieldUpdater.newUpdater(DiscoveryRequestStreamObserver.class, "streamNonce"); private static final Logger LOGGER = LoggerFactory.getLogger(DiscoveryServer.class); @@ -32,28 +32,29 @@ public abstract class DiscoveryRequestStreamObserver implements StreamObserver responseObserver; + private final StreamObserver responseObserver; private final Executor executor; - private final DiscoveryServer discoverySever; + private final DiscoveryServer discoveryServer; private volatile long streamNonce; private volatile boolean isClosing; DiscoveryRequestStreamObserver(String defaultTypeUrl, - StreamObserver responseObserver, - long streamId, - Executor executor, - DiscoveryServer discoveryServer) { + StreamObserver responseObserver, + long streamId, + Executor executor, + DiscoveryServer discoveryServer) { this.defaultTypeUrl = defaultTypeUrl; this.responseObserver = responseObserver; this.streamId = streamId; this.executor = executor; this.streamNonce = 0; - this.discoverySever = discoveryServer; + this.discoveryServer = discoveryServer; this.hasClusterChanged = false; } @Override - public void onNext(DiscoveryRequest request) { + public void onNext(T rawRequest) { + XdsRequest request = discoveryServer.wrapXdsRequest(rawRequest); String requestTypeUrl = request.getTypeUrl().isEmpty() ? defaultTypeUrl : request.getTypeUrl(); String nonce = request.getResponseNonce(); @@ -67,7 +68,7 @@ public void onNext(DiscoveryRequest request) { } try { - discoverySever.callbacks.forEach(cb -> cb.onStreamRequest(streamId, request)); + discoveryServer.runStreamRequestCallbacks(streamId, rawRequest); } catch (RequestException e) { closeWithError(e); return; @@ -81,7 +82,7 @@ public void onNext(DiscoveryRequest request) { setAckedResources(requestTypeUrl, latestDiscoveryResponse.resourceNames()); } - computeWatch(requestTypeUrl, () -> discoverySever.configWatcher.createWatch( + computeWatch(requestTypeUrl, () -> discoveryServer.configWatcher.createWatch( ads(), request, ackedResources(requestTypeUrl), @@ -98,7 +99,7 @@ public void onError(Throwable t) { } try { - discoverySever.callbacks.forEach(cb -> cb.onStreamCloseWithError(streamId, defaultTypeUrl, t)); + discoveryServer.callbacks.forEach(cb -> cb.onStreamCloseWithError(streamId, defaultTypeUrl, t)); closeWithError(Status.fromThrowable(t).asException()); } finally { cancel(); @@ -110,7 +111,7 @@ public void onCompleted() { LOGGER.debug("[{}] stream closed", streamId); try { - discoverySever.callbacks.forEach(cb -> cb.onStreamClose(streamId, defaultTypeUrl)); + discoveryServer.callbacks.forEach(cb -> cb.onStreamClose(streamId, defaultTypeUrl)); synchronized (responseObserver) { if (!isClosing) { isClosing = true; @@ -140,17 +141,16 @@ void closeWithError(Throwable exception) { private void send(Response response, String typeUrl) { String nonce = Long.toString(streamNonceUpdater.getAndIncrement(this)); - Collection resources = discoverySever.protoResourcesSerializer.serialize(response.resources()); - DiscoveryResponse discoveryResponse = DiscoveryResponse.newBuilder() - .setVersionInfo(response.version()) - .addAllResources(resources) - .setTypeUrl(typeUrl) - .setNonce(nonce) - .build(); + Collection resources = + discoveryServer.protoResourcesSerializer.serialize(response.resources(), + Resources.getResourceApiVersion(typeUrl)); - LOGGER.debug("[{}] response {} with nonce {} version {}", streamId, typeUrl, nonce, response.version()); + LOGGER.debug("[{}] response {} with nonce {} version {}", streamId, typeUrl, nonce, + response.version()); - discoverySever.callbacks.forEach(cb -> cb.onStreamResponse(streamId, response.request(), discoveryResponse)); + U discoveryResponse = discoveryServer.makeResponse(response.version(), resources, typeUrl, + nonce); + discoveryServer.runStreamResponseCallbacks(streamId, response.request(), discoveryResponse); // Store the latest response *before* we send the response. This ensures that by the time the request // is processed the map is guaranteed to be updated. Doing it afterwards leads to a race conditions @@ -162,6 +162,7 @@ private void send(Response response, String typeUrl) { response.resources().stream().map(Resources::getResourceName).collect(Collectors.toSet()) ) ); + synchronized (responseObserver) { if (!isClosing) { try { diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryServer.java b/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryServer.java index ba64b271a..ca028882d 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryServer.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryServer.java @@ -1,28 +1,20 @@ package io.envoyproxy.controlplane.server; import com.google.common.base.Preconditions; +import com.google.protobuf.Any; import io.envoyproxy.controlplane.cache.ConfigWatcher; -import io.envoyproxy.controlplane.cache.Resources; -import io.envoyproxy.controlplane.server.serializer.DefaultProtoResourcesSerializer; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.controlplane.server.serializer.ProtoResourcesSerializer; -import io.envoyproxy.envoy.api.v2.ClusterDiscoveryServiceGrpc.ClusterDiscoveryServiceImplBase; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; -import io.envoyproxy.envoy.api.v2.EndpointDiscoveryServiceGrpc.EndpointDiscoveryServiceImplBase; -import io.envoyproxy.envoy.api.v2.ListenerDiscoveryServiceGrpc.ListenerDiscoveryServiceImplBase; -import io.envoyproxy.envoy.api.v2.RouteDiscoveryServiceGrpc.RouteDiscoveryServiceImplBase; -import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; -import io.envoyproxy.envoy.service.discovery.v2.SecretDiscoveryServiceGrpc; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; -import java.util.Collections; +import java.util.Collection; import java.util.List; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DiscoveryServer { +public abstract class DiscoveryServer { static final String ANY_TYPE_URL = ""; private static final Logger LOGGER = LoggerFactory.getLogger(DiscoveryServer.class); final List callbacks; @@ -31,132 +23,39 @@ public class DiscoveryServer { private final ExecutorGroup executorGroup; private final AtomicLong streamCount = new AtomicLong(); - public DiscoveryServer(ConfigWatcher configWatcher) { - this(Collections.emptyList(), configWatcher); - } - - public DiscoveryServer(DiscoveryServerCallbacks callbacks, ConfigWatcher configWatcher) { - this(Collections.singletonList(callbacks), configWatcher); - } - - /** - * Creates the server. - * - * @param callbacks server callbacks - * @param configWatcher source of configuration updates - */ - public DiscoveryServer(List callbacks, ConfigWatcher configWatcher) { - this(callbacks, configWatcher, new DefaultExecutorGroup(), new DefaultProtoResourcesSerializer()); - } - /** * Creates the server. * * @param callbacks server callbacks * @param configWatcher source of configuration updates - * @param executorGroup executor group to use for responding stream requests * @param protoResourcesSerializer serializer of proto buffer messages */ - public DiscoveryServer(List callbacks, + protected DiscoveryServer(List callbacks, ConfigWatcher configWatcher, ExecutorGroup executorGroup, ProtoResourcesSerializer protoResourcesSerializer) { + Preconditions.checkNotNull(executorGroup, "executorGroup cannot be null"); Preconditions.checkNotNull(callbacks, "callbacks cannot be null"); Preconditions.checkNotNull(configWatcher, "configWatcher cannot be null"); - Preconditions.checkNotNull(executorGroup, "executorGroup cannot be null"); Preconditions.checkNotNull(protoResourcesSerializer, "protoResourcesSerializer cannot be null"); this.callbacks = callbacks; this.configWatcher = configWatcher; - this.executorGroup = executorGroup; this.protoResourcesSerializer = protoResourcesSerializer; + this.executorGroup = executorGroup; } - /** - * Returns an ADS implementation that uses this server's {@link ConfigWatcher}. - */ - public AggregatedDiscoveryServiceImplBase getAggregatedDiscoveryServiceImpl() { - return new AggregatedDiscoveryServiceImplBase() { - @Override - public StreamObserver streamAggregatedResources( - StreamObserver responseObserver) { - - return createRequestHandler(responseObserver, true, ANY_TYPE_URL); - } - }; - } - - /** - * Returns a CDS implementation that uses this server's {@link ConfigWatcher}. - */ - public ClusterDiscoveryServiceImplBase getClusterDiscoveryServiceImpl() { - return new ClusterDiscoveryServiceImplBase() { - @Override - public StreamObserver streamClusters( - StreamObserver responseObserver) { - - return createRequestHandler(responseObserver, false, Resources.CLUSTER_TYPE_URL); - } - }; - } - - /** - * Returns an EDS implementation that uses this server's {@link ConfigWatcher}. - */ - public EndpointDiscoveryServiceImplBase getEndpointDiscoveryServiceImpl() { - return new EndpointDiscoveryServiceImplBase() { - @Override - public StreamObserver streamEndpoints( - StreamObserver responseObserver) { + protected abstract XdsRequest wrapXdsRequest(T request); - return createRequestHandler(responseObserver, false, Resources.ENDPOINT_TYPE_URL); - } - }; - } + protected abstract U makeResponse(String version, Collection resources, String typeUrl, + String nonce); - /** - * Returns a LDS implementation that uses this server's {@link ConfigWatcher}. - */ - public ListenerDiscoveryServiceImplBase getListenerDiscoveryServiceImpl() { - return new ListenerDiscoveryServiceImplBase() { - @Override - public StreamObserver streamListeners( - StreamObserver responseObserver) { + protected abstract void runStreamRequestCallbacks(long streamId, T request); - return createRequestHandler(responseObserver, false, Resources.LISTENER_TYPE_URL); - } - }; - } - - /** - * Returns a RDS implementation that uses this server's {@link ConfigWatcher}. - */ - public RouteDiscoveryServiceImplBase getRouteDiscoveryServiceImpl() { - return new RouteDiscoveryServiceImplBase() { - @Override - public StreamObserver streamRoutes( - StreamObserver responseObserver) { - - return createRequestHandler(responseObserver, false, Resources.ROUTE_TYPE_URL); - } - }; - } - - /** - * Returns a SDS implementation that uses this server's {@link ConfigWatcher}. - */ - public SecretDiscoveryServiceGrpc.SecretDiscoveryServiceImplBase getSecretDiscoveryServiceImpl() { - return new SecretDiscoveryServiceGrpc.SecretDiscoveryServiceImplBase() { - @Override - public StreamObserver streamSecrets( - StreamObserver responseObserver) { - return createRequestHandler(responseObserver, false, Resources.SECRET_TYPE_URL); - } - }; - } + protected abstract void runStreamResponseCallbacks(long streamId, XdsRequest request, U response); - private StreamObserver createRequestHandler( - StreamObserver responseObserver, + StreamObserver createRequestHandler( + StreamObserver responseObserver, boolean ads, String defaultTypeUrl) { @@ -167,22 +66,20 @@ private StreamObserver createRequestHandler( callbacks.forEach(cb -> cb.onStreamOpen(streamId, defaultTypeUrl)); - final DiscoveryRequestStreamObserver requestStreamObserver; + final DiscoveryRequestStreamObserver requestStreamObserver; if (ads) { - requestStreamObserver = new AdsDiscoveryRequestStreamObserver( + requestStreamObserver = new AdsDiscoveryRequestStreamObserver<>( responseObserver, streamId, executor, - this - ); + this); } else { - requestStreamObserver = new XdsDiscoveryRequestStreamObserver( + requestStreamObserver = new XdsDiscoveryRequestStreamObserver<>( defaultTypeUrl, responseObserver, streamId, executor, - this - ); + this); } if (responseObserver instanceof ServerCallStreamObserver) { diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryServerCallbacks.java b/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryServerCallbacks.java index 73516230e..79a0a07c7 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryServerCallbacks.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/DiscoveryServerCallbacks.java @@ -45,7 +45,8 @@ default void onStreamOpen(long streamId, String typeUrl) { } /** - * {@code onStreamRequest} is called for each {@link DiscoveryRequest} that is received on the stream. + * {@code onStreamRequest} is called for each {@link DiscoveryRequest} that is received on the + * stream. * * @param streamId an ID for this stream that is only unique to this discovery server instance * @param request the discovery request sent by the envoy instance @@ -53,18 +54,42 @@ default void onStreamOpen(long streamId, String typeUrl) { * @throws RequestException optionally can throw {@link RequestException} with custom status. That status * will be returned to the client and the stream will be closed with error. */ - default void onStreamRequest(long streamId, DiscoveryRequest request) { + void onV2StreamRequest(long streamId, DiscoveryRequest request); - } + /** + * {@code onV3StreamRequest} is called for each {@link io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest} + * that is received on the stream. + * + * @param streamId an ID for this stream that is only unique to this discovery server instance + * @param request the discovery request sent by the envoy instance + * + * @throws RequestException optionally can throw {@link RequestException} with custom status. That status + * will be returned to the client and the stream will be closed with error. + */ + void onV3StreamRequest(long streamId, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest request); /** - * {@code onStreamResponse} is called just before each {@link DiscoveryResponse} that is sent on the stream. + * {@code onStreamResponse} is called just before each {@link DiscoveryResponse} that is sent + * on the stream. * * @param streamId an ID for this stream that is only unique to this discovery server instance * @param request the discovery request sent by the envoy instance * @param response the discovery response sent by the discovery server */ default void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryResponse response) { + } + /** + * {@code onV3StreamResponse} is called just before each + * {@link io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse} that is sent on the stream. + * + * @param streamId an ID for this stream that is only unique to this discovery server instance + * @param request the discovery request sent by the envoy instance + * @param response the discovery response sent by the discovery server + */ + default void onV3StreamResponse(long streamId, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest request, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse response) { } } diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/V2DiscoveryServer.java b/server/src/main/java/io/envoyproxy/controlplane/server/V2DiscoveryServer.java new file mode 100644 index 000000000..1d8616282 --- /dev/null +++ b/server/src/main/java/io/envoyproxy/controlplane/server/V2DiscoveryServer.java @@ -0,0 +1,161 @@ +package io.envoyproxy.controlplane.server; + +import com.google.common.base.Preconditions; +import com.google.protobuf.Any; +import io.envoyproxy.controlplane.cache.ConfigWatcher; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.XdsRequest; +import io.envoyproxy.controlplane.server.serializer.DefaultProtoResourcesSerializer; +import io.envoyproxy.controlplane.server.serializer.ProtoResourcesSerializer; +import io.envoyproxy.envoy.api.v2.ClusterDiscoveryServiceGrpc; +import io.envoyproxy.envoy.api.v2.DiscoveryRequest; +import io.envoyproxy.envoy.api.v2.DiscoveryResponse; +import io.envoyproxy.envoy.api.v2.EndpointDiscoveryServiceGrpc; +import io.envoyproxy.envoy.api.v2.ListenerDiscoveryServiceGrpc; +import io.envoyproxy.envoy.api.v2.RouteDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.discovery.v2.SecretDiscoveryServiceGrpc; +import io.grpc.stub.StreamObserver; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class V2DiscoveryServer extends DiscoveryServer { + + public V2DiscoveryServer(ConfigWatcher configWatcher) { + this(Collections.emptyList(), configWatcher); + } + + public V2DiscoveryServer(DiscoveryServerCallbacks callbacks, + ConfigWatcher configWatcher) { + this(Collections.singletonList(callbacks), configWatcher); + } + + public V2DiscoveryServer( + List callbacks, + ConfigWatcher configWatcher) { + this(callbacks, configWatcher, new DefaultExecutorGroup(), + new DefaultProtoResourcesSerializer()); + } + + public V2DiscoveryServer(List callbacks, + ConfigWatcher configWatcher, ExecutorGroup executorGroup, ProtoResourcesSerializer protoResourcesSerializer) { + super(callbacks, configWatcher, executorGroup, protoResourcesSerializer); + } + + /** + * Returns an ADS implementation that uses this server's {@link ConfigWatcher}. + */ + public AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase getAggregatedDiscoveryServiceImpl() { + return new AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase() { + @Override + public StreamObserver streamAggregatedResources( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, true, ANY_TYPE_URL); + } + }; + } + + /** + * Returns a CDS implementation that uses this server's {@link ConfigWatcher}. + */ + public ClusterDiscoveryServiceGrpc.ClusterDiscoveryServiceImplBase getClusterDiscoveryServiceImpl() { + return new ClusterDiscoveryServiceGrpc.ClusterDiscoveryServiceImplBase() { + @Override + public StreamObserver streamClusters( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, false, Resources.V2.CLUSTER_TYPE_URL); + } + }; + } + + /** + * Returns an EDS implementation that uses this server's {@link ConfigWatcher}. + */ + public EndpointDiscoveryServiceGrpc.EndpointDiscoveryServiceImplBase getEndpointDiscoveryServiceImpl() { + return new EndpointDiscoveryServiceGrpc.EndpointDiscoveryServiceImplBase() { + @Override + public StreamObserver streamEndpoints( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, false, Resources.V2.ENDPOINT_TYPE_URL); + } + }; + } + + /** + * Returns a LDS implementation that uses this server's {@link ConfigWatcher}. + */ + public ListenerDiscoveryServiceGrpc.ListenerDiscoveryServiceImplBase getListenerDiscoveryServiceImpl() { + return new ListenerDiscoveryServiceGrpc.ListenerDiscoveryServiceImplBase() { + @Override + public StreamObserver streamListeners( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, false, Resources.V2.LISTENER_TYPE_URL); + } + }; + } + + /** + * Returns a RDS implementation that uses this server's {@link ConfigWatcher}. + */ + public RouteDiscoveryServiceGrpc.RouteDiscoveryServiceImplBase getRouteDiscoveryServiceImpl() { + return new RouteDiscoveryServiceGrpc.RouteDiscoveryServiceImplBase() { + @Override + public StreamObserver streamRoutes( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, false, Resources.V2.ROUTE_TYPE_URL); + } + }; + } + + /** + * Returns a SDS implementation that uses this server's {@link ConfigWatcher}. + */ + public SecretDiscoveryServiceGrpc.SecretDiscoveryServiceImplBase getSecretDiscoveryServiceImpl() { + return new SecretDiscoveryServiceGrpc.SecretDiscoveryServiceImplBase() { + @Override + public StreamObserver streamSecrets( + StreamObserver responseObserver) { + return createRequestHandler(responseObserver, false, Resources.V2.SECRET_TYPE_URL); + } + }; + } + + @Override + protected XdsRequest wrapXdsRequest(DiscoveryRequest request) { + return XdsRequest.create(request); + } + + @Override + protected void runStreamRequestCallbacks(long streamId, DiscoveryRequest discoveryRequest) { + callbacks.forEach( + cb -> cb.onV2StreamRequest(streamId, discoveryRequest)); + } + + @Override + protected void runStreamResponseCallbacks(long streamId, XdsRequest request, + DiscoveryResponse discoveryResponse) { + Preconditions.checkArgument(request.v2Request() != null); + callbacks.forEach( + cb -> cb.onStreamResponse(streamId, + request.v2Request(), + discoveryResponse)); + } + + @Override + protected DiscoveryResponse makeResponse(String version, Collection resources, + String typeUrl, + String nonce) { + return DiscoveryResponse.newBuilder() + .setNonce(nonce) + .setVersionInfo(version) + .addAllResources(resources) + .setTypeUrl(typeUrl) + .build(); + } +} diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/V3DiscoveryServer.java b/server/src/main/java/io/envoyproxy/controlplane/server/V3DiscoveryServer.java new file mode 100644 index 000000000..601c9a059 --- /dev/null +++ b/server/src/main/java/io/envoyproxy/controlplane/server/V3DiscoveryServer.java @@ -0,0 +1,161 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase; +import static io.envoyproxy.envoy.service.endpoint.v3.EndpointDiscoveryServiceGrpc.EndpointDiscoveryServiceImplBase; +import static io.envoyproxy.envoy.service.listener.v3.ListenerDiscoveryServiceGrpc.ListenerDiscoveryServiceImplBase; +import static io.envoyproxy.envoy.service.route.v3.RouteDiscoveryServiceGrpc.RouteDiscoveryServiceImplBase; +import static io.envoyproxy.envoy.service.secret.v3.SecretDiscoveryServiceGrpc.SecretDiscoveryServiceImplBase; + +import com.google.common.base.Preconditions; +import com.google.protobuf.Any; +import io.envoyproxy.controlplane.cache.ConfigWatcher; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.XdsRequest; +import io.envoyproxy.controlplane.server.serializer.DefaultProtoResourcesSerializer; +import io.envoyproxy.controlplane.server.serializer.ProtoResourcesSerializer; +import io.envoyproxy.envoy.service.cluster.v3.ClusterDiscoveryServiceGrpc.ClusterDiscoveryServiceImplBase; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import io.grpc.stub.StreamObserver; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class V3DiscoveryServer extends DiscoveryServer { + public V3DiscoveryServer(ConfigWatcher configWatcher) { + this(Collections.emptyList(), configWatcher); + } + + public V3DiscoveryServer(DiscoveryServerCallbacks callbacks, + ConfigWatcher configWatcher) { + this(Collections.singletonList(callbacks), configWatcher); + } + + public V3DiscoveryServer( + List callbacks, + ConfigWatcher configWatcher) { + this(callbacks, configWatcher, new DefaultExecutorGroup(), + new DefaultProtoResourcesSerializer()); + } + + public V3DiscoveryServer(List callbacks, + ConfigWatcher configWatcher, ExecutorGroup executorGroup, ProtoResourcesSerializer protoResourcesSerializer) { + super(callbacks, configWatcher, executorGroup, protoResourcesSerializer); + } + + /** + * Returns an ADS implementation that uses this server's {@link ConfigWatcher}. + */ + public AggregatedDiscoveryServiceImplBase getAggregatedDiscoveryServiceImpl() { + return new AggregatedDiscoveryServiceImplBase() { + @Override + public StreamObserver streamAggregatedResources( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, true, ANY_TYPE_URL); + } + }; + } + + /** + * Returns a CDS implementation that uses this server's {@link ConfigWatcher}. + */ + public ClusterDiscoveryServiceImplBase getClusterDiscoveryServiceImpl() { + return new ClusterDiscoveryServiceImplBase() { + @Override + public StreamObserver streamClusters( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, false, Resources.V3.CLUSTER_TYPE_URL); + } + }; + } + + /** + * Returns an EDS implementation that uses this server's {@link ConfigWatcher}. + */ + public EndpointDiscoveryServiceImplBase getEndpointDiscoveryServiceImpl() { + return new EndpointDiscoveryServiceImplBase() { + @Override + public StreamObserver streamEndpoints( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, false, Resources.V3.ENDPOINT_TYPE_URL); + } + }; + } + + /** + * Returns a LDS implementation that uses this server's {@link ConfigWatcher}. + */ + public ListenerDiscoveryServiceImplBase getListenerDiscoveryServiceImpl() { + return new ListenerDiscoveryServiceImplBase() { + @Override + public StreamObserver streamListeners( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, false, Resources.V3.LISTENER_TYPE_URL); + } + }; + } + + /** + * Returns a RDS implementation that uses this server's {@link ConfigWatcher}. + */ + public RouteDiscoveryServiceImplBase getRouteDiscoveryServiceImpl() { + return new RouteDiscoveryServiceImplBase() { + @Override + public StreamObserver streamRoutes( + StreamObserver responseObserver) { + + return createRequestHandler(responseObserver, false, Resources.V3.ROUTE_TYPE_URL); + } + }; + } + + /** + * Returns a SDS implementation that uses this server's {@link ConfigWatcher}. + */ + public SecretDiscoveryServiceImplBase getSecretDiscoveryServiceImpl() { + return new SecretDiscoveryServiceImplBase() { + @Override + public StreamObserver streamSecrets( + StreamObserver responseObserver) { + return createRequestHandler(responseObserver, false, Resources.V3.SECRET_TYPE_URL); + } + }; + } + + @Override + protected XdsRequest wrapXdsRequest(DiscoveryRequest request) { + return XdsRequest.create(request); + } + + @Override + protected void runStreamRequestCallbacks(long streamId, DiscoveryRequest discoveryRequest) { + callbacks.forEach( + cb -> cb.onV3StreamRequest(streamId, discoveryRequest)); + } + + @Override + protected void runStreamResponseCallbacks(long streamId, XdsRequest request, + DiscoveryResponse discoveryResponse) { + Preconditions.checkArgument(request.v3Request() != null); + callbacks.forEach( + cb -> cb.onV3StreamResponse(streamId, + request.v3Request(), + discoveryResponse)); + } + + @Override + protected DiscoveryResponse makeResponse(String version, Collection resources, + String typeUrl, + String nonce) { + return DiscoveryResponse.newBuilder() + .setNonce(nonce) + .setVersionInfo(version) + .addAllResources(resources) + .setTypeUrl(typeUrl) + .build(); + } +} diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/XdsDiscoveryRequestStreamObserver.java b/server/src/main/java/io/envoyproxy/controlplane/server/XdsDiscoveryRequestStreamObserver.java index 3dc163597..26c3929d0 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/XdsDiscoveryRequestStreamObserver.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/XdsDiscoveryRequestStreamObserver.java @@ -1,7 +1,6 @@ package io.envoyproxy.controlplane.server; import io.envoyproxy.controlplane.cache.Watch; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; import io.grpc.stub.StreamObserver; import java.util.Collections; import java.util.Set; @@ -12,17 +11,17 @@ * {@code XdsDiscoveryRequestStreamObserver} is a lightweight implementation of {@link DiscoveryRequestStreamObserver} * tailored for non-ADS streams which handle a single watch. */ -public class XdsDiscoveryRequestStreamObserver extends DiscoveryRequestStreamObserver { +public class XdsDiscoveryRequestStreamObserver extends DiscoveryRequestStreamObserver { private volatile Watch watch; private volatile LatestDiscoveryResponse latestDiscoveryResponse; // ackedResources is only used in the same thread so it need not be volatile private Set ackedResources; XdsDiscoveryRequestStreamObserver(String defaultTypeUrl, - StreamObserver responseObserver, - long streamId, - Executor executor, - DiscoveryServer discoveryServer) { + StreamObserver responseObserver, + long streamId, + Executor executor, + DiscoveryServer discoveryServer) { super(defaultTypeUrl, responseObserver, streamId, executor, discoveryServer); this.ackedResources = Collections.emptySet(); } diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/callback/SnapshotCollectingCallback.java b/server/src/main/java/io/envoyproxy/controlplane/server/callback/SnapshotCollectingCallback.java index 1b8cecd14..8e2e3f55f 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/callback/SnapshotCollectingCallback.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/callback/SnapshotCollectingCallback.java @@ -3,8 +3,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.envoyproxy.controlplane.cache.NodeGroup; -import io.envoyproxy.controlplane.cache.Snapshot; import io.envoyproxy.controlplane.cache.SnapshotCache; +import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; import java.time.Clock; @@ -40,13 +40,14 @@ * causing it to get cleaned up and wipe the state of the other callback even though we now have an active stream * for that group. */ -public class SnapshotCollectingCallback implements DiscoveryServerCallbacks { +public class SnapshotCollectingCallback + implements DiscoveryServerCallbacks { private static class SnapshotState { int streamCount; Instant lastSeen; } - private final SnapshotCache snapshotCache; + private final SnapshotCache snapshotCache; private final NodeGroup nodeGroup; private final Clock clock; private final Set> collectorCallbacks; @@ -64,7 +65,7 @@ private static class SnapshotState { * @param collectAfterMillis how long a snapshot must be referenced for before being collected * @param collectionIntervalMillis how often the collection background action should run */ - public SnapshotCollectingCallback(SnapshotCache snapshotCache, + public SnapshotCollectingCallback(SnapshotCache snapshotCache, NodeGroup nodeGroup, Clock clock, Set> collectorCallbacks, long collectAfterMillis, long collectionIntervalMillis) { this.snapshotCache = snapshotCache; @@ -79,9 +80,19 @@ public SnapshotCollectingCallback(SnapshotCache snapshotCache, } @Override - public synchronized void onStreamRequest(long streamId, DiscoveryRequest request) { + public synchronized void onV2StreamRequest(long streamId, DiscoveryRequest request) { T groupIdentifier = nodeGroup.hash(request.getNode()); + updateState(streamId, groupIdentifier); + } + + @Override + public synchronized void onV3StreamRequest(long streamId, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest request) { + T groupIdentifier = nodeGroup.hash(request.getNode()); + updateState(streamId, groupIdentifier); + } + private void updateState(long streamId, T groupIdentifier) { SnapshotState snapshotState = this.snapshotStates.computeIfAbsent(groupIdentifier, x -> new SnapshotState()); snapshotState.lastSeen = clock.instant(); diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/serializer/CachedProtoResourcesSerializer.java b/server/src/main/java/io/envoyproxy/controlplane/server/serializer/CachedProtoResourcesSerializer.java index f5816e070..d0f22d72c 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/serializer/CachedProtoResourcesSerializer.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/serializer/CachedProtoResourcesSerializer.java @@ -1,10 +1,16 @@ package io.envoyproxy.controlplane.server.serializer; +import static io.envoyproxy.controlplane.cache.Resources.ApiVersion.V2; +import static io.envoyproxy.controlplane.cache.Resources.ApiVersion.V3; + import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.protobuf.Any; import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.Resources.ApiVersion; import io.envoyproxy.envoy.api.v2.DiscoveryResponse; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ExecutionException; /** @@ -16,19 +22,27 @@ * for the same group, because the last serialized proto instance of {@link DiscoveryResponse} is set in * DiscoveryRequestStreamObserver#latestResponse. */ -public class CachedProtoResourcesSerializer implements ProtoResourcesSerializer { +public class CachedProtoResourcesSerializer extends DefaultProtoResourcesSerializer { - private static final Cache cache = CacheBuilder.newBuilder() - .weakValues() - .build(); + private static final Map> caches = + new HashMap>() { + { + put(V2, CacheBuilder.newBuilder() + .weakValues() + .build()); + put(V3, CacheBuilder.newBuilder() + .weakValues() + .build()); + }}; /** * {@inheritDoc} */ @Override - public Any serialize(Message resource) { + public Any serialize(Message resource, ApiVersion apiVersion) { try { - return cache.get(resource, () -> Any.pack(resource)); + return caches.get(apiVersion).get(resource, + () -> super.maybeRewriteTypeUrl(Any.pack(resource), apiVersion)); } catch (ExecutionException e) { throw new ProtoSerializerException("Error while serializing resources", e); } diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/serializer/DefaultProtoResourcesSerializer.java b/server/src/main/java/io/envoyproxy/controlplane/server/serializer/DefaultProtoResourcesSerializer.java index dcafc26da..f645021ab 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/serializer/DefaultProtoResourcesSerializer.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/serializer/DefaultProtoResourcesSerializer.java @@ -2,6 +2,8 @@ import com.google.protobuf.Any; import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Resources.ApiVersion; /** * Default implementation of ProtoResourcesSerializer that uses {@link Any#pack(Message)} method on {@link Message}. @@ -12,7 +14,34 @@ public class DefaultProtoResourcesSerializer implements ProtoResourcesSerializer * {@inheritDoc} */ @Override - public Any serialize(Message resource) { - return Any.pack(resource); + public Any serialize(Message resource, ApiVersion apiVersion) { + Any output = Any.pack(resource); + + return maybeRewriteTypeUrl(output, apiVersion); + } + + protected Any maybeRewriteTypeUrl(Any output, ApiVersion apiVersion) { + // If the requested version and output version match, we can just return the output. + if (Resources.getResourceApiVersion(output.getTypeUrl()) == apiVersion) { + return output; + } + + // Rewrite the type URL to the requested version. This takes advantage of the fact that v2 + // and v3 resources are wire compatible (as long as callers are careful about deprecated v2 + // fields or new v3 fields). + // TODO: revisit this if https://github.com/envoyproxy/envoy/issues/10776 is implemented which + // would mean we don't need to do any type URL rewriting since Envoy can handle either. + switch (apiVersion) { + case V2: + return output.toBuilder() + .setTypeUrl(Resources.V3_TYPE_URLS_TO_V2.get(output.getTypeUrl())) + .build(); + case V3: + return output.toBuilder() + .setTypeUrl(Resources.V2_TYPE_URLS_TO_V3.get(output.getTypeUrl())) + .build(); + default: + throw new RuntimeException(String.format("Unsupported API version %s", apiVersion)); + } } } diff --git a/server/src/main/java/io/envoyproxy/controlplane/server/serializer/ProtoResourcesSerializer.java b/server/src/main/java/io/envoyproxy/controlplane/server/serializer/ProtoResourcesSerializer.java index 426025c2a..f8652b8fb 100644 --- a/server/src/main/java/io/envoyproxy/controlplane/server/serializer/ProtoResourcesSerializer.java +++ b/server/src/main/java/io/envoyproxy/controlplane/server/serializer/ProtoResourcesSerializer.java @@ -2,7 +2,7 @@ import com.google.protobuf.Any; import com.google.protobuf.Message; - +import io.envoyproxy.controlplane.cache.Resources.ApiVersion; import java.util.Collection; import java.util.stream.Collectors; @@ -14,10 +14,13 @@ public interface ProtoResourcesSerializer { /** * Serialize messages to proto buffers. * @param resources list of resources to serialize + * @param apiVersion the API version * @return serialized resources */ - default Collection serialize(Collection resources) { - return resources.stream().map(this::serialize).collect(Collectors.toList()); + default Collection serialize(Collection resources, ApiVersion apiVersion) { + return resources.stream() + .map(resource -> serialize(resource, apiVersion)) + .collect(Collectors.toList()); } /** @@ -25,7 +28,7 @@ default Collection serialize(Collection resources) { * @param resource the resource to serialize * @return serialized resource */ - Any serialize(Message resource); + Any serialize(Message resource, ApiVersion apiVersion); class ProtoSerializerException extends RuntimeException { ProtoSerializerException(String message, Throwable cause) { diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/TestMain.java b/server/src/test/java/io/envoyproxy/controlplane/server/TestMain.java index 8576aad71..4086945b5 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/TestMain.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/TestMain.java @@ -2,11 +2,13 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.Duration; -import io.envoyproxy.controlplane.cache.SimpleCache; -import io.envoyproxy.controlplane.cache.Snapshot; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v2.SimpleCache; +import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.envoy.api.v2.Cluster; import io.envoyproxy.envoy.api.v2.Cluster.DiscoveryType; import io.envoyproxy.envoy.api.v2.core.Address; +import io.envoyproxy.envoy.api.v2.core.Node; import io.envoyproxy.envoy.api.v2.core.SocketAddress; import io.grpc.Server; import io.grpc.ServerBuilder; @@ -18,12 +20,21 @@ public class TestMain { private static final String GROUP = "key"; /** - * Example minimal xDS implementation using the java-control-plane lib. + * Example minimal xDS implementation using the java-control-plane lib. This example configures + * a DiscoveryServer with a v2 cache, but handles v2 or v3 requests from data planes. * * @param arg command-line args */ public static void main(String[] arg) throws IOException, InterruptedException { - SimpleCache cache = new SimpleCache<>(node -> GROUP); + SimpleCache cache = new SimpleCache<>(new NodeGroup() { + @Override public String hash(Node node) { + return GROUP; + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + return GROUP; + } + }); cache.setSnapshot( GROUP, @@ -42,14 +53,20 @@ public static void main(String[] arg) throws IOException, InterruptedException { ImmutableList.of(), "1")); - DiscoveryServer discoveryServer = new DiscoveryServer(cache); + V2DiscoveryServer discoveryServer = new V2DiscoveryServer(cache); + V3DiscoveryServer v3DiscoveryServer = new V3DiscoveryServer(cache); ServerBuilder builder = NettyServerBuilder.forPort(12345) .addService(discoveryServer.getAggregatedDiscoveryServiceImpl()) .addService(discoveryServer.getClusterDiscoveryServiceImpl()) .addService(discoveryServer.getEndpointDiscoveryServiceImpl()) .addService(discoveryServer.getListenerDiscoveryServiceImpl()) - .addService(discoveryServer.getRouteDiscoveryServiceImpl()); + .addService(discoveryServer.getRouteDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getAggregatedDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getClusterDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getEndpointDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getListenerDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getRouteDiscoveryServiceImpl()); Server server = builder.build(); diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerAdsIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerAdsIT.java similarity index 73% rename from server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerAdsIT.java rename to server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerAdsIT.java index 87ca58996..6b89c3ea1 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerAdsIT.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerAdsIT.java @@ -1,14 +1,14 @@ package io.envoyproxy.controlplane.server; -import static io.envoyproxy.controlplane.server.TestSnapshots.createSnapshot; +import static io.envoyproxy.controlplane.server.V2TestSnapshots.createSnapshot; import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; -import io.envoyproxy.controlplane.cache.SimpleCache; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v2.SimpleCache; +import io.envoyproxy.envoy.api.v2.core.Node; import io.grpc.netty.NettyServerBuilder; import io.restassured.http.ContentType; import java.util.concurrent.CountDownLatch; @@ -18,9 +18,9 @@ import org.junit.rules.RuleChain; import org.testcontainers.containers.Network; -public class DiscoveryServerAdsIT { +public class V2DiscoveryServerAdsIT { - private static final String CONFIG = "envoy/ads.config.yaml"; + private static final String CONFIG = "envoy/ads.v2.config.yaml"; private static final String GROUP = "key"; private static final Integer LISTENER_PORT = 10000; @@ -31,24 +31,19 @@ public class DiscoveryServerAdsIT { private static final NettyGrpcServerRule ADS = new NettyGrpcServerRule() { @Override protected void configureServerBuilder(NettyServerBuilder builder) { - final SimpleCache cache = new SimpleCache<>(node -> GROUP); - - final DiscoveryServerCallbacks callbacks = new DiscoveryServerCallbacks() { - @Override - public void onStreamOpen(long streamId, String typeUrl) { - onStreamOpenLatch.countDown(); + final SimpleCache cache = new SimpleCache<>(new NodeGroup() { + @Override public String hash(Node node) { + return GROUP; } - @Override - public void onStreamRequest(long streamId, DiscoveryRequest request) { - onStreamRequestLatch.countDown(); + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + throw new IllegalStateException("unexpected v3 node in v2 test"); } + }); - @Override - public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryResponse response) { - onStreamResponseLatch.countDown(); - } - }; + final DiscoveryServerCallbacks callbacks = + new V2OnlyDiscoveryServerCallbacks(onStreamOpenLatch, onStreamRequestLatch, + onStreamResponseLatch); cache.setSnapshot( GROUP, @@ -62,7 +57,7 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR "1") ); - DiscoveryServer server = new DiscoveryServer(callbacks, cache); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, cache); builder.addService(server.getAggregatedDiscoveryServiceImpl()); } diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerAdsWarmingClusterIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerAdsWarmingClusterIT.java similarity index 78% rename from server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerAdsWarmingClusterIT.java rename to server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerAdsWarmingClusterIT.java index c74419c0c..dac14f5bc 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerAdsWarmingClusterIT.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerAdsWarmingClusterIT.java @@ -1,6 +1,7 @@ package io.envoyproxy.controlplane.server; -import static io.envoyproxy.controlplane.server.TestSnapshots.createSnapshot; +import static io.envoyproxy.controlplane.server.V2TestSnapshots.createSnapshot; +import static io.envoyproxy.envoy.api.v2.core.ApiVersion.V2; import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -10,9 +11,9 @@ import io.envoyproxy.controlplane.cache.CacheStatusInfo; import io.envoyproxy.controlplane.cache.NodeGroup; import io.envoyproxy.controlplane.cache.Resources; -import io.envoyproxy.controlplane.cache.SimpleCache; -import io.envoyproxy.controlplane.cache.Snapshot; import io.envoyproxy.controlplane.cache.TestResources; +import io.envoyproxy.controlplane.cache.v2.SimpleCache; +import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.envoy.api.v2.Cluster; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; @@ -22,6 +23,7 @@ import io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource; import io.envoyproxy.envoy.api.v2.core.ConfigSource; import io.envoyproxy.envoy.api.v2.core.Http2ProtocolOptions; +import io.envoyproxy.envoy.api.v2.core.Node; import io.grpc.netty.NettyServerBuilder; import io.restassured.http.ContentType; import java.util.concurrent.ConcurrentMap; @@ -36,12 +38,20 @@ import org.testcontainers.shaded.com.google.common.collect.ImmutableList; import org.testcontainers.shaded.org.apache.commons.lang.math.RandomUtils; -public class DiscoveryServerAdsWarmingClusterIT { +public class V2DiscoveryServerAdsWarmingClusterIT { - private static final String CONFIG = "envoy/ads.config.yaml"; + private static final String CONFIG = "envoy/ads.v2.config.yaml"; private static final String GROUP = "key"; private static final Integer LISTENER_PORT = 10000; - private static final CustomCache cache = new CustomCache<>(node -> GROUP); + private static final CustomCache cache = new CustomCache<>(new NodeGroup() { + @Override public String hash(Node node) { + return GROUP; + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + throw new IllegalStateException("unexpected v3 node for v2 test"); + } + }); private static final CountDownLatch onStreamOpenLatch = new CountDownLatch(1); private static final CountDownLatch onStreamRequestLatch = new CountDownLatch(1); @@ -52,24 +62,30 @@ public class DiscoveryServerAdsWarmingClusterIT { protected void configureServerBuilder(NettyServerBuilder builder) { ExecutorService executorService = Executors.newSingleThreadExecutor(); final DiscoveryServerCallbacks callbacks = new DiscoveryServerCallbacks() { - @Override - public void onStreamOpen(long streamId, String typeUrl) { - onStreamOpenLatch.countDown(); - } - - @Override - public void onStreamRequest(long streamId, DiscoveryRequest request) { - onStreamRequestLatch.countDown(); - } - - @Override - public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryResponse response) { - // Here we update a Snapshot with working cluster, but we change only CDS version, not EDS version. - // This change allows to test if EDS will be sent anyway after CDS was sent. - createSnapshotWithWorkingClusterWithTheSameEdsVersion(request, executorService); - onStreamResponseLatch.countDown(); - } - }; + @Override + public void onStreamOpen(long streamId, String typeUrl) { + onStreamOpenLatch.countDown(); + } + + @Override + public void onV2StreamRequest(long streamId, DiscoveryRequest request) { + onStreamRequestLatch.countDown(); + } + + @Override + public void onV3StreamRequest(long streamId, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest request) { + throw new IllegalStateException("unexpected v3 request for v2 test"); + } + + @Override + public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryResponse response) { + // Here we update a Snapshot with working cluster, but we change only CDS version, not EDS version. + // This change allows to test if EDS will be sent anyway after CDS was sent. + createSnapshotWithWorkingClusterWithTheSameEdsVersion(request, executorService); + onStreamResponseLatch.countDown(); + } + }; cache.setSnapshot( GROUP, @@ -81,7 +97,7 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR LISTENER_PORT, "route0")); - DiscoveryServer server = new DiscoveryServer(callbacks, cache); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, cache); builder.addService(server.getAggregatedDiscoveryServiceImpl()); } @@ -124,7 +140,7 @@ public void validateTestRequestToEchoServerViaEnvoy() throws InterruptedExceptio private static void createSnapshotWithWorkingClusterWithTheSameEdsVersion(DiscoveryRequest request, ExecutorService executorService) { - if (request.getTypeUrl().equals(Resources.CLUSTER_TYPE_URL)) { + if (request.getTypeUrl().equals(Resources.V2.CLUSTER_TYPE_URL)) { executorService.submit(() -> cache.setSnapshot( GROUP, createSnapshot(true, @@ -162,7 +178,8 @@ private static Snapshot createSnapshotWithNotWorkingCluster(boolean ads, .setType(Cluster.DiscoveryType.EDS) .build(); ClusterLoadAssignment endpoint = TestResources.createEndpoint(clusterName, endpointAddress, endpointPort); - Listener listener = TestResources.createListener(ads, listenerName, listenerPort, routeName); + Listener listener = TestResources.createListener(ads, V2, V2, listenerName, + listenerPort, routeName); RouteConfiguration route = TestResources.createRoute(routeName, clusterName); // here we have new version of resources other than CDS. @@ -194,7 +211,7 @@ public CustomCache(NodeGroup groups) { @Override protected void respondWithSpecificOrder(T group, Snapshot snapshot, - ConcurrentMap> status) { + ConcurrentMap> status) { // This code has been removed to show specific case which is hard to reproduce in integration test: // 1. Envoy connects to control-plane // 2. Snapshot already exists in control-plane <- other instance share same group diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerTest.java b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerTest.java similarity index 88% rename from server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerTest.java rename to server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerTest.java index 3c713b705..abb80b8d8 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerTest.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerTest.java @@ -16,6 +16,7 @@ import io.envoyproxy.controlplane.cache.TestResources; import io.envoyproxy.controlplane.cache.Watch; import io.envoyproxy.controlplane.cache.WatchCancelledException; +import io.envoyproxy.controlplane.cache.XdsRequest; import io.envoyproxy.controlplane.server.exception.RequestException; import io.envoyproxy.envoy.api.v2.Cluster; import io.envoyproxy.envoy.api.v2.ClusterDiscoveryServiceGrpc; @@ -32,6 +33,7 @@ import io.envoyproxy.envoy.api.v2.RouteDiscoveryServiceGrpc; import io.envoyproxy.envoy.api.v2.RouteDiscoveryServiceGrpc.RouteDiscoveryServiceStub; import io.envoyproxy.envoy.api.v2.auth.Secret; +import io.envoyproxy.envoy.api.v2.core.ApiVersion; import io.envoyproxy.envoy.api.v2.core.Node; import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc; import io.envoyproxy.envoy.service.discovery.v2.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub; @@ -57,12 +59,11 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Collectors; - import org.assertj.core.api.Condition; import org.junit.Rule; import org.junit.Test; -public class DiscoveryServerTest { +public class V2DiscoveryServerTest { private static final boolean ADS = ThreadLocalRandom.current().nextBoolean(); @@ -83,7 +84,8 @@ public class DiscoveryServerTest { private static final Cluster CLUSTER = TestResources.createCluster(CLUSTER_NAME); private static final ClusterLoadAssignment ENDPOINT = TestResources.createEndpoint(CLUSTER_NAME, ENDPOINT_PORT); - private static final Listener LISTENER = TestResources.createListener(ADS, LISTENER_NAME, LISTENER_PORT, ROUTE_NAME); + private static final Listener LISTENER = TestResources.createListener(ADS, ApiVersion.V2, ApiVersion.V2, + LISTENER_NAME, LISTENER_PORT, ROUTE_NAME); private static final RouteConfiguration ROUTE = TestResources.createRoute(ROUTE_NAME, CLUSTER_NAME); private static final Secret SECRET = TestResources.createSecret(SECRET_NAME); @@ -93,7 +95,7 @@ public class DiscoveryServerTest { @Test public void testAggregatedHandler() throws InterruptedException { MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); @@ -105,29 +107,29 @@ public void testAggregatedHandler() throws InterruptedException { requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.LISTENER_TYPE_URL) + .setTypeUrl(Resources.V2.LISTENER_TYPE_URL) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.CLUSTER_TYPE_URL) + .setTypeUrl(Resources.V2.CLUSTER_TYPE_URL) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.ENDPOINT_TYPE_URL) + .setTypeUrl(Resources.V2.ENDPOINT_TYPE_URL) .addResourceNames(CLUSTER_NAME) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.ROUTE_TYPE_URL) + .setTypeUrl(Resources.V2.ROUTE_TYPE_URL) .addResourceNames(ROUTE_NAME) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.SECRET_TYPE_URL) + .setTypeUrl(Resources.V2.SECRET_TYPE_URL) .addResourceNames(SECRET_NAME) .build()); @@ -139,13 +141,13 @@ public void testAggregatedHandler() throws InterruptedException { responseObserver.assertThatNoErrors(); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { assertThat(configWatcher.counts).containsEntry(typeUrl, 1); } - assertThat(configWatcher.counts).hasSize(Resources.TYPE_URLS.size()); + assertThat(configWatcher.counts).hasSize(Resources.V2.TYPE_URLS.size()); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { assertThat(responseObserver.responses).haveAtLeastOne(new Condition<>( r -> r.getTypeUrl().equals(typeUrl) && r.getVersionInfo().equals(VERSION), "missing expected response of type %s", typeUrl)); @@ -155,7 +157,7 @@ public void testAggregatedHandler() throws InterruptedException { @Test public void testSeparateHandlers() throws InterruptedException { MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); grpcServer.getServiceRegistry().addService(server.getClusterDiscoveryServiceImpl()); grpcServer.getServiceRegistry().addService(server.getEndpointDiscoveryServiceImpl()); @@ -169,7 +171,7 @@ public void testSeparateHandlers() throws InterruptedException { RouteDiscoveryServiceStub routeStub = RouteDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); SecretDiscoveryServiceStub secretStub = SecretDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); StreamObserver requestObserver = null; @@ -178,21 +180,21 @@ public void testSeparateHandlers() throws InterruptedException { .setTypeUrl(typeUrl); switch (typeUrl) { - case Resources.CLUSTER_TYPE_URL: + case Resources.V2.CLUSTER_TYPE_URL: requestObserver = clusterStub.streamClusters(responseObserver); break; - case Resources.ENDPOINT_TYPE_URL: + case Resources.V2.ENDPOINT_TYPE_URL: requestObserver = endpointStub.streamEndpoints(responseObserver); discoveryRequestBuilder.addResourceNames(CLUSTER_NAME); break; - case Resources.LISTENER_TYPE_URL: + case Resources.V2.LISTENER_TYPE_URL: requestObserver = listenerStub.streamListeners(responseObserver); break; - case Resources.ROUTE_TYPE_URL: + case Resources.V2.ROUTE_TYPE_URL: requestObserver = routeStub.streamRoutes(responseObserver); discoveryRequestBuilder.addResourceNames(ROUTE_NAME); break; - case Resources.SECRET_TYPE_URL: + case Resources.V2.SECRET_TYPE_URL: requestObserver = secretStub.streamSecrets(responseObserver); discoveryRequestBuilder.addResourceNames(SECRET_NAME); break; @@ -215,19 +217,19 @@ public void testSeparateHandlers() throws InterruptedException { "missing expected response of type %s", typeUrl)); } - assertThat(configWatcher.counts).hasSize(Resources.TYPE_URLS.size()); + assertThat(configWatcher.counts).hasSize(Resources.V2.TYPE_URLS.size()); } @Test public void testWatchClosed() throws InterruptedException { MockConfigWatcher configWatcher = new MockConfigWatcher(true, ImmutableTable.of()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); @@ -255,13 +257,13 @@ public void testWatchClosed() throws InterruptedException { @Test public void testSendError() throws InterruptedException { MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); responseObserver.sendError = true; @@ -283,13 +285,13 @@ public void testSendError() throws InterruptedException { @Test public void testStaleNonce() throws InterruptedException { MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); @@ -330,7 +332,7 @@ public void testStaleNonce() throws InterruptedException { @Test public void testAggregateHandlerDefaultRequestType() throws InterruptedException { MockConfigWatcher configWatcher = new MockConfigWatcher(true, ImmutableTable.of()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); @@ -356,7 +358,7 @@ public void testAggregateHandlerDefaultRequestType() throws InterruptedException @Test public void testSeparateHandlersDefaultRequestType() throws InterruptedException { MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); grpcServer.getServiceRegistry().addService(server.getClusterDiscoveryServiceImpl()); grpcServer.getServiceRegistry().addService(server.getEndpointDiscoveryServiceImpl()); @@ -370,25 +372,25 @@ public void testSeparateHandlersDefaultRequestType() throws InterruptedException RouteDiscoveryServiceStub routeStub = RouteDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); SecretDiscoveryServiceStub secretStub = SecretDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); StreamObserver requestObserver = null; switch (typeUrl) { - case Resources.CLUSTER_TYPE_URL: + case Resources.V2.CLUSTER_TYPE_URL: requestObserver = clusterStub.streamClusters(responseObserver); break; - case Resources.ENDPOINT_TYPE_URL: + case Resources.V2.ENDPOINT_TYPE_URL: requestObserver = endpointStub.streamEndpoints(responseObserver); break; - case Resources.LISTENER_TYPE_URL: + case Resources.V2.LISTENER_TYPE_URL: requestObserver = listenerStub.streamListeners(responseObserver); break; - case Resources.ROUTE_TYPE_URL: + case Resources.V2.ROUTE_TYPE_URL: requestObserver = routeStub.streamRoutes(responseObserver); break; - case Resources.SECRET_TYPE_URL: + case Resources.V2.SECRET_TYPE_URL: requestObserver = secretStub.streamSecrets(responseObserver); break; default: @@ -416,9 +418,9 @@ public void testCallbacksAggregateHandler() throws InterruptedException { final CountDownLatch streamCloseLatch = new CountDownLatch(1); final CountDownLatch streamOpenLatch = new CountDownLatch(1); final AtomicReference streamRequestLatch = - new AtomicReference<>(new CountDownLatch(Resources.TYPE_URLS.size())); + new AtomicReference<>(new CountDownLatch(Resources.V2.TYPE_URLS.size())); final AtomicReference streamResponseLatch = - new AtomicReference<>(new CountDownLatch(Resources.TYPE_URLS.size())); + new AtomicReference<>(new CountDownLatch(Resources.V2.TYPE_URLS.size())); MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { @Override @@ -450,8 +452,8 @@ public void onStreamOpen(long streamId, String typeUrl) { } @Override - public void onStreamRequest(long streamId, DiscoveryRequest request) { - super.onStreamRequest(streamId, request); + public void onV2StreamRequest(long streamId, DiscoveryRequest request) { + super.onV2StreamRequest(streamId, request); streamRequestLatch.get().countDown(); } @@ -465,7 +467,7 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR }; MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(callbacks, configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); @@ -477,7 +479,7 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.LISTENER_TYPE_URL) + .setTypeUrl(Resources.V2.LISTENER_TYPE_URL) .build()); if (!streamOpenLatch.await(1, TimeUnit.SECONDS)) { @@ -486,24 +488,24 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.CLUSTER_TYPE_URL) + .setTypeUrl(Resources.V2.CLUSTER_TYPE_URL) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.ENDPOINT_TYPE_URL) + .setTypeUrl(Resources.V2.ENDPOINT_TYPE_URL) .addResourceNames(CLUSTER_NAME) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.ROUTE_TYPE_URL) + .setTypeUrl(Resources.V2.ROUTE_TYPE_URL) .addResourceNames(ROUTE_NAME) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.SECRET_TYPE_URL) + .setTypeUrl(Resources.V2.SECRET_TYPE_URL) .addResourceNames(SECRET_NAME) .build()); @@ -517,26 +519,26 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR // Send another round of requests. These should not trigger any responses. streamResponseLatch.set(new CountDownLatch(1)); - streamRequestLatch.set(new CountDownLatch(Resources.TYPE_URLS.size())); + streamRequestLatch.set(new CountDownLatch(Resources.V2.TYPE_URLS.size())); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) .setResponseNonce("0") .setVersionInfo(VERSION) - .setTypeUrl(Resources.LISTENER_TYPE_URL) + .setTypeUrl(Resources.V2.LISTENER_TYPE_URL) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) .setResponseNonce("1") - .setTypeUrl(Resources.CLUSTER_TYPE_URL) + .setTypeUrl(Resources.V2.CLUSTER_TYPE_URL) .setVersionInfo(VERSION) .build()); requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) .setResponseNonce("2") - .setTypeUrl(Resources.ENDPOINT_TYPE_URL) + .setTypeUrl(Resources.V2.ENDPOINT_TYPE_URL) .addResourceNames(CLUSTER_NAME) .setVersionInfo(VERSION) .build()); @@ -544,7 +546,7 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) .setResponseNonce("3") - .setTypeUrl(Resources.ROUTE_TYPE_URL) + .setTypeUrl(Resources.V2.ROUTE_TYPE_URL) .addResourceNames(ROUTE_NAME) .setVersionInfo(VERSION) .build()); @@ -552,7 +554,7 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) .setResponseNonce("4") - .setTypeUrl(Resources.SECRET_TYPE_URL) + .setTypeUrl(Resources.V2.SECRET_TYPE_URL) .addResourceNames(SECRET_NAME) .setVersionInfo(VERSION) .build()); @@ -576,8 +578,8 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR assertThat(callbacks.streamCloseCount).hasValue(1); assertThat(callbacks.streamCloseWithErrorCount).hasValue(0); assertThat(callbacks.streamOpenCount).hasValue(1); - assertThat(callbacks.streamRequestCount).hasValue(Resources.TYPE_URLS.size() * 2); - assertThat(callbacks.streamResponseCount).hasValue(Resources.TYPE_URLS.size()); + assertThat(callbacks.streamRequestCount).hasValue(Resources.V2.TYPE_URLS.size() * 2); + assertThat(callbacks.streamResponseCount).hasValue(Resources.V2.TYPE_URLS.size()); } @Test @@ -587,7 +589,7 @@ public void testCallbacksSeparateHandlers() throws InterruptedException { final Map streamRequestLatches = new ConcurrentHashMap<>(); final Map streamResponseLatches = new ConcurrentHashMap<>(); - Resources.TYPE_URLS.forEach(typeUrl -> { + Resources.V2.TYPE_URLS.forEach(typeUrl -> { streamCloseLatches.put(typeUrl, new CountDownLatch(1)); streamOpenLatches.put(typeUrl, new CountDownLatch(1)); streamRequestLatches.put(typeUrl, new CountDownLatch(1)); @@ -600,10 +602,10 @@ public void testCallbacksSeparateHandlers() throws InterruptedException { public void onStreamClose(long streamId, String typeUrl) { super.onStreamClose(streamId, typeUrl); - if (!Resources.TYPE_URLS.contains(typeUrl)) { + if (!Resources.V2.TYPE_URLS.contains(typeUrl)) { this.assertionErrors.add(format( "onStreamClose#typeUrl => expected one of [%s], got %s", - String.join(",", Resources.TYPE_URLS), + String.join(",", Resources.V2.TYPE_URLS), typeUrl)); } @@ -614,10 +616,10 @@ public void onStreamClose(long streamId, String typeUrl) { public void onStreamOpen(long streamId, String typeUrl) { super.onStreamOpen(streamId, typeUrl); - if (!Resources.TYPE_URLS.contains(typeUrl)) { + if (!Resources.V2.TYPE_URLS.contains(typeUrl)) { this.assertionErrors.add(format( "onStreamOpen#typeUrl => expected one of [%s], got %s", - String.join(",", Resources.TYPE_URLS), + String.join(",", Resources.V2.TYPE_URLS), typeUrl)); } @@ -625,8 +627,8 @@ public void onStreamOpen(long streamId, String typeUrl) { } @Override - public void onStreamRequest(long streamId, DiscoveryRequest request) { - super.onStreamRequest(streamId, request); + public void onV2StreamRequest(long streamId, DiscoveryRequest request) { + super.onV2StreamRequest(streamId, request); streamRequestLatches.get(request.getTypeUrl()).countDown(); } @@ -640,7 +642,7 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR }; MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(callbacks, configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, configWatcher); grpcServer.getServiceRegistry().addService(server.getClusterDiscoveryServiceImpl()); grpcServer.getServiceRegistry().addService(server.getEndpointDiscoveryServiceImpl()); @@ -654,25 +656,25 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR RouteDiscoveryServiceStub routeStub = RouteDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); SecretDiscoveryServiceStub secretStub = SecretDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); - for (String typeUrl : Resources.TYPE_URLS) { + for (String typeUrl : Resources.V2.TYPE_URLS) { MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); StreamObserver requestObserver = null; switch (typeUrl) { - case Resources.CLUSTER_TYPE_URL: + case Resources.V2.CLUSTER_TYPE_URL: requestObserver = clusterStub.streamClusters(responseObserver); break; - case Resources.ENDPOINT_TYPE_URL: + case Resources.V2.ENDPOINT_TYPE_URL: requestObserver = endpointStub.streamEndpoints(responseObserver); break; - case Resources.LISTENER_TYPE_URL: + case Resources.V2.LISTENER_TYPE_URL: requestObserver = listenerStub.streamListeners(responseObserver); break; - case Resources.ROUTE_TYPE_URL: + case Resources.V2.ROUTE_TYPE_URL: requestObserver = routeStub.streamRoutes(responseObserver); break; - case Resources.SECRET_TYPE_URL: + case Resources.V2.SECRET_TYPE_URL: requestObserver = secretStub.streamSecrets(responseObserver); break; default: @@ -728,7 +730,7 @@ public void onStreamCloseWithError(long streamId, String typeUrl, Throwable erro }; MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(callbacks, configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); @@ -756,7 +758,7 @@ public void onStreamCloseWithError(long streamId, String typeUrl, Throwable erro @Test public void callbackOnError_logsError_onException() { MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase service = server.getAggregatedDiscoveryServiceImpl(); @@ -782,7 +784,7 @@ public void callbackOnError_logsError_onException() { @Test public void callbackOnError_doesNotLogError_whenCancelled() { MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(configWatcher); AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase service = server.getAggregatedDiscoveryServiceImpl(); @@ -823,14 +825,14 @@ public void onStreamCloseWithError(long streamId, String typeUrl, Throwable erro MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()) { @Override - public Watch createWatch(boolean ads, DiscoveryRequest request, Set knownResources, + public Watch createWatch(boolean ads, XdsRequest request, Set knownResources, Consumer responseConsumer, boolean hasClusterChanged) { watchCreated.countDown(); watch.set(super.createWatch(ads, request, knownResources, responseConsumer, false)); return watch.get(); } }; - DiscoveryServer server = new DiscoveryServer(callbacks, configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, configWatcher); grpcServer.getServiceRegistry().addService(server.getClusterDiscoveryServiceImpl()); @@ -843,7 +845,7 @@ public Watch createWatch(boolean ads, DiscoveryRequest request, Set know requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) .setResponseNonce("1") - .setTypeUrl(Resources.CLUSTER_TYPE_URL) + .setTypeUrl(Resources.V2.CLUSTER_TYPE_URL) .setVersionInfo(VERSION) .build()); @@ -870,14 +872,14 @@ public Watch createWatch(boolean ads, DiscoveryRequest request, Set know public void testCallbacksRequestException() throws InterruptedException { MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { @Override - public void onStreamRequest(long streamId, DiscoveryRequest request) { - super.onStreamRequest(streamId, request); + public void onV2StreamRequest(long streamId, DiscoveryRequest request) { + super.onV2StreamRequest(streamId, request); throw new RequestException(Status.INVALID_ARGUMENT.withDescription("request not valid")); } }; MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(callbacks, configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); @@ -887,7 +889,7 @@ public void onStreamRequest(long streamId, DiscoveryRequest request) { requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.LISTENER_TYPE_URL) + .setTypeUrl(Resources.V2.LISTENER_TYPE_URL) .build()); if (!responseObserver.errorLatch.await(1, TimeUnit.SECONDS) || responseObserver.completed.get()) { @@ -912,14 +914,14 @@ public void onStreamRequest(long streamId, DiscoveryRequest request) { public void testCallbacksOtherStatusException() throws InterruptedException { MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { @Override - public void onStreamRequest(long streamId, DiscoveryRequest request) { - super.onStreamRequest(streamId, request); + public void onV2StreamRequest(long streamId, DiscoveryRequest request) { + super.onV2StreamRequest(streamId, request); throw new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("request not valid")); } }; MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); - DiscoveryServer server = new DiscoveryServer(callbacks, configWatcher); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, configWatcher); grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); @@ -929,7 +931,7 @@ public void onStreamRequest(long streamId, DiscoveryRequest request) { requestObserver.onNext(DiscoveryRequest.newBuilder() .setNode(NODE) - .setTypeUrl(Resources.LISTENER_TYPE_URL) + .setTypeUrl(Resources.V2.LISTENER_TYPE_URL) .build()); if (!responseObserver.errorLatch.await(1, TimeUnit.SECONDS) || responseObserver.completed.get()) { @@ -952,11 +954,11 @@ public void onStreamRequest(long streamId, DiscoveryRequest request) { private static Table> createResponses() { return ImmutableTable.>builder() - .put(Resources.CLUSTER_TYPE_URL, VERSION, ImmutableList.of(CLUSTER)) - .put(Resources.ENDPOINT_TYPE_URL, VERSION, ImmutableList.of(ENDPOINT)) - .put(Resources.LISTENER_TYPE_URL, VERSION, ImmutableList.of(LISTENER)) - .put(Resources.ROUTE_TYPE_URL, VERSION, ImmutableList.of(ROUTE)) - .put(Resources.SECRET_TYPE_URL, VERSION, ImmutableList.of(SECRET)) + .put(Resources.V2.CLUSTER_TYPE_URL, VERSION, ImmutableList.of(CLUSTER)) + .put(Resources.V2.ENDPOINT_TYPE_URL, VERSION, ImmutableList.of(ENDPOINT)) + .put(Resources.V2.LISTENER_TYPE_URL, VERSION, ImmutableList.of(LISTENER)) + .put(Resources.V2.ROUTE_TYPE_URL, VERSION, ImmutableList.of(ROUTE)) + .put(Resources.V2.SECRET_TYPE_URL, VERSION, ImmutableList.of(SECRET)) .build(); } @@ -976,7 +978,7 @@ private static class MockConfigWatcher implements ConfigWatcher { @Override public Watch createWatch( boolean ads, - DiscoveryRequest request, + XdsRequest request, Set knownResources, Consumer responseConsumer, boolean hasClusterChanged) { @@ -1018,7 +1020,8 @@ public Watch createWatch( } } - private static class MockDiscoveryServerCallbacks implements DiscoveryServerCallbacks { + private static class MockDiscoveryServerCallbacks + implements DiscoveryServerCallbacks { private final AtomicInteger streamCloseCount = new AtomicInteger(); private final AtomicInteger streamCloseWithErrorCount = new AtomicInteger(); @@ -1044,7 +1047,7 @@ public void onStreamOpen(long streamId, String typeUrl) { } @Override - public void onStreamRequest(long streamId, DiscoveryRequest request) { + public void onV2StreamRequest(long streamId, DiscoveryRequest request) { streamRequestCount.getAndIncrement(); if (request == null) { @@ -1057,6 +1060,12 @@ public void onStreamRequest(long streamId, DiscoveryRequest request) { } } + @Override + public void onV3StreamRequest(long streamId, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest request) { + throw new IllegalStateException("Unexpected v3 request in v2 test"); + } + @Override public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryResponse response) { streamResponseCount.getAndIncrement(); @@ -1075,6 +1084,13 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR } } + @Override + public void onV3StreamResponse(long streamId, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest request, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse response) { + throw new IllegalStateException("Unexpected v3 response in v2 test"); + } + void assertThatNoErrors() { if (!assertionErrors.isEmpty()) { throw new AssertionError(String.join(", ", assertionErrors)); diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerV3ResourcesAdsIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerV3ResourcesAdsIT.java new file mode 100644 index 000000000..8672b7b47 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerV3ResourcesAdsIT.java @@ -0,0 +1,106 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.controlplane.server.V3TestSnapshots.createSnapshot; +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.containsString; + +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.envoy.api.v2.core.Node; +import io.grpc.netty.NettyServerBuilder; +import io.restassured.http.ContentType; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.testcontainers.containers.Network; + +/** + * Tests a v2 DiscoveryServer with a v3 SimpleCache. This tests the migration path for end users + * where the control plane migrates to v3 before the data planes migrate to the v3 ADS transport + * protocol. + */ +public class V2DiscoveryServerV3ResourcesAdsIT { + + private static final String CONFIG = "envoy/ads.v2.config.yaml"; + private static final String GROUP = "key"; + private static final Integer LISTENER_PORT = 10000; + + private static final CountDownLatch onStreamOpenLatch = new CountDownLatch(1); + private static final CountDownLatch onStreamRequestLatch = new CountDownLatch(1); + private static final CountDownLatch onStreamResponseLatch = new CountDownLatch(1); + + private static final NettyGrpcServerRule ADS = new NettyGrpcServerRule() { + @Override + protected void configureServerBuilder(NettyServerBuilder builder) { + final SimpleCache cache = new SimpleCache<>(new NodeGroup() { + @Override public String hash(Node node) { + return GROUP; + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + throw new IllegalStateException("Unexpected v3 request in v2 test"); + } + }); + + final DiscoveryServerCallbacks callbacks = + new V2OnlyDiscoveryServerCallbacks(onStreamOpenLatch, onStreamRequestLatch, + onStreamResponseLatch); + + cache.setSnapshot( + GROUP, + createSnapshot(true, + "upstream", + UPSTREAM.ipAddress(), + EchoContainer.PORT, + "listener0", + LISTENER_PORT, + "route0", + "1") + ); + + V2DiscoveryServer server = new V2DiscoveryServer(Collections.singletonList(callbacks), cache); + + builder.addService(server.getAggregatedDiscoveryServiceImpl()); + } + }; + + private static final Network NETWORK = Network.newNetwork(); + + private static final EnvoyContainer ENVOY = new EnvoyContainer(CONFIG, () -> ADS.getServer().getPort()) + .withExposedPorts(LISTENER_PORT) + .withNetwork(NETWORK); + + private static final EchoContainer UPSTREAM = new EchoContainer() + .withNetwork(NETWORK) + .withNetworkAliases("upstream"); + + @ClassRule + public static final RuleChain RULES = RuleChain.outerRule(UPSTREAM) + .around(ADS) + .around(ENVOY); + + @Test + public void validateTestRequestToEchoServerViaEnvoy() throws InterruptedException { + assertThat(onStreamOpenLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to open ADS stream"); + + assertThat(onStreamRequestLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to receive ADS request"); + + assertThat(onStreamResponseLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to send ADS response"); + + String baseUri = String.format("http://%s:%d", ENVOY.getContainerIpAddress(), ENVOY.getMappedPort(LISTENER_PORT)); + + await().atMost(5, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> given().baseUri(baseUri).contentType(ContentType.TEXT) + .when().get("/") + .then().statusCode(200) + .and().body(containsString(UPSTREAM.response))); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerV3ResourcesXdsIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerV3ResourcesXdsIT.java new file mode 100644 index 000000000..dc40fa2e4 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerV3ResourcesXdsIT.java @@ -0,0 +1,110 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.controlplane.server.V3TestSnapshots.createSnapshotNoEdsV2Transport; +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.containsString; + +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.envoy.api.v2.core.Node; +import io.grpc.netty.NettyServerBuilder; +import io.restassured.http.ContentType; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.testcontainers.containers.Network; + +/** + * Tests a V2 discovery server with a V3 cache. This provides a migration path for end users + * where they can migrate the control plane to generating v3 resources before making data planes + * consume the v2 xDS APIs. + */ +public class V2DiscoveryServerV3ResourcesXdsIT { + + private static final String CONFIG = "envoy/xds.v2.config.yaml"; + private static final String GROUP = "key"; + private static final Integer LISTENER_PORT = 10000; + + private static final CountDownLatch onStreamOpenLatch = new CountDownLatch(2); + private static final CountDownLatch onStreamRequestLatch = new CountDownLatch(2); + private static final CountDownLatch onStreamResponseLatch = new CountDownLatch(2); + + private static final NettyGrpcServerRule XDS = new NettyGrpcServerRule() { + @Override + protected void configureServerBuilder(NettyServerBuilder builder) { + final SimpleCache cache = new SimpleCache<>(new NodeGroup() { + @Override public String hash(Node node) { + return GROUP; + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + throw new IllegalStateException("Unexpected v3 request in v2 test"); + } + }); + + final DiscoveryServerCallbacks callbacks = + new V2OnlyDiscoveryServerCallbacks(onStreamOpenLatch, onStreamRequestLatch, + onStreamResponseLatch); + + // Make sure to configure v2 transport on the RDS config source on the listener, we don't + // expose v3 in this test. + cache.setSnapshot( + GROUP, + createSnapshotNoEdsV2Transport(false, + "upstream", + "upstream", + EchoContainer.PORT, + "listener0", + LISTENER_PORT, + "route0", + "1") + ); + + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, cache); + + builder.addService(server.getClusterDiscoveryServiceImpl()); + builder.addService(server.getEndpointDiscoveryServiceImpl()); + builder.addService(server.getListenerDiscoveryServiceImpl()); + builder.addService(server.getRouteDiscoveryServiceImpl()); + } + }; + + private static final Network NETWORK = Network.newNetwork(); + + private static final EnvoyContainer ENVOY = new EnvoyContainer(CONFIG, () -> XDS.getServer().getPort()) + .withExposedPorts(LISTENER_PORT) + .withNetwork(NETWORK); + + private static final EchoContainer UPSTREAM = new EchoContainer() + .withNetwork(NETWORK) + .withNetworkAliases("upstream"); + + @ClassRule + public static final RuleChain RULES = RuleChain.outerRule(UPSTREAM) + .around(XDS) + .around(ENVOY); + + @Test + public void validateTestRequestToEchoServerViaEnvoy() throws InterruptedException { + assertThat(onStreamOpenLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to open XDS streams"); + + assertThat(onStreamRequestLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to receive XDS requests"); + + assertThat(onStreamResponseLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to send XDS responses"); + + String baseUri = String.format("http://%s:%d", ENVOY.getContainerIpAddress(), ENVOY.getMappedPort(LISTENER_PORT)); + + await().atMost(5, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> given().baseUri(baseUri).contentType(ContentType.TEXT) + .when().get("/") + .then().statusCode(200) + .and().body(containsString(UPSTREAM.response))); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerXdsIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerXdsIT.java similarity index 74% rename from server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerXdsIT.java rename to server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerXdsIT.java index 2f450e864..0f0e8a2d6 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/DiscoveryServerXdsIT.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V2DiscoveryServerXdsIT.java @@ -1,14 +1,14 @@ package io.envoyproxy.controlplane.server; -import static io.envoyproxy.controlplane.server.TestSnapshots.createSnapshotNoEds; +import static io.envoyproxy.controlplane.server.V2TestSnapshots.createSnapshotNoEds; import static io.restassured.RestAssured.given; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; -import io.envoyproxy.controlplane.cache.SimpleCache; -import io.envoyproxy.envoy.api.v2.DiscoveryRequest; -import io.envoyproxy.envoy.api.v2.DiscoveryResponse; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v2.SimpleCache; +import io.envoyproxy.envoy.api.v2.core.Node; import io.grpc.netty.NettyServerBuilder; import io.restassured.http.ContentType; import java.util.concurrent.CountDownLatch; @@ -18,9 +18,9 @@ import org.junit.rules.RuleChain; import org.testcontainers.containers.Network; -public class DiscoveryServerXdsIT { +public class V2DiscoveryServerXdsIT { - private static final String CONFIG = "envoy/xds.config.yaml"; + private static final String CONFIG = "envoy/xds.v2.config.yaml"; private static final String GROUP = "key"; private static final Integer LISTENER_PORT = 10000; @@ -31,24 +31,19 @@ public class DiscoveryServerXdsIT { private static final NettyGrpcServerRule XDS = new NettyGrpcServerRule() { @Override protected void configureServerBuilder(NettyServerBuilder builder) { - final SimpleCache cache = new SimpleCache<>(node -> GROUP); - - final DiscoveryServerCallbacks callbacks = new DiscoveryServerCallbacks() { - @Override - public void onStreamOpen(long streamId, String typeUrl) { - onStreamOpenLatch.countDown(); + final SimpleCache cache = new SimpleCache<>(new NodeGroup() { + @Override public String hash(Node node) { + return GROUP; } - @Override - public void onStreamRequest(long streamId, DiscoveryRequest request) { - onStreamRequestLatch.countDown(); + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + throw new IllegalStateException("Unexpected v3 request in v2 test"); } + }); - @Override - public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryResponse response) { - onStreamResponseLatch.countDown(); - } - }; + final DiscoveryServerCallbacks callbacks = + new V2OnlyDiscoveryServerCallbacks(onStreamOpenLatch, onStreamRequestLatch, + onStreamResponseLatch); cache.setSnapshot( GROUP, @@ -62,7 +57,7 @@ public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryR "1") ); - DiscoveryServer server = new DiscoveryServer(callbacks, cache); + V2DiscoveryServer server = new V2DiscoveryServer(callbacks, cache); builder.addService(server.getClusterDiscoveryServiceImpl()); builder.addService(server.getEndpointDiscoveryServiceImpl()); diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V2OnlyDiscoveryServerCallbacks.java b/server/src/test/java/io/envoyproxy/controlplane/server/V2OnlyDiscoveryServerCallbacks.java new file mode 100644 index 000000000..d1376a812 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V2OnlyDiscoveryServerCallbacks.java @@ -0,0 +1,53 @@ +package io.envoyproxy.controlplane.server; + +import io.envoyproxy.envoy.api.v2.DiscoveryRequest; +import io.envoyproxy.envoy.api.v2.DiscoveryResponse; +import java.util.concurrent.CountDownLatch; + +public class V2OnlyDiscoveryServerCallbacks implements DiscoveryServerCallbacks { + private final CountDownLatch onStreamOpenLatch; + private final CountDownLatch onStreamRequestLatch; + private final CountDownLatch onStreamResponseLatch; + + /** + * Returns an implementation of DiscoveryServerCallbacks that throws if it sees a v3 request, + * and counts down on provided latches in response to certain events. + * + * @param onStreamOpenLatch latch to call countDown() on when a v2 stream is opened. + * @param onStreamRequestLatch latch to call countDown() on when a v2 request is seen. + * @param onStreamResponseLatch latch to call countDown() on when a v2 response is seen. + */ + public V2OnlyDiscoveryServerCallbacks(CountDownLatch onStreamOpenLatch, + CountDownLatch onStreamRequestLatch, CountDownLatch onStreamResponseLatch) { + this.onStreamOpenLatch = onStreamOpenLatch; + this.onStreamRequestLatch = onStreamRequestLatch; + this.onStreamResponseLatch = onStreamResponseLatch; + } + + @Override + public void onStreamOpen(long streamId, String typeUrl) { + onStreamOpenLatch.countDown(); + } + + @Override + public void onV2StreamRequest(long streamId, DiscoveryRequest request) { + onStreamRequestLatch.countDown(); + } + + @Override + public void onV3StreamRequest(long streamId, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest request) { + throw new IllegalStateException("unexpected v3 request in v2 test"); + } + + @Override + public void onStreamResponse(long streamId, DiscoveryRequest request, DiscoveryResponse response) { + onStreamResponseLatch.countDown(); + } + + @Override + public void onV3StreamResponse(long streamId, io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest request, + io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse response) { + throw new IllegalStateException("unexpected v3 response in v2 test"); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/TestSnapshots.java b/server/src/test/java/io/envoyproxy/controlplane/server/V2TestSnapshots.java similarity index 55% rename from server/src/test/java/io/envoyproxy/controlplane/server/TestSnapshots.java rename to server/src/test/java/io/envoyproxy/controlplane/server/V2TestSnapshots.java index c89951333..16d05c278 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/TestSnapshots.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V2TestSnapshots.java @@ -1,14 +1,15 @@ package io.envoyproxy.controlplane.server; -import io.envoyproxy.controlplane.cache.Snapshot; import io.envoyproxy.controlplane.cache.TestResources; +import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.envoy.api.v2.Cluster; import io.envoyproxy.envoy.api.v2.ClusterLoadAssignment; import io.envoyproxy.envoy.api.v2.Listener; import io.envoyproxy.envoy.api.v2.RouteConfiguration; +import io.envoyproxy.envoy.api.v2.core.ApiVersion; import org.testcontainers.shaded.com.google.common.collect.ImmutableList; -class TestSnapshots { +class V2TestSnapshots { static Snapshot createSnapshot( boolean ads, @@ -22,7 +23,8 @@ static Snapshot createSnapshot( Cluster cluster = TestResources.createCluster(clusterName); ClusterLoadAssignment endpoint = TestResources.createEndpoint(clusterName, endpointAddress, endpointPort); - Listener listener = TestResources.createListener(ads, listenerName, listenerPort, routeName); + Listener listener = TestResources.createListener(ads, ApiVersion.V2, ApiVersion.V2, + listenerName, listenerPort, routeName); RouteConfiguration route = TestResources.createRoute(routeName, clusterName); return Snapshot.create( @@ -34,6 +36,19 @@ static Snapshot createSnapshot( version); } + static Snapshot createSnapshotNoEdsV3Transport( + boolean ads, + String clusterName, + String endpointAddress, + int endpointPort, + String listenerName, + int listenerPort, + String routeName, + String version) { + return createSnapshotNoEds(ads, ApiVersion.V3, ApiVersion.V3, clusterName, endpointAddress, + endpointPort, listenerName, listenerPort, routeName, version); + } + static Snapshot createSnapshotNoEds( boolean ads, String clusterName, @@ -43,9 +58,24 @@ static Snapshot createSnapshotNoEds( int listenerPort, String routeName, String version) { + return createSnapshotNoEds(ads, ApiVersion.V2, ApiVersion.V2, clusterName, endpointAddress, + endpointPort, listenerName, listenerPort, routeName, version); + } + + private static Snapshot createSnapshotNoEds( + boolean ads, + ApiVersion rdsTransportVersion, + ApiVersion rdsResourceVersion, String clusterName, + String endpointAddress, + int endpointPort, + String listenerName, + int listenerPort, + String routeName, + String version) { Cluster cluster = TestResources.createCluster(clusterName, endpointAddress, endpointPort); - Listener listener = TestResources.createListener(ads, listenerName, listenerPort, routeName); + Listener listener = TestResources.createListener(ads, rdsTransportVersion, rdsResourceVersion, + listenerName, listenerPort, routeName); RouteConfiguration route = TestResources.createRoute(routeName, clusterName); return Snapshot.create( @@ -57,5 +87,5 @@ static Snapshot createSnapshotNoEds( version); } - private TestSnapshots() { } + private V2TestSnapshots() { } } diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerAdsIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerAdsIT.java new file mode 100644 index 000000000..f9635f459 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerAdsIT.java @@ -0,0 +1,102 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.controlplane.server.V3TestSnapshots.createSnapshot; +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.containsString; + +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.envoy.api.v2.core.Node; +import io.grpc.netty.NettyServerBuilder; +import io.restassured.http.ContentType; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.testcontainers.containers.Network; + +public class V3DiscoveryServerAdsIT { + + private static final String CONFIG = "envoy/ads.v3.config.yaml"; + private static final String GROUP = "key"; + private static final Integer LISTENER_PORT = 10000; + + private static final CountDownLatch onStreamOpenLatch = new CountDownLatch(1); + private static final CountDownLatch onStreamRequestLatch = new CountDownLatch(1); + private static final CountDownLatch onStreamResponseLatch = new CountDownLatch(1); + + private static final NettyGrpcServerRule ADS = new NettyGrpcServerRule() { + @Override + protected void configureServerBuilder(NettyServerBuilder builder) { + final SimpleCache cache = new SimpleCache<>( + new NodeGroup() { + @Override public String hash(Node node) { + throw new IllegalStateException("unexpected v2 node in v3 test"); + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + return GROUP; + } + } + ); + + final DiscoveryServerCallbacks callbacks = + new V3OnlyDiscoveryServerCallbacks(onStreamOpenLatch, onStreamRequestLatch, + onStreamResponseLatch); + + cache.setSnapshot( + GROUP, + createSnapshot(true, + "upstream", + UPSTREAM.ipAddress(), + EchoContainer.PORT, + "listener0", + LISTENER_PORT, + "route0", + "1") + ); + + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, cache); + + builder.addService(server.getAggregatedDiscoveryServiceImpl()); + } + }; + + private static final Network NETWORK = Network.newNetwork(); + + private static final EnvoyContainer ENVOY = new EnvoyContainer(CONFIG, () -> ADS.getServer().getPort()) + .withExposedPorts(LISTENER_PORT) + .withNetwork(NETWORK); + + private static final EchoContainer UPSTREAM = new EchoContainer() + .withNetwork(NETWORK) + .withNetworkAliases("upstream"); + + @ClassRule + public static final RuleChain RULES = RuleChain.outerRule(UPSTREAM) + .around(ADS) + .around(ENVOY); + + @Test + public void validateTestRequestToEchoServerViaEnvoy() throws InterruptedException { + assertThat(onStreamOpenLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to open ADS stream"); + + assertThat(onStreamRequestLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to receive ADS request"); + + assertThat(onStreamResponseLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to send ADS response"); + + String baseUri = String.format("http://%s:%d", ENVOY.getContainerIpAddress(), ENVOY.getMappedPort(LISTENER_PORT)); + + await().atMost(5, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> given().baseUri(baseUri).contentType(ContentType.TEXT) + .when().get("/") + .then().statusCode(200) + .and().body(containsString(UPSTREAM.response))); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerAdsWarmingClusterIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerAdsWarmingClusterIT.java new file mode 100644 index 000000000..4531dddae --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerAdsWarmingClusterIT.java @@ -0,0 +1,238 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.controlplane.server.V3TestSnapshots.createSnapshot; +import static io.envoyproxy.envoy.config.core.v3.ApiVersion.V3; +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.containsString; + +import com.google.protobuf.util.Durations; +import io.envoyproxy.controlplane.cache.CacheStatusInfo; +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.TestResources; +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.controlplane.cache.v3.Snapshot; +import io.envoyproxy.envoy.api.v2.core.Node; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource; +import io.envoyproxy.envoy.config.core.v3.ConfigSource; +import io.envoyproxy.envoy.config.core.v3.Http2ProtocolOptions; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import io.grpc.netty.NettyServerBuilder; +import io.restassured.http.ContentType; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.testcontainers.containers.Network; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; +import org.testcontainers.shaded.org.apache.commons.lang.math.RandomUtils; + +public class V3DiscoveryServerAdsWarmingClusterIT { + + private static final String CONFIG = "envoy/ads.v3.config.yaml"; + private static final String GROUP = "key"; + private static final Integer LISTENER_PORT = 10000; + private static final CustomCache cache = new CustomCache<>(new NodeGroup() { + @Override public String hash(Node node) { + throw new IllegalStateException("Unexpected v2 request in v3 test"); + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + return GROUP; + } + }); + + private static final CountDownLatch onStreamOpenLatch = new CountDownLatch(1); + private static final CountDownLatch onStreamRequestLatch = new CountDownLatch(1); + private static final CountDownLatch onStreamResponseLatch = new CountDownLatch(1); + + private static final NettyGrpcServerRule ADS = new NettyGrpcServerRule() { + @Override + protected void configureServerBuilder(NettyServerBuilder builder) { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + final DiscoveryServerCallbacks callbacks = new DiscoveryServerCallbacks() { + @Override + public void onStreamOpen(long streamId, String typeUrl) { + onStreamOpenLatch.countDown(); + } + + @Override + public void onV2StreamRequest(long streamId, + io.envoyproxy.envoy.api.v2.DiscoveryRequest request) { + throw new IllegalStateException("Unexpected v2 request in v3 test"); + } + + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) { + onStreamRequestLatch.countDown(); + } + + @Override + public void onStreamResponse(long streamId, io.envoyproxy.envoy.api.v2.DiscoveryRequest request, + io.envoyproxy.envoy.api.v2.DiscoveryResponse response) { + throw new IllegalStateException("Unexpected v2 response in v3 test"); + } + + @Override + public void onV3StreamResponse(long streamId, DiscoveryRequest request, + DiscoveryResponse response) { + // Here we update a Snapshot with working cluster, but we change only CDS version, not EDS version. + // This change allows to test if EDS will be sent anyway after CDS was sent. + createSnapshotWithWorkingClusterWithTheSameEdsVersion(request, executorService); + onStreamResponseLatch.countDown(); + } + }; + + cache.setSnapshot( + GROUP, + createSnapshotWithNotWorkingCluster(true, + "upstream", + UPSTREAM.ipAddress(), + EchoContainer.PORT, + "listener0", + LISTENER_PORT, + "route0")); + + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, cache); + + builder.addService(server.getAggregatedDiscoveryServiceImpl()); + } + }; + + private static final Network NETWORK = Network.newNetwork(); + + private static final EnvoyContainer ENVOY = new EnvoyContainer(CONFIG, () -> ADS.getServer().getPort()) + .withExposedPorts(LISTENER_PORT) + .withNetwork(NETWORK); + + private static final EchoContainer UPSTREAM = new EchoContainer() + .withNetwork(NETWORK) + .withNetworkAliases("upstream"); + + @ClassRule + public static final RuleChain RULES = RuleChain.outerRule(UPSTREAM) + .around(ADS) + .around(ENVOY); + + @Test + public void validateTestRequestToEchoServerViaEnvoy() throws InterruptedException { + assertThat(onStreamOpenLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to open ADS stream"); + + assertThat(onStreamRequestLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to receive ADS request"); + + assertThat(onStreamResponseLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to send ADS response"); + + String baseUri = String.format("http://%s:%d", ENVOY.getContainerIpAddress(), ENVOY.getMappedPort(LISTENER_PORT)); + + await().atMost(5, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> given().baseUri(baseUri).contentType(ContentType.TEXT) + .when().get("/") + .then().statusCode(200) + .and().body(containsString(UPSTREAM.response))); + } + + private static void createSnapshotWithWorkingClusterWithTheSameEdsVersion(DiscoveryRequest request, + ExecutorService executorService) { + if (request.getTypeUrl().equals(Resources.V3.CLUSTER_TYPE_URL)) { + executorService.submit(() -> cache.setSnapshot( + GROUP, + createSnapshot(true, + "upstream", + UPSTREAM.ipAddress(), + EchoContainer.PORT, + "listener0", + LISTENER_PORT, + "route0", + "2")) + ); + } + } + + private static Snapshot createSnapshotWithNotWorkingCluster(boolean ads, + String clusterName, + String endpointAddress, + int endpointPort, + String listenerName, + int listenerPort, + String routeName) { + + ConfigSource edsSource = ConfigSource.newBuilder() + .setAds(AggregatedConfigSource.getDefaultInstance()) + .build(); + + Cluster cluster = Cluster.newBuilder() + .setName(clusterName) + .setConnectTimeout(Durations.fromSeconds(RandomUtils.nextInt(5))) + // we are enabling HTTP2 - communication with cluster won't work + .setHttp2ProtocolOptions(Http2ProtocolOptions.newBuilder().build()) + .setEdsClusterConfig(Cluster.EdsClusterConfig.newBuilder() + .setEdsConfig(edsSource) + .setServiceName(clusterName)) + .setType(Cluster.DiscoveryType.EDS) + .build(); + ClusterLoadAssignment + endpoint = TestResources.createEndpointV3(clusterName, endpointAddress, endpointPort); + Listener listener = TestResources.createListenerV3(ads, V3, V3, listenerName, + listenerPort, routeName); + RouteConfiguration route = TestResources.createRouteV3(routeName, clusterName); + + // here we have new version of resources other than CDS. + return Snapshot.create( + ImmutableList.of(cluster), + "1", + ImmutableList.of(endpoint), + "2", + ImmutableList.of(listener), + "2", + ImmutableList.of(route), + "2", + ImmutableList.of(), + "2"); + } + + + /** + * Code has been copied from io.envoyproxy.controlplane.cache.SimpleCache to show specific case when + * Envoy might stuck with warming cluster. Class has changed lines from method respondWithSpecificOrder which are + * responsible for responding for watches. Because to reproduce this problem we need a lot of connected Envoy's and + * changes to snapshot it is easier to reproduce this way. + */ + static class CustomCache extends SimpleCache { + + public CustomCache(NodeGroup groups) { + super(groups); + } + + @Override + protected void respondWithSpecificOrder(T group, Snapshot snapshot, + ConcurrentMap> status) { + // This code has been removed to show specific case which is hard to reproduce in integration test: + // 1. Envoy connects to control-plane + // 2. Snapshot already exists in control-plane <- other instance share same group + // 3. Control-plane respond with CDS in createWatch method + // 4. There is snapshot update which change CDS and EDS versions + // 5. Envoy sends EDS request + // 6. Control-plane respond with EDS in createWatch method + // 7. Envoy resume CDS and EDS requests. + // 8. Envoy sends request CDS + // 9. Control plane respond with CDS in createWatch method + // 10. Envoy sends EDS requests + // 11. Control plane doesn't respond because version hasn't changed + // 12. Cluster of service stays in warming phase + } + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerTest.java b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerTest.java new file mode 100644 index 000000000..60ae43e41 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerTest.java @@ -0,0 +1,1175 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.envoy.config.core.v3.ApiVersion.V3; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import com.google.common.base.Strings; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableTable; +import com.google.common.collect.Table; +import com.google.protobuf.Message; +import io.envoyproxy.controlplane.cache.ConfigWatcher; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.Response; +import io.envoyproxy.controlplane.cache.TestResources; +import io.envoyproxy.controlplane.cache.Watch; +import io.envoyproxy.controlplane.cache.WatchCancelledException; +import io.envoyproxy.controlplane.cache.XdsRequest; +import io.envoyproxy.controlplane.server.exception.RequestException; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.core.v3.Node; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret; +import io.envoyproxy.envoy.service.cluster.v3.ClusterDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.cluster.v3.ClusterDiscoveryServiceGrpc.ClusterDiscoveryServiceStub; +import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import io.envoyproxy.envoy.service.endpoint.v3.EndpointDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.endpoint.v3.EndpointDiscoveryServiceGrpc.EndpointDiscoveryServiceStub; +import io.envoyproxy.envoy.service.listener.v3.ListenerDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.listener.v3.ListenerDiscoveryServiceGrpc.ListenerDiscoveryServiceStub; +import io.envoyproxy.envoy.service.route.v3.RouteDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.route.v3.RouteDiscoveryServiceGrpc.RouteDiscoveryServiceStub; +import io.envoyproxy.envoy.service.secret.v3.SecretDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.secret.v3.SecretDiscoveryServiceGrpc.SecretDiscoveryServiceStub; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; +import io.grpc.testing.GrpcServerRule; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.assertj.core.api.Condition; +import org.junit.Rule; +import org.junit.Test; + +public class V3DiscoveryServerTest { + + private static final boolean ADS = ThreadLocalRandom.current().nextBoolean(); + + private static final String CLUSTER_NAME = "cluster0"; + private static final String LISTENER_NAME = "listener0"; + private static final String ROUTE_NAME = "route0"; + private static final String SECRET_NAME = "secret0"; + + private static final int ENDPOINT_PORT = Ports.getAvailablePort(); + private static final int LISTENER_PORT = Ports.getAvailablePort(); + + private static final Node NODE = Node.newBuilder() + .setId("test-id") + .setCluster("test-cluster") + .build(); + + private static final String VERSION = Integer.toString(ThreadLocalRandom.current().nextInt(1, 1000)); + + private static final Cluster CLUSTER = TestResources.createClusterV3(CLUSTER_NAME); + private static final ClusterLoadAssignment + ENDPOINT = TestResources.createEndpointV3(CLUSTER_NAME, ENDPOINT_PORT); + private static final Listener + LISTENER = TestResources.createListenerV3(ADS, V3, V3, LISTENER_NAME, LISTENER_PORT, + ROUTE_NAME); + private static final RouteConfiguration ROUTE = TestResources.createRouteV3(ROUTE_NAME, + CLUSTER_NAME); + private static final Secret SECRET = TestResources.createSecretV3(SECRET_NAME); + + @Rule + public final GrpcServerRule grpcServer = new GrpcServerRule().directExecutor(); + + @Test + public void testAggregatedHandler() throws InterruptedException { + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.LISTENER_TYPE_URL) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.CLUSTER_TYPE_URL) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.ENDPOINT_TYPE_URL) + .addResourceNames(CLUSTER_NAME) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.ROUTE_TYPE_URL) + .addResourceNames(ROUTE_NAME) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.SECRET_TYPE_URL) + .addResourceNames(SECRET_NAME) + .build()); + + requestObserver.onCompleted(); + + if (!responseObserver.completedLatch.await(1, TimeUnit.SECONDS) || responseObserver.error.get()) { + fail(format("failed to complete request before timeout, error = %b", responseObserver.error.get())); + } + + responseObserver.assertThatNoErrors(); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + assertThat(configWatcher.counts).containsEntry(typeUrl, 1); + } + + assertThat(configWatcher.counts).hasSize(Resources.V3.TYPE_URLS.size()); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + assertThat(responseObserver.responses).haveAtLeastOne(new Condition<>( + r -> r.getTypeUrl().equals(typeUrl) && r.getVersionInfo().equals(VERSION), + "missing expected response of type %s", typeUrl)); + } + } + + @Test + public void testSeparateHandlers() throws InterruptedException { + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + grpcServer.getServiceRegistry().addService(server.getClusterDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getEndpointDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getListenerDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getRouteDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getSecretDiscoveryServiceImpl()); + + ClusterDiscoveryServiceStub clusterStub = ClusterDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + EndpointDiscoveryServiceStub endpointStub = EndpointDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + ListenerDiscoveryServiceStub listenerStub = ListenerDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + RouteDiscoveryServiceStub routeStub = RouteDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + SecretDiscoveryServiceStub secretStub = SecretDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = null; + DiscoveryRequest.Builder discoveryRequestBuilder = DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(typeUrl); + + switch (typeUrl) { + case Resources.V3.CLUSTER_TYPE_URL: + requestObserver = clusterStub.streamClusters(responseObserver); + break; + case Resources.V3.ENDPOINT_TYPE_URL: + requestObserver = endpointStub.streamEndpoints(responseObserver); + discoveryRequestBuilder.addResourceNames(CLUSTER_NAME); + break; + case Resources.V3.LISTENER_TYPE_URL: + requestObserver = listenerStub.streamListeners(responseObserver); + break; + case Resources.V3.ROUTE_TYPE_URL: + requestObserver = routeStub.streamRoutes(responseObserver); + discoveryRequestBuilder.addResourceNames(ROUTE_NAME); + break; + case Resources.V3.SECRET_TYPE_URL: + requestObserver = secretStub.streamSecrets(responseObserver); + discoveryRequestBuilder.addResourceNames(SECRET_NAME); + break; + default: + fail("Unsupported resource type: " + typeUrl); + } + + requestObserver.onNext(discoveryRequestBuilder.build()); + requestObserver.onCompleted(); + + if (!responseObserver.completedLatch.await(1, TimeUnit.SECONDS) || responseObserver.error.get()) { + fail(format("failed to complete request before timeout, error = %b", responseObserver.error.get())); + } + + responseObserver.assertThatNoErrors(); + + assertThat(configWatcher.counts).containsEntry(typeUrl, 1); + assertThat(responseObserver.responses).haveAtLeastOne(new Condition<>( + r -> r.getTypeUrl().equals(typeUrl) && r.getVersionInfo().equals(VERSION), + "missing expected response of type %s", typeUrl)); + } + + assertThat(configWatcher.counts).hasSize(Resources.V3.TYPE_URLS.size()); + } + + @Test + public void testWatchClosed() throws InterruptedException { + MockConfigWatcher configWatcher = new MockConfigWatcher(true, ImmutableTable.of()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(typeUrl) + .build()); + + requestObserver.onError(new RuntimeException("send error")); + + if (!responseObserver.errorLatch.await(1, TimeUnit.SECONDS) + || responseObserver.completed.get() + || !responseObserver.responses.isEmpty()) { + fail(format("failed to error before timeout, completed = %b, responses.count = %d", + responseObserver.completed.get(), + responseObserver.responses.size())); + } + + responseObserver.assertThatNoErrors(); + } + } + + @Test + public void testSendError() throws InterruptedException { + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + responseObserver.sendError = true; + + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(typeUrl) + .build()); + + if (!responseObserver.errorLatch.await(1, TimeUnit.SECONDS) || responseObserver.completed.get()) { + fail(format("failed to error before timeout, completed = %b", responseObserver.completed.get())); + } + + responseObserver.assertThatNoErrors(); + } + } + + @Test + public void testStaleNonce() throws InterruptedException { + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(typeUrl) + .build()); + + // Stale request, should not create a new watch. + requestObserver.onNext( + DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(typeUrl) + .setResponseNonce("xyz") + .build()); + + // Fresh request, should create a new watch. + requestObserver.onNext( + DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(typeUrl) + .setResponseNonce("0") + .setVersionInfo("0") + .build()); + + requestObserver.onCompleted(); + + if (!responseObserver.completedLatch.await(1, TimeUnit.SECONDS) || responseObserver.error.get()) { + fail(format("failed to complete request before timeout, error = %b", responseObserver.error.get())); + } + + // Assert that 2 watches have been created for this resource type. + assertThat(configWatcher.counts.get(typeUrl)).isEqualTo(2); + } + } + + @Test + public void testAggregateHandlerDefaultRequestType() throws InterruptedException { + MockConfigWatcher configWatcher = new MockConfigWatcher(true, ImmutableTable.of()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + // Leave off the type URL. For ADS requests it should fail because the type URL is required. + requestObserver.onNext( + DiscoveryRequest.newBuilder() + .setNode(NODE) + .build()); + + requestObserver.onCompleted(); + + if (!responseObserver.errorLatch.await(1, TimeUnit.SECONDS) || responseObserver.completed.get()) { + fail(format("failed to error before timeout, completed = %b", responseObserver.completed.get())); + } + } + + @Test + public void testSeparateHandlersDefaultRequestType() throws InterruptedException { + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + grpcServer.getServiceRegistry().addService(server.getClusterDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getEndpointDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getListenerDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getRouteDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getSecretDiscoveryServiceImpl()); + + ClusterDiscoveryServiceStub clusterStub = ClusterDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + EndpointDiscoveryServiceStub endpointStub = EndpointDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + ListenerDiscoveryServiceStub listenerStub = ListenerDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + RouteDiscoveryServiceStub routeStub = RouteDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + SecretDiscoveryServiceStub secretStub = SecretDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = null; + + switch (typeUrl) { + case Resources.V3.CLUSTER_TYPE_URL: + requestObserver = clusterStub.streamClusters(responseObserver); + break; + case Resources.V3.ENDPOINT_TYPE_URL: + requestObserver = endpointStub.streamEndpoints(responseObserver); + break; + case Resources.V3.LISTENER_TYPE_URL: + requestObserver = listenerStub.streamListeners(responseObserver); + break; + case Resources.V3.ROUTE_TYPE_URL: + requestObserver = routeStub.streamRoutes(responseObserver); + break; + case Resources.V3.SECRET_TYPE_URL: + requestObserver = secretStub.streamSecrets(responseObserver); + break; + default: + fail("Unsupported resource type: " + typeUrl); + } + + // Leave off the type URL. For xDS requests it should default to the value for that handler's type. + DiscoveryRequest discoveryRequest = DiscoveryRequest.newBuilder() + .setNode(NODE) + .build(); + + requestObserver.onNext(discoveryRequest); + requestObserver.onCompleted(); + + if (!responseObserver.completedLatch.await(1, TimeUnit.SECONDS) || responseObserver.error.get()) { + fail(format("failed to complete request before timeout, error = %b", responseObserver.error.get())); + } + + responseObserver.assertThatNoErrors(); + } + } + + @Test + public void testCallbacksAggregateHandler() throws InterruptedException { + final CountDownLatch streamCloseLatch = new CountDownLatch(1); + final CountDownLatch streamOpenLatch = new CountDownLatch(1); + final AtomicReference streamRequestLatch = + new AtomicReference<>(new CountDownLatch(Resources.V3.TYPE_URLS.size())); + final AtomicReference streamResponseLatch = + new AtomicReference<>(new CountDownLatch(Resources.V3.TYPE_URLS.size())); + + MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { + @Override + public void onStreamClose(long streamId, String typeUrl) { + super.onStreamClose(streamId, typeUrl); + + if (!typeUrl.equals(DiscoveryServer.ANY_TYPE_URL)) { + this.assertionErrors.add(format( + "onStreamClose#typeUrl => expected %s, got %s", + DiscoveryServer.ANY_TYPE_URL, + typeUrl)); + } + + streamCloseLatch.countDown(); + } + + @Override + public void onStreamOpen(long streamId, String typeUrl) { + super.onStreamOpen(streamId, typeUrl); + + if (!typeUrl.equals(DiscoveryServer.ANY_TYPE_URL)) { + this.assertionErrors.add(format( + "onStreamOpen#typeUrl => expected %s, got %s", + DiscoveryServer.ANY_TYPE_URL, + typeUrl)); + } + + streamOpenLatch.countDown(); + } + + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) { + super.onV3StreamRequest(streamId, request); + streamRequestLatch.get().countDown(); + } + + @Override + public void onV3StreamResponse(long streamId, DiscoveryRequest request, + DiscoveryResponse response) { + super.onV3StreamResponse(streamId, request, response); + + streamResponseLatch.get().countDown(); + } + }; + + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.LISTENER_TYPE_URL) + .build()); + + if (!streamOpenLatch.await(1, TimeUnit.SECONDS)) { + fail("failed to execute onStreamOpen callback before timeout"); + } + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.CLUSTER_TYPE_URL) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.ENDPOINT_TYPE_URL) + .addResourceNames(CLUSTER_NAME) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.ROUTE_TYPE_URL) + .addResourceNames(ROUTE_NAME) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.SECRET_TYPE_URL) + .addResourceNames(SECRET_NAME) + .build()); + + if (!streamRequestLatch.get().await(1, TimeUnit.SECONDS)) { + fail("failed to execute onStreamRequest callback before timeout"); + } + + if (!streamResponseLatch.get().await(1, TimeUnit.SECONDS)) { + fail("failed to execute onStreamResponse callback before timeout"); + } + + // Send another round of requests. These should not trigger any responses. + streamResponseLatch.set(new CountDownLatch(1)); + streamRequestLatch.set(new CountDownLatch(Resources.V3.TYPE_URLS.size())); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setResponseNonce("0") + .setVersionInfo(VERSION) + .setTypeUrl(Resources.V3.LISTENER_TYPE_URL) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setResponseNonce("1") + .setTypeUrl(Resources.V3.CLUSTER_TYPE_URL) + .setVersionInfo(VERSION) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setResponseNonce("2") + .setTypeUrl(Resources.V3.ENDPOINT_TYPE_URL) + .addResourceNames(CLUSTER_NAME) + .setVersionInfo(VERSION) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setResponseNonce("3") + .setTypeUrl(Resources.V3.ROUTE_TYPE_URL) + .addResourceNames(ROUTE_NAME) + .setVersionInfo(VERSION) + .build()); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setResponseNonce("4") + .setTypeUrl(Resources.V3.SECRET_TYPE_URL) + .addResourceNames(SECRET_NAME) + .setVersionInfo(VERSION) + .build()); + + if (!streamRequestLatch.get().await(1, TimeUnit.SECONDS)) { + fail("failed to execute onStreamRequest callback before timeout"); + } + + if (streamResponseLatch.get().await(1, TimeUnit.SECONDS)) { + fail("unexpected onStreamResponse callback"); + } + + requestObserver.onCompleted(); + + if (!streamCloseLatch.await(1, TimeUnit.SECONDS)) { + fail("failed to execute onStreamClose callback before timeout"); + } + + callbacks.assertThatNoErrors(); + + assertThat(callbacks.streamCloseCount).hasValue(1); + assertThat(callbacks.streamCloseWithErrorCount).hasValue(0); + assertThat(callbacks.streamOpenCount).hasValue(1); + assertThat(callbacks.streamRequestCount).hasValue(Resources.V3.TYPE_URLS.size() * 2); + assertThat(callbacks.streamResponseCount).hasValue(Resources.V3.TYPE_URLS.size()); + } + + @Test + public void testCallbacksSeparateHandlers() throws InterruptedException { + final Map streamCloseLatches = new ConcurrentHashMap<>(); + final Map streamOpenLatches = new ConcurrentHashMap<>(); + final Map streamRequestLatches = new ConcurrentHashMap<>(); + final Map streamResponseLatches = new ConcurrentHashMap<>(); + + Resources.V3.TYPE_URLS.forEach(typeUrl -> { + streamCloseLatches.put(typeUrl, new CountDownLatch(1)); + streamOpenLatches.put(typeUrl, new CountDownLatch(1)); + streamRequestLatches.put(typeUrl, new CountDownLatch(1)); + streamResponseLatches.put(typeUrl, new CountDownLatch(1)); + }); + + MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { + + @Override + public void onStreamClose(long streamId, String typeUrl) { + super.onStreamClose(streamId, typeUrl); + + if (!Resources.V3.TYPE_URLS.contains(typeUrl)) { + this.assertionErrors.add(format( + "onStreamClose#typeUrl => expected one of [%s], got %s", + String.join(",", Resources.V3.TYPE_URLS), + typeUrl)); + } + + streamCloseLatches.get(typeUrl).countDown(); + } + + @Override + public void onStreamOpen(long streamId, String typeUrl) { + super.onStreamOpen(streamId, typeUrl); + + if (!Resources.V3.TYPE_URLS.contains(typeUrl)) { + this.assertionErrors.add(format( + "onStreamOpen#typeUrl => expected one of [%s], got %s", + String.join(",", Resources.V3.TYPE_URLS), + typeUrl)); + } + + streamOpenLatches.get(typeUrl).countDown(); + } + + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) { + super.onV3StreamRequest(streamId, request); + + streamRequestLatches.get(request.getTypeUrl()).countDown(); + } + + @Override + public void onV3StreamResponse(long streamId, DiscoveryRequest request, + DiscoveryResponse response) { + super.onV3StreamResponse(streamId, request, response); + + streamResponseLatches.get(request.getTypeUrl()).countDown(); + } + }; + + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, configWatcher); + + grpcServer.getServiceRegistry().addService(server.getClusterDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getEndpointDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getListenerDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getRouteDiscoveryServiceImpl()); + grpcServer.getServiceRegistry().addService(server.getSecretDiscoveryServiceImpl()); + + ClusterDiscoveryServiceStub clusterStub = ClusterDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + EndpointDiscoveryServiceStub endpointStub = EndpointDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + ListenerDiscoveryServiceStub listenerStub = ListenerDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + RouteDiscoveryServiceStub routeStub = RouteDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + SecretDiscoveryServiceStub secretStub = SecretDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + for (String typeUrl : Resources.V3.TYPE_URLS) { + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = null; + + switch (typeUrl) { + case Resources.V3.CLUSTER_TYPE_URL: + requestObserver = clusterStub.streamClusters(responseObserver); + break; + case Resources.V3.ENDPOINT_TYPE_URL: + requestObserver = endpointStub.streamEndpoints(responseObserver); + break; + case Resources.V3.LISTENER_TYPE_URL: + requestObserver = listenerStub.streamListeners(responseObserver); + break; + case Resources.V3.ROUTE_TYPE_URL: + requestObserver = routeStub.streamRoutes(responseObserver); + break; + case Resources.V3.SECRET_TYPE_URL: + requestObserver = secretStub.streamSecrets(responseObserver); + break; + default: + fail("Unsupported resource type: " + typeUrl); + } + + DiscoveryRequest discoveryRequest = DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(typeUrl) + .build(); + + requestObserver.onNext(discoveryRequest); + + if (!streamOpenLatches.get(typeUrl).await(1, TimeUnit.SECONDS)) { + fail(format("failed to execute onStreamOpen callback for typeUrl %s before timeout", typeUrl)); + } + + if (!streamRequestLatches.get(typeUrl).await(1, TimeUnit.SECONDS)) { + fail(format("failed to execute onStreamOpen callback for typeUrl %s before timeout", typeUrl)); + } + + requestObserver.onCompleted(); + + if (!streamResponseLatches.get(typeUrl).await(1, TimeUnit.SECONDS)) { + fail(format("failed to execute onStreamResponse callback for typeUrl %s before timeout", typeUrl)); + } + + if (!streamCloseLatches.get(typeUrl).await(1, TimeUnit.SECONDS)) { + fail(format("failed to execute onStreamClose callback for typeUrl %s before timeout", typeUrl)); + } + } + + callbacks.assertThatNoErrors(); + + assertThat(callbacks.streamCloseCount).hasValue(5); + assertThat(callbacks.streamCloseWithErrorCount).hasValue(0); + assertThat(callbacks.streamOpenCount).hasValue(5); + assertThat(callbacks.streamRequestCount).hasValue(5); + assertThat(callbacks.streamResponseCount).hasValue(5); + } + + @Test + public void testCallbacksOnError() throws InterruptedException { + final CountDownLatch streamCloseWithErrorLatch = new CountDownLatch(1); + + MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { + @Override + public void onStreamCloseWithError(long streamId, String typeUrl, Throwable error) { + super.onStreamCloseWithError(streamId, typeUrl, error); + + streamCloseWithErrorLatch.countDown(); + } + }; + + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + requestObserver.onError(new RuntimeException("send error")); + + if (!streamCloseWithErrorLatch.await(1, TimeUnit.SECONDS)) { + fail("failed to execute onStreamCloseWithError callback before timeout"); + } + + callbacks.assertThatNoErrors(); + + assertThat(callbacks.streamCloseCount).hasValue(0); + assertThat(callbacks.streamCloseWithErrorCount).hasValue(1); + assertThat(callbacks.streamOpenCount).hasValue(1); + assertThat(callbacks.streamRequestCount).hasValue(0); + assertThat(callbacks.streamResponseCount).hasValue(0); + } + + @Test + public void callbackOnError_logsError_onException() { + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase service = + server.getAggregatedDiscoveryServiceImpl(); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + StreamObserver requestObserver = service.streamAggregatedResources(responseObserver); + + try { + ByteArrayOutputStream stdErr = new ByteArrayOutputStream(); + System.setErr(new PrintStream(stdErr)); + + requestObserver.onError(new StatusRuntimeException(Status.INTERNAL + .withDescription("internal error") + .withCause(new RuntimeException("some error")))); + + assertThat(stdErr.toString()).contains("ERROR "); + assertThat(stdErr.toString()).contains("io.grpc.StatusRuntimeException: INTERNAL: internal error"); + } finally { + System.setErr(System.err); + } + } + + @Test + public void callbackOnError_doesNotLogError_whenCancelled() { + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(configWatcher); + + AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase service = + server.getAggregatedDiscoveryServiceImpl(); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + StreamObserver requestObserver = service.streamAggregatedResources(responseObserver); + + try { + ByteArrayOutputStream stdErr = new ByteArrayOutputStream(); + System.setErr(new PrintStream(stdErr)); + + requestObserver.onError(new StatusRuntimeException(Status.CANCELLED + .withDescription("internal error") + .withCause(new RuntimeException("some error")))); + + assertThat(stdErr.toString()).doesNotContain("ERROR "); + assertThat(stdErr.toString()).doesNotContain("io.grpc.StatusRuntimeException: CANCELLED:"); + } finally { + System.setErr(System.err); + } + } + + @Test + public void testCallbacksOnCancelled() throws InterruptedException, ClassNotFoundException { + final CountDownLatch streamCloseWithErrorLatch = new CountDownLatch(1); + final CountDownLatch watchCreated = new CountDownLatch(1); + AtomicReference watch = new AtomicReference<>(); + + MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { + @Override + public void onStreamCloseWithError(long streamId, String typeUrl, Throwable error) { + // watch should already be closed by the time we report a stream close error + assertThat(watch.get().isCancelled()).isTrue(); + super.onStreamCloseWithError(streamId, typeUrl, error); + streamCloseWithErrorLatch.countDown(); + } + }; + + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()) { + @Override + public Watch createWatch(boolean ads, XdsRequest request, Set knownResources, + Consumer responseConsumer, boolean hasClusterChanged) { + watchCreated.countDown(); + watch.set(super.createWatch(ads, request, knownResources, responseConsumer, false)); + return watch.get(); + } + }; + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, configWatcher); + + grpcServer.getServiceRegistry().addService(server.getClusterDiscoveryServiceImpl()); + + ClusterDiscoveryServiceStub stub = ClusterDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + + StreamObserver requestObserver = stub.streamClusters(responseObserver); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setResponseNonce("1") + .setTypeUrl(Resources.V3.CLUSTER_TYPE_URL) + .setVersionInfo(VERSION) + .build()); + + if (!watchCreated.await(1, TimeUnit.SECONDS)) { + fail("failed to execute watchCreated callback before timeout"); + } + + requestObserver.onError(Status.CANCELLED.asException()); + + if (!streamCloseWithErrorLatch.await(1, TimeUnit.SECONDS)) { + fail("failed to execute onStreamCloseWithError callback before timeout"); + } + + callbacks.assertThatNoErrors(); + + assertThat(callbacks.streamCloseCount).hasValue(0); + assertThat(callbacks.streamCloseWithErrorCount).hasValue(1); + assertThat(callbacks.streamOpenCount).hasValue(1); + assertThat(callbacks.streamRequestCount).hasValue(1); + assertThat(callbacks.streamResponseCount).hasValue(1); + } + + @Test + public void testCallbacksRequestException() throws InterruptedException { + MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) { + super.onV3StreamRequest(streamId, request); + throw new RequestException(Status.INVALID_ARGUMENT.withDescription("request not valid")); + } + }; + + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.LISTENER_TYPE_URL) + .build()); + + if (!responseObserver.errorLatch.await(1, TimeUnit.SECONDS) || responseObserver.completed.get()) { + fail(format("failed to error before timeout, completed = %b", responseObserver.completed.get())); + } + + callbacks.assertThatNoErrors(); + + assertThat(responseObserver.errorException).isInstanceOfSatisfying(StatusRuntimeException.class, ex -> { + assertThat(ex.getStatus().getCode()).isEqualTo(Status.Code.INVALID_ARGUMENT); + assertThat(ex.getStatus().getDescription()).isEqualTo("request not valid"); + }); + + assertThat(callbacks.streamCloseCount).hasValue(0); + assertThat(callbacks.streamCloseWithErrorCount).hasValue(0); + assertThat(callbacks.streamOpenCount).hasValue(1); + assertThat(callbacks.streamRequestCount).hasValue(1); + assertThat(callbacks.streamResponseCount).hasValue(0); + } + + @Test + public void testCallbacksOtherStatusException() throws InterruptedException { + MockDiscoveryServerCallbacks callbacks = new MockDiscoveryServerCallbacks() { + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) { + super.onV3StreamRequest(streamId, request); + throw new StatusRuntimeException(Status.INVALID_ARGUMENT.withDescription("request not valid")); + } + }; + + MockConfigWatcher configWatcher = new MockConfigWatcher(false, createResponses()); + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, configWatcher); + + grpcServer.getServiceRegistry().addService(server.getAggregatedDiscoveryServiceImpl()); + AggregatedDiscoveryServiceStub stub = AggregatedDiscoveryServiceGrpc.newStub(grpcServer.getChannel()); + + MockDiscoveryResponseObserver responseObserver = new MockDiscoveryResponseObserver(); + StreamObserver requestObserver = stub.streamAggregatedResources(responseObserver); + + requestObserver.onNext(DiscoveryRequest.newBuilder() + .setNode(NODE) + .setTypeUrl(Resources.V3.LISTENER_TYPE_URL) + .build()); + + if (!responseObserver.errorLatch.await(1, TimeUnit.SECONDS) || responseObserver.completed.get()) { + fail(format("failed to error before timeout, completed = %b", responseObserver.completed.get())); + } + + callbacks.assertThatNoErrors(); + + assertThat(responseObserver.errorException).isInstanceOfSatisfying(StatusRuntimeException.class, ex -> { + assertThat(ex.getStatus().getCode()).isEqualTo(Status.Code.UNKNOWN); + assertThat(ex.getStatus().getDescription()).isNull(); + }); + + assertThat(callbacks.streamCloseCount).hasValue(0); + assertThat(callbacks.streamCloseWithErrorCount).hasValue(0); + assertThat(callbacks.streamOpenCount).hasValue(1); + assertThat(callbacks.streamRequestCount).hasValue(1); + assertThat(callbacks.streamResponseCount).hasValue(0); + } + + private static Table> createResponses() { + return ImmutableTable.>builder() + .put(Resources.V3.CLUSTER_TYPE_URL, VERSION, ImmutableList.of(CLUSTER)) + .put(Resources.V3.ENDPOINT_TYPE_URL, VERSION, ImmutableList.of(ENDPOINT)) + .put(Resources.V3.LISTENER_TYPE_URL, VERSION, ImmutableList.of(LISTENER)) + .put(Resources.V3.ROUTE_TYPE_URL, VERSION, ImmutableList.of(ROUTE)) + .put(Resources.V3.SECRET_TYPE_URL, VERSION, ImmutableList.of(SECRET)) + .build(); + } + + private static class MockConfigWatcher implements ConfigWatcher { + + private final boolean closeWatch; + private final Map counts; + private final Table> responses; + private final Map> expectedKnownResources = new ConcurrentHashMap<>(); + + MockConfigWatcher(boolean closeWatch, Table> responses) { + this.closeWatch = closeWatch; + this.counts = new HashMap<>(); + this.responses = HashBasedTable.create(responses); + } + + @Override + public Watch createWatch( + boolean ads, + XdsRequest request, + Set knownResources, + Consumer responseConsumer, + boolean hasClusterChanged) { + + counts.put(request.getTypeUrl(), counts.getOrDefault(request.getTypeUrl(), 0) + 1); + + Watch watch = new Watch(ads, request, responseConsumer); + + if (responses.row(request.getTypeUrl()).size() > 0) { + final Response response; + + synchronized (responses) { + String version = responses.row(request.getTypeUrl()).keySet().iterator().next(); + Collection resources = responses.row(request.getTypeUrl()).remove(version); + response = Response.create(request, resources, version); + } + + expectedKnownResources.put( + request.getTypeUrl(), + response.resources().stream() + .map(Resources::getResourceName) + .collect(Collectors.toSet())); + + try { + watch.respond(response); + } catch (WatchCancelledException e) { + fail("watch should not be cancelled", e); + } + } else if (closeWatch) { + watch.cancel(); + } else { + Set expectedKnown = expectedKnownResources.get(request.getTypeUrl()); + if (expectedKnown != null && !expectedKnown.equals(knownResources)) { + fail("unexpected known resources after sending all responses"); + } + } + + return watch; + } + } + + private static class MockDiscoveryServerCallbacks + implements DiscoveryServerCallbacks { + + private final AtomicInteger streamCloseCount = new AtomicInteger(); + private final AtomicInteger streamCloseWithErrorCount = new AtomicInteger(); + private final AtomicInteger streamOpenCount = new AtomicInteger(); + private final AtomicInteger streamRequestCount = new AtomicInteger(); + private final AtomicInteger streamResponseCount = new AtomicInteger(); + + final Collection assertionErrors = new LinkedList<>(); + + @Override + public void onStreamClose(long streamId, String typeUrl) { + streamCloseCount.getAndIncrement(); + } + + @Override + public void onStreamCloseWithError(long streamId, String typeUrl, Throwable error) { + streamCloseWithErrorCount.getAndIncrement(); + } + + @Override + public void onStreamOpen(long streamId, String typeUrl) { + streamOpenCount.getAndIncrement(); + } + + @Override + public void onV2StreamRequest(long streamId, + io.envoyproxy.envoy.api.v2.DiscoveryRequest request) { + throw new IllegalStateException("Unexpected v2 request in v3 test"); + } + + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) { + streamRequestCount.getAndIncrement(); + + if (request == null) { + this.assertionErrors.add("onStreamRequest#request => expected not null"); + } else if (!request.getNode().equals(NODE)) { + this.assertionErrors.add(format( + "onStreamRequest#request => expected node = %s, got %s", + NODE, + request.getNode())); + } + } + + @Override + public void onStreamResponse(long streamId, io.envoyproxy.envoy.api.v2.DiscoveryRequest request, + io.envoyproxy.envoy.api.v2.DiscoveryResponse response) { + throw new IllegalStateException("Unexpected v2 response in v3 test"); + } + + @Override + public void onV3StreamResponse(long streamId, DiscoveryRequest request, + DiscoveryResponse response) { + streamResponseCount.getAndIncrement(); + + if (request == null) { + this.assertionErrors.add("onStreamResponse#request => expected not null"); + } else if (!request.getNode().equals(NODE)) { + this.assertionErrors.add(format( + "onStreamResponse#request => expected node = %s, got %s", + NODE, + request.getNode())); + } + + if (response == null) { + this.assertionErrors.add("onStreamResponse#response => expected not null"); + } + } + + void assertThatNoErrors() { + if (!assertionErrors.isEmpty()) { + throw new AssertionError(String.join(", ", assertionErrors)); + } + } + } + + private static class MockDiscoveryResponseObserver implements StreamObserver { + + private final Collection assertionErrors = new LinkedList<>(); + private final AtomicBoolean completed = new AtomicBoolean(); + private final CountDownLatch completedLatch = new CountDownLatch(1); + private final AtomicBoolean error = new AtomicBoolean(); + private final CountDownLatch errorLatch = new CountDownLatch(1); + private final AtomicInteger nonce = new AtomicInteger(); + private final Collection responses = new LinkedList<>(); + + private Throwable errorException; + private boolean sendError = false; + + void assertThatNoErrors() { + if (!assertionErrors.isEmpty()) { + throw new AssertionError(String.join(", ", assertionErrors)); + } + } + + @Override + public void onNext(DiscoveryResponse value) { + // Assert that the nonce is monotonically increasing. + String nonce = Integer.toString(this.nonce.getAndIncrement()); + + if (!nonce.equals(value.getNonce())) { + assertionErrors.add(String.format("Nonce => got %s, wanted %s", value.getNonce(), nonce)); + } + + // Assert that the version is set. + if (Strings.isNullOrEmpty(value.getVersionInfo())) { + assertionErrors.add("VersionInfo => got none, wanted non-empty"); + } + + // Assert that resources are non-empty. + if (value.getResourcesList().isEmpty()) { + assertionErrors.add("Resources => got none, wanted non-empty"); + } + + if (Strings.isNullOrEmpty(value.getTypeUrl())) { + assertionErrors.add("TypeUrl => got none, wanted non-empty"); + } + + value.getResourcesList().forEach(r -> { + if (!value.getTypeUrl().equals(r.getTypeUrl())) { + assertionErrors.add(String.format("TypeUrl => got %s, wanted %s", r.getTypeUrl(), value.getTypeUrl())); + } + }); + + responses.add(value); + + if (sendError) { + throw Status.INTERNAL + .withDescription("send error") + .asRuntimeException(); + } + } + + @Override + public void onError(Throwable t) { + error.set(true); + errorException = t; + errorLatch.countDown(); + } + + @Override + public void onCompleted() { + completed.set(true); + completedLatch.countDown(); + } + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerV2ResourcesAdsIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerV2ResourcesAdsIT.java new file mode 100644 index 000000000..6594e0912 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerV2ResourcesAdsIT.java @@ -0,0 +1,105 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.controlplane.server.V2TestSnapshots.createSnapshot; +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.containsString; + +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v2.SimpleCache; +import io.envoyproxy.envoy.api.v2.core.Node; +import io.grpc.netty.NettyServerBuilder; +import io.restassured.http.ContentType; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.testcontainers.containers.Network; + +/** + * Tests a v3 DiscoveryServer with a v2 SimpleCache. This tests the migration path for end users + * where the data planes migrate to v2 ADS transport before the control plane migrates to + * generating v3 resources. + */ +public class V3DiscoveryServerV2ResourcesAdsIT { + + private static final String CONFIG = "envoy/ads.v3.config.yaml"; + private static final String GROUP = "key"; + private static final Integer LISTENER_PORT = 10000; + + private static final CountDownLatch onStreamOpenLatch = new CountDownLatch(1); + private static final CountDownLatch onStreamRequestLatch = new CountDownLatch(1); + private static final CountDownLatch onStreamResponseLatch = new CountDownLatch(1); + + private static final NettyGrpcServerRule ADS = new NettyGrpcServerRule() { + @Override + protected void configureServerBuilder(NettyServerBuilder builder) { + final SimpleCache cache = new SimpleCache<>(new NodeGroup() { + @Override public String hash(Node node) { + throw new IllegalStateException("Unexpected v2 request in v3 test"); + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + return GROUP; + } + }); + + final DiscoveryServerCallbacks callbacks = + new V3OnlyDiscoveryServerCallbacks(onStreamOpenLatch, onStreamRequestLatch, + onStreamResponseLatch); + + cache.setSnapshot( + GROUP, + createSnapshot(true, + "upstream", + UPSTREAM.ipAddress(), + EchoContainer.PORT, + "listener0", + LISTENER_PORT, + "route0", + "1") + ); + + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, cache); + + builder.addService(server.getAggregatedDiscoveryServiceImpl()); + } + }; + + private static final Network NETWORK = Network.newNetwork(); + + private static final EnvoyContainer ENVOY = new EnvoyContainer(CONFIG, () -> ADS.getServer().getPort()) + .withExposedPorts(LISTENER_PORT) + .withNetwork(NETWORK); + + private static final EchoContainer UPSTREAM = new EchoContainer() + .withNetwork(NETWORK) + .withNetworkAliases("upstream"); + + @ClassRule + public static final RuleChain RULES = RuleChain.outerRule(UPSTREAM) + .around(ADS) + .around(ENVOY); + + @Test + public void validateTestRequestToEchoServerViaEnvoy() throws InterruptedException { + assertThat(onStreamOpenLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to open ADS stream"); + + assertThat(onStreamRequestLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to receive ADS request"); + + assertThat(onStreamResponseLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to send ADS response"); + + String baseUri = String.format("http://%s:%d", ENVOY.getContainerIpAddress(), ENVOY.getMappedPort(LISTENER_PORT)); + + await().atMost(5, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> given().baseUri(baseUri).contentType(ContentType.TEXT) + .when().get("/") + .then().statusCode(200) + .and().body(containsString(UPSTREAM.response))); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerV2ResourcesXdsIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerV2ResourcesXdsIT.java new file mode 100644 index 000000000..10071505a --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerV2ResourcesXdsIT.java @@ -0,0 +1,108 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.controlplane.server.V2TestSnapshots.createSnapshotNoEdsV3Transport; +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.containsString; + +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v2.SimpleCache; +import io.envoyproxy.envoy.api.v2.core.Node; +import io.grpc.netty.NettyServerBuilder; +import io.restassured.http.ContentType; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.testcontainers.containers.Network; + +/** + * Tests a V3 discovery server with a V2 cache. This provides a migration path for end users + * where they can make data planes consume the V3 xDS APIs before migrating the control plane + * server to V3. + */ +public class V3DiscoveryServerV2ResourcesXdsIT { + + private static final String CONFIG = "envoy/xds.v3.config.yaml"; + private static final String GROUP = "key"; + private static final Integer LISTENER_PORT = 10000; + + private static final CountDownLatch onStreamOpenLatch = new CountDownLatch(2); + private static final CountDownLatch onStreamRequestLatch = new CountDownLatch(2); + private static final CountDownLatch onStreamResponseLatch = new CountDownLatch(2); + + private static final NettyGrpcServerRule XDS = new NettyGrpcServerRule() { + @Override + protected void configureServerBuilder(NettyServerBuilder builder) { + final SimpleCache cache = new SimpleCache<>(new NodeGroup() { + @Override public String hash(Node node) { + throw new IllegalStateException("Unexpected v2 request in v3 test"); + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + return GROUP; + } + }); + + final DiscoveryServerCallbacks callbacks = + new V3OnlyDiscoveryServerCallbacks(onStreamOpenLatch, onStreamRequestLatch, + onStreamResponseLatch); + + cache.setSnapshot( + GROUP, + createSnapshotNoEdsV3Transport(false, + "upstream", + "upstream", + EchoContainer.PORT, + "listener0", + LISTENER_PORT, + "route0", + "1") + ); + + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, cache); + + builder.addService(server.getClusterDiscoveryServiceImpl()); + builder.addService(server.getEndpointDiscoveryServiceImpl()); + builder.addService(server.getListenerDiscoveryServiceImpl()); + builder.addService(server.getRouteDiscoveryServiceImpl()); + } + }; + + private static final Network NETWORK = Network.newNetwork(); + + private static final EnvoyContainer ENVOY = new EnvoyContainer(CONFIG, () -> XDS.getServer().getPort()) + .withExposedPorts(LISTENER_PORT) + .withNetwork(NETWORK); + + private static final EchoContainer UPSTREAM = new EchoContainer() + .withNetwork(NETWORK) + .withNetworkAliases("upstream"); + + @ClassRule + public static final RuleChain RULES = RuleChain.outerRule(UPSTREAM) + .around(XDS) + .around(ENVOY); + + @Test + public void validateTestRequestToEchoServerViaEnvoy() throws InterruptedException { + assertThat(onStreamOpenLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to open XDS streams"); + + assertThat(onStreamRequestLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to receive XDS requests"); + + assertThat(onStreamResponseLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to send XDS responses"); + + String baseUri = String.format("http://%s:%d", ENVOY.getContainerIpAddress(), ENVOY.getMappedPort(LISTENER_PORT)); + + await().atMost(5, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> given().baseUri(baseUri).contentType(ContentType.TEXT) + .when().get("/") + .then().statusCode(200) + .and().body(containsString(UPSTREAM.response))); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerXdsIT.java b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerXdsIT.java new file mode 100644 index 000000000..43317d960 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V3DiscoveryServerXdsIT.java @@ -0,0 +1,103 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.controlplane.server.V3TestSnapshots.createSnapshotNoEds; +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.containsString; + +import io.envoyproxy.controlplane.cache.NodeGroup; +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.envoy.api.v2.core.Node; +import io.grpc.netty.NettyServerBuilder; +import io.restassured.http.ContentType; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.testcontainers.containers.Network; + +public class V3DiscoveryServerXdsIT { + + private static final String CONFIG = "envoy/xds.v3.config.yaml"; + private static final String GROUP = "key"; + private static final Integer LISTENER_PORT = 10000; + + private static final CountDownLatch onStreamOpenLatch = new CountDownLatch(2); + private static final CountDownLatch onStreamRequestLatch = new CountDownLatch(2); + private static final CountDownLatch onStreamResponseLatch = new CountDownLatch(2); + + private static final NettyGrpcServerRule XDS = new NettyGrpcServerRule() { + @Override + protected void configureServerBuilder(NettyServerBuilder builder) { + final SimpleCache cache = new SimpleCache<>(new NodeGroup() { + @Override public String hash(Node node) { + throw new IllegalStateException("Unexpected v2 request in a v3 test"); + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + return GROUP; + } + }); + + final DiscoveryServerCallbacks callbacks = + new V3OnlyDiscoveryServerCallbacks(onStreamOpenLatch, onStreamRequestLatch, + onStreamResponseLatch); + + cache.setSnapshot( + GROUP, + createSnapshotNoEds(false, + "upstream", + "upstream", + EchoContainer.PORT, + "listener0", + LISTENER_PORT, + "route0", + "1") + ); + + V3DiscoveryServer server = new V3DiscoveryServer(callbacks, cache); + + builder.addService(server.getClusterDiscoveryServiceImpl()); + builder.addService(server.getEndpointDiscoveryServiceImpl()); + builder.addService(server.getListenerDiscoveryServiceImpl()); + builder.addService(server.getRouteDiscoveryServiceImpl()); + } + }; + + private static final Network NETWORK = Network.newNetwork(); + + private static final EnvoyContainer ENVOY = new EnvoyContainer(CONFIG, () -> XDS.getServer().getPort()) + .withExposedPorts(LISTENER_PORT) + .withNetwork(NETWORK); + + private static final EchoContainer UPSTREAM = new EchoContainer() + .withNetwork(NETWORK) + .withNetworkAliases("upstream"); + + @ClassRule + public static final RuleChain RULES = RuleChain.outerRule(UPSTREAM) + .around(XDS) + .around(ENVOY); + + @Test + public void validateTestRequestToEchoServerViaEnvoy() throws InterruptedException { + assertThat(onStreamOpenLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to open XDS streams"); + + assertThat(onStreamRequestLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to receive XDS requests"); + + assertThat(onStreamResponseLatch.await(15, TimeUnit.SECONDS)).isTrue() + .overridingErrorMessage("failed to send XDS responses"); + + String baseUri = String.format("http://%s:%d", ENVOY.getContainerIpAddress(), ENVOY.getMappedPort(LISTENER_PORT)); + + await().atMost(5, TimeUnit.SECONDS).ignoreExceptions().untilAsserted( + () -> given().baseUri(baseUri).contentType(ContentType.TEXT) + .when().get("/") + .then().statusCode(200) + .and().body(containsString(UPSTREAM.response))); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V3OnlyDiscoveryServerCallbacks.java b/server/src/test/java/io/envoyproxy/controlplane/server/V3OnlyDiscoveryServerCallbacks.java new file mode 100644 index 000000000..e1297a5b7 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V3OnlyDiscoveryServerCallbacks.java @@ -0,0 +1,55 @@ +package io.envoyproxy.controlplane.server; + +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import java.util.concurrent.CountDownLatch; + +public class V3OnlyDiscoveryServerCallbacks implements DiscoveryServerCallbacks { + private final CountDownLatch onStreamOpenLatch; + private final CountDownLatch onStreamRequestLatch; + private final CountDownLatch onStreamResponseLatch; + + /** + * Returns an implementation of DiscoveryServerCallbacks that throws if it sees a v2 request, + * and counts down on provided latches in response to certain events. + * + * @param onStreamOpenLatch latch to call countDown() on when a v3 stream is opened. + * @param onStreamRequestLatch latch to call countDown() on when a v3 request is seen. + * @param onStreamResponseLatch latch to call countDown() on when a v3 response is seen. + */ + public V3OnlyDiscoveryServerCallbacks(CountDownLatch onStreamOpenLatch, + CountDownLatch onStreamRequestLatch, CountDownLatch onStreamResponseLatch) { + this.onStreamOpenLatch = onStreamOpenLatch; + this.onStreamRequestLatch = onStreamRequestLatch; + this.onStreamResponseLatch = onStreamResponseLatch; + } + + @Override + public void onStreamOpen(long streamId, String typeUrl) { + onStreamOpenLatch.countDown(); + } + + @Override + public void onV2StreamRequest(long streamId, + io.envoyproxy.envoy.api.v2.DiscoveryRequest request) { + throw new IllegalStateException("Unexpected v2 request in v3 test"); + } + + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) { + onStreamRequestLatch.countDown(); + } + + @Override + public void onStreamResponse(long streamId, + io.envoyproxy.envoy.api.v2.DiscoveryRequest request, + io.envoyproxy.envoy.api.v2.DiscoveryResponse response) { + throw new IllegalStateException("Unexpected v2 response in v3 test"); + } + + @Override + public void onV3StreamResponse(long streamId, DiscoveryRequest request, + DiscoveryResponse response) { + onStreamResponseLatch.countDown(); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/V3TestSnapshots.java b/server/src/test/java/io/envoyproxy/controlplane/server/V3TestSnapshots.java new file mode 100644 index 000000000..c6c27437c --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/V3TestSnapshots.java @@ -0,0 +1,96 @@ +package io.envoyproxy.controlplane.server; + +import static io.envoyproxy.envoy.config.core.v3.ApiVersion.V2; +import static io.envoyproxy.envoy.config.core.v3.ApiVersion.V3; + +import io.envoyproxy.controlplane.cache.TestResources; +import io.envoyproxy.controlplane.cache.v3.Snapshot; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.core.v3.ApiVersion; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; + +class V3TestSnapshots { + + static Snapshot createSnapshot( + boolean ads, + String clusterName, + String endpointAddress, + int endpointPort, + String listenerName, + int listenerPort, + String routeName, + String version) { + + Cluster cluster = TestResources.createClusterV3(clusterName); + ClusterLoadAssignment + endpoint = TestResources.createEndpointV3(clusterName, endpointAddress, endpointPort); + Listener listener = TestResources.createListenerV3(ads, V3, V3, listenerName, + listenerPort, routeName); + RouteConfiguration route = TestResources.createRouteV3(routeName, clusterName); + + return Snapshot.create( + ImmutableList.of(cluster), + ImmutableList.of(endpoint), + ImmutableList.of(listener), + ImmutableList.of(route), + ImmutableList.of(), + version); + } + + static Snapshot createSnapshotNoEdsV2Transport( + boolean ads, + String clusterName, + String endpointAddress, + int endpointPort, + String listenerName, + int listenerPort, + String routeName, + String version) { + return createSnapshotNoEds(ads, V2, V2, clusterName, endpointAddress, + endpointPort, listenerName, listenerPort, routeName, version); + } + + static Snapshot createSnapshotNoEds( + boolean ads, + String clusterName, + String endpointAddress, + int endpointPort, + String listenerName, + int listenerPort, + String routeName, + String version) { + return createSnapshotNoEds(ads, V3, V3, clusterName, endpointAddress, + endpointPort, listenerName, listenerPort, routeName, version); + } + + private static Snapshot createSnapshotNoEds( + boolean ads, + ApiVersion rdsTransportVersion, + ApiVersion rdsResourceVersion, + String clusterName, + String endpointAddress, + int endpointPort, + String listenerName, + int listenerPort, + String routeName, + String version) { + + Cluster cluster = TestResources.createClusterV3(clusterName, endpointAddress, endpointPort); + Listener listener = TestResources.createListenerV3(ads, rdsTransportVersion, rdsResourceVersion, + listenerName, listenerPort, routeName); + RouteConfiguration route = TestResources.createRouteV3(routeName, clusterName); + + return Snapshot.create( + ImmutableList.of(cluster), + ImmutableList.of(), + ImmutableList.of(listener), + ImmutableList.of(route), + ImmutableList.of(), + version); + } + + private V3TestSnapshots() { } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/callback/SnapshotCollectingCallbackTest.java b/server/src/test/java/io/envoyproxy/controlplane/server/callback/SnapshotCollectingCallbackTest.java index 8810daee4..d50838c1d 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/callback/SnapshotCollectingCallbackTest.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/callback/SnapshotCollectingCallbackTest.java @@ -4,9 +4,10 @@ import com.google.common.collect.ImmutableSet; import io.envoyproxy.controlplane.cache.NodeGroup; -import io.envoyproxy.controlplane.cache.SimpleCache; -import io.envoyproxy.controlplane.cache.Snapshot; +import io.envoyproxy.controlplane.cache.v2.SimpleCache; +import io.envoyproxy.controlplane.cache.v2.Snapshot; import io.envoyproxy.envoy.api.v2.DiscoveryRequest; +import io.envoyproxy.envoy.api.v2.core.Node; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -21,9 +22,17 @@ public class SnapshotCollectingCallbackTest { private static final Clock CLOCK = Clock.fixed(Instant.now(), ZoneId.systemDefault()); - private static final NodeGroup NODE_GROUP = node -> "group"; + private static final NodeGroup NODE_GROUP = new NodeGroup() { + @Override public String hash(Node node) { + return "group"; + } + + @Override public String hash(io.envoyproxy.envoy.config.core.v3.Node node) { + return "group"; + } + }; private final ArrayList collectedGroups = new ArrayList<>(); - private SnapshotCollectingCallback callback; + private SnapshotCollectingCallback callback; private SimpleCache cache; @Before @@ -31,14 +40,14 @@ public void setUp() { collectedGroups.clear(); cache = new SimpleCache<>(NODE_GROUP); cache.setSnapshot("group", Snapshot.createEmpty("")); - callback = new SnapshotCollectingCallback<>(cache, NODE_GROUP, CLOCK, + callback = new SnapshotCollectingCallback(cache, NODE_GROUP, CLOCK, Collections.singleton(collectedGroups::add), 3, 100); } @Test public void testSingleSnapshot() { - callback.onStreamRequest(0, DiscoveryRequest.getDefaultInstance()); - callback.onStreamRequest(1, DiscoveryRequest.getDefaultInstance()); + callback.onV2StreamRequest(0, DiscoveryRequest.getDefaultInstance()); + callback.onV2StreamRequest(1, DiscoveryRequest.getDefaultInstance()); // We have 2 references to the snapshot, this should do nothing. callback.deleteUnreferenced(Clock.offset(CLOCK, Duration.ofMillis(5))); @@ -67,7 +76,7 @@ public void testAsyncCollection() throws InterruptedException { CountDownLatch deleteUnreferencedLatch = new CountDownLatch(1); // Create a cache with 0 expiry delay, which means the snapshot should get collected immediately. - callback = new SnapshotCollectingCallback(cache, NODE_GROUP, CLOCK, + callback = new SnapshotCollectingCallback(cache, NODE_GROUP, CLOCK, ImmutableSet.of(collectedGroups::add, group -> snapshotCollectedLatch.countDown()), -3, 1) { @Override synchronized void deleteUnreferenced(Clock clock) { super.deleteUnreferenced(clock); @@ -75,7 +84,7 @@ public void testAsyncCollection() throws InterruptedException { } }; - callback.onStreamRequest(0, DiscoveryRequest.getDefaultInstance()); + callback.onV2StreamRequest(0, DiscoveryRequest.getDefaultInstance()); assertThat(deleteUnreferencedLatch.await(100, TimeUnit.MILLISECONDS)).isTrue(); assertThat(collectedGroups).isEmpty(); diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/serializer/CachedProtoResourcesSerializerTest.java b/server/src/test/java/io/envoyproxy/controlplane/server/serializer/CachedProtoResourcesSerializerTest.java index dfd4370b7..4d30c6aaa 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/serializer/CachedProtoResourcesSerializerTest.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/serializer/CachedProtoResourcesSerializerTest.java @@ -1,5 +1,7 @@ package io.envoyproxy.controlplane.server.serializer; +import static io.envoyproxy.controlplane.cache.Resources.ApiVersion.V2; +import static io.envoyproxy.controlplane.cache.Resources.ApiVersion.V3; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.Lists; @@ -14,19 +16,23 @@ public class CachedProtoResourcesSerializerTest { CachedProtoResourcesSerializer serializer = new CachedProtoResourcesSerializer(); @Test - public void shouldKeepCachedProtoWhenSerializingSameMessage() { + public void shouldKeepCachedProtoWhenSerializingSameV2Message() { ClusterLoadAssignment endpoint = ClusterLoadAssignment.newBuilder() .setClusterName("service1") .build(); - Any serializedEndpoint = serializer.serialize(endpoint); - Any serializedSameEndpoint = serializer.serialize(endpoint); + Any serializedEndpoint = serializer.serialize(endpoint, V2); + Any serializedSameEndpoint = serializer.serialize(endpoint, V2); + Any v3SerializedEndpoint = serializer.serialize(endpoint, V3); + Any v3SerializedSameEndpoint = serializer.serialize(endpoint, V3); assertThat(serializedEndpoint).isSameAs(serializedSameEndpoint); + assertThat(serializedEndpoint).isNotSameAs(v3SerializedEndpoint); + assertThat(v3SerializedEndpoint).isSameAs(v3SerializedSameEndpoint); } @Test - public void shouldKeepCachedProtoWhenSerializingSameMessages() { + public void shouldKeepCachedProtoWhenSerializingSameV2Messages() { List endpoints = Lists.newArrayList( ClusterLoadAssignment.newBuilder() .setClusterName("service1") @@ -36,13 +42,70 @@ public void shouldKeepCachedProtoWhenSerializingSameMessages() { .build() ); - Collection serializedEndpoints = serializer.serialize(endpoints); - Collection serializedSameEndpoints = serializer.serialize(endpoints); + Collection serializedEndpoints = serializer.serialize(endpoints, V2); + Collection serializedSameEndpoints = serializer.serialize(endpoints, V2); + Collection v3SerializedEndpoints = serializer.serialize(endpoints, V3); + Collection v3SerializedSameEndpoints = serializer.serialize(endpoints, V3); assertThat(serializedEndpoints).isEqualTo(serializedSameEndpoints); assertThat(serializedEndpoints).isNotSameAs(serializedSameEndpoints); + assertThat(serializedEndpoints).isNotSameAs(v3SerializedSameEndpoints); assertThat(serializedEndpoints) // elements are the same instances .usingElementComparator((x, y) -> x == y ? 0 : 1) .hasSameElementsAs(serializedSameEndpoints); + + assertThat(v3SerializedEndpoints).isEqualTo(v3SerializedSameEndpoints); + assertThat(v3SerializedEndpoints).isNotSameAs(v3SerializedSameEndpoints); + assertThat(v3SerializedEndpoints) // elements are the same instances + .usingElementComparator((x, y) -> x == y ? 0 : 1) + .hasSameElementsAs(v3SerializedSameEndpoints); + } + + @Test + public void shouldKeepCachedProtoWhenSerializingSameV3Message() { + io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment endpoint = + io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment + .newBuilder() + .setClusterName("service1") + .build(); + + Any serializedEndpoint = serializer.serialize(endpoint, V2); + Any serializedSameEndpoint = serializer.serialize(endpoint, V2); + Any v3SerializedEndpoint = serializer.serialize(endpoint, V3); + Any v3SerializedSameEndpoint = serializer.serialize(endpoint, V3); + + assertThat(serializedEndpoint).isSameAs(serializedSameEndpoint); + assertThat(serializedEndpoint).isNotSameAs(v3SerializedEndpoint); + assertThat(v3SerializedEndpoint).isSameAs(v3SerializedSameEndpoint); + } + + @Test + public void shouldKeepCachedProtoWhenSerializingSameV3Messages() { + List endpoints = Lists.newArrayList( + io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.newBuilder() + .setClusterName("service1") + .build(), + io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.newBuilder() + .setClusterName("service2") + .build() + ); + + Collection serializedEndpoints = serializer.serialize(endpoints, V2); + Collection serializedSameEndpoints = serializer.serialize(endpoints, V2); + Collection v3SerializedEndpoints = serializer.serialize(endpoints, V3); + Collection v3SerializedSameEndpoints = serializer.serialize(endpoints, V3); + + assertThat(serializedEndpoints).isEqualTo(serializedSameEndpoints); + assertThat(serializedEndpoints).isNotSameAs(serializedSameEndpoints); + assertThat(serializedEndpoints).isNotSameAs(v3SerializedSameEndpoints); + assertThat(serializedEndpoints) // elements are the same instances + .usingElementComparator((x, y) -> x == y ? 0 : 1) + .hasSameElementsAs(serializedSameEndpoints); + + assertThat(v3SerializedEndpoints).isEqualTo(v3SerializedSameEndpoints); + assertThat(v3SerializedEndpoints).isNotSameAs(v3SerializedSameEndpoints); + assertThat(v3SerializedEndpoints) // elements are the same instances + .usingElementComparator((x, y) -> x == y ? 0 : 1) + .hasSameElementsAs(v3SerializedSameEndpoints); } } diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/serializer/DefaultProtoResourcesSerializerTest.java b/server/src/test/java/io/envoyproxy/controlplane/server/serializer/DefaultProtoResourcesSerializerTest.java index b747eabbf..0456cfa70 100644 --- a/server/src/test/java/io/envoyproxy/controlplane/server/serializer/DefaultProtoResourcesSerializerTest.java +++ b/server/src/test/java/io/envoyproxy/controlplane/server/serializer/DefaultProtoResourcesSerializerTest.java @@ -1,5 +1,7 @@ package io.envoyproxy.controlplane.server.serializer; +import static io.envoyproxy.controlplane.cache.Resources.ApiVersion.V2; +import static io.envoyproxy.controlplane.cache.Resources.ApiVersion.V3; import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.Lists; @@ -14,20 +16,24 @@ public class DefaultProtoResourcesSerializerTest { DefaultProtoResourcesSerializer serializer = new DefaultProtoResourcesSerializer(); @Test - public void shouldReturnDifferentInstanceOfSerializedProtoWhenResourcesAreTheSame() { + public void shouldReturnDifferentInstanceOfSerializedProtoWhenV2ResourcesAreTheSame() { ClusterLoadAssignment endpoint = ClusterLoadAssignment.newBuilder() .setClusterName("service1") .build(); - Any serializedEndpoint = serializer.serialize(endpoint); - Any serializedSameEndpoint = serializer.serialize(endpoint); + Any serializedEndpoint = serializer.serialize(endpoint, V2); + Any serializedSameEndpoint = serializer.serialize(endpoint, V2); + Any v3SerializedEndpoint = serializer.serialize(endpoint, V3); + Any v3SerializedSameEndpoint = serializer.serialize(endpoint, V3); assertThat(serializedEndpoint).isEqualTo(serializedSameEndpoint); assertThat(serializedEndpoint).isNotSameAs(serializedSameEndpoint); + assertThat(v3SerializedEndpoint).isEqualTo(v3SerializedSameEndpoint); + assertThat(v3SerializedEndpoint).isNotSameAs(v3SerializedSameEndpoint); } @Test - public void shouldReturnDifferentInstancesOfSerializedProtoWhenResourcesAreTheSame() { + public void shouldReturnDifferentInstancesOfSerializedProtoWhenV2ResourcesAreTheSame() { List endpoints = Lists.newArrayList( ClusterLoadAssignment.newBuilder() .setClusterName("service1") @@ -37,13 +43,69 @@ public void shouldReturnDifferentInstancesOfSerializedProtoWhenResourcesAreTheSa .build() ); - Collection serializedEndpoints = serializer.serialize(endpoints); - Collection serializedSameEndpoints = serializer.serialize(endpoints); + Collection serializedEndpoints = serializer.serialize(endpoints, V2); + Collection serializedSameEndpoints = serializer.serialize(endpoints, V2); + Collection v3SerializedEndpoints = serializer.serialize(endpoints, V3); + Collection v3SerializedSameEndpoints = serializer.serialize(endpoints, V3); assertThat(serializedEndpoints).isEqualTo(serializedSameEndpoints); assertThat(serializedEndpoints).isNotSameAs(serializedSameEndpoints); assertThat(serializedEndpoints) // elements are not the same instances .usingElementComparator((x, y) -> x == y ? 0 : 1) .doesNotContainAnyElementsOf(serializedSameEndpoints); + + assertThat(v3SerializedEndpoints).isEqualTo(v3SerializedSameEndpoints); + assertThat(v3SerializedEndpoints).isNotSameAs(v3SerializedSameEndpoints); + assertThat(v3SerializedEndpoints) // elements are not the same instances + .usingElementComparator((x, y) -> x == y ? 0 : 1) + .doesNotContainAnyElementsOf(v3SerializedSameEndpoints); + } + + @Test + public void shouldReturnDifferentInstanceOfSerializedProtoWhenV3ResourcesAreTheSame() { + io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment endpoint = + io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment + .newBuilder() + .setClusterName("service1") + .build(); + + Any serializedEndpoint = serializer.serialize(endpoint, V2); + Any serializedSameEndpoint = serializer.serialize(endpoint, V2); + Any v3SerializedEndpoint = serializer.serialize(endpoint, V3); + Any v3SerializedSameEndpoint = serializer.serialize(endpoint, V3); + + assertThat(serializedEndpoint).isEqualTo(serializedSameEndpoint); + assertThat(serializedEndpoint).isNotSameAs(serializedSameEndpoint); + assertThat(v3SerializedEndpoint).isEqualTo(v3SerializedSameEndpoint); + assertThat(v3SerializedEndpoint).isNotSameAs(v3SerializedSameEndpoint); + } + + @Test + public void shouldReturnDifferentInstancesOfSerializedProtoWhenV3ResourcesAreTheSame() { + List endpoints = Lists.newArrayList( + io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.newBuilder() + .setClusterName("service1") + .build(), + io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment.newBuilder() + .setClusterName("service2") + .build() + ); + + Collection serializedEndpoints = serializer.serialize(endpoints, V2); + Collection serializedSameEndpoints = serializer.serialize(endpoints, V2); + Collection v3SerializedEndpoints = serializer.serialize(endpoints, V3); + Collection v3SerializedSameEndpoints = serializer.serialize(endpoints, V3); + + assertThat(serializedEndpoints).isEqualTo(serializedSameEndpoints); + assertThat(serializedEndpoints).isNotSameAs(serializedSameEndpoints); + assertThat(serializedEndpoints) // elements are not the same instances + .usingElementComparator((x, y) -> x == y ? 0 : 1) + .doesNotContainAnyElementsOf(serializedSameEndpoints); + + assertThat(v3SerializedEndpoints).isEqualTo(v3SerializedSameEndpoints); + assertThat(v3SerializedEndpoints).isNotSameAs(v3SerializedSameEndpoints); + assertThat(v3SerializedEndpoints) // elements are not the same instances + .usingElementComparator((x, y) -> x == y ? 0 : 1) + .doesNotContainAnyElementsOf(v3SerializedSameEndpoints); } } diff --git a/server/src/test/resources/envoy/ads.config.yaml b/server/src/test/resources/envoy/ads.v2.config.yaml similarity index 63% rename from server/src/test/resources/envoy/ads.config.yaml rename to server/src/test/resources/envoy/ads.v2.config.yaml index 1da12b438..1acd636f4 100644 --- a/server/src/test/resources/envoy/ads.config.yaml +++ b/server/src/test/resources/envoy/ads.v2.config.yaml @@ -18,9 +18,14 @@ node: static_resources: clusters: - connect_timeout: 1s - hosts: - - socket_address: - address: HOST_IP - port_value: HOST_PORT + load_assignment: + cluster_name: ads_server + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: HOST_IP + port_value: HOST_PORT http2_protocol_options: {} name: ads_cluster diff --git a/server/src/test/resources/envoy/ads.v3.config.yaml b/server/src/test/resources/envoy/ads.v3.config.yaml new file mode 100644 index 000000000..991db7c8e --- /dev/null +++ b/server/src/test/resources/envoy/ads.v3.config.yaml @@ -0,0 +1,34 @@ +admin: + access_log_path: /dev/null + address: + socket_address: { address: 0.0.0.0, port_value: 9901 } +dynamic_resources: + ads_config: + api_type: GRPC + grpc_services: + envoy_grpc: + cluster_name: ads_cluster + transport_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + lds_config: + ads: {} + resource_api_version: V3 +node: + cluster: test-cluster + id: test-id +static_resources: + clusters: + - connect_timeout: 1s + load_assignment: + cluster_name: ads_server + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: HOST_IP + port_value: HOST_PORT + http2_protocol_options: {} + name: ads_cluster diff --git a/server/src/test/resources/envoy/xds.config.yaml b/server/src/test/resources/envoy/xds.v2.config.yaml similarity index 100% rename from server/src/test/resources/envoy/xds.config.yaml rename to server/src/test/resources/envoy/xds.v2.config.yaml diff --git a/server/src/test/resources/envoy/xds.v3.config.yaml b/server/src/test/resources/envoy/xds.v3.config.yaml new file mode 100644 index 000000000..c44db99c8 --- /dev/null +++ b/server/src/test/resources/envoy/xds.v3.config.yaml @@ -0,0 +1,33 @@ +admin: + access_log_path: /dev/null + address: + socket_address: { address: 0.0.0.0, port_value: 9901 } +dynamic_resources: + cds_config: + api_config_source: + api_type: GRPC + grpc_services: + envoy_grpc: + cluster_name: xds_cluster + transport_api_version: V3 + resource_api_version: V3 + lds_config: + api_config_source: + api_type: GRPC + grpc_services: + envoy_grpc: + cluster_name: xds_cluster + transport_api_version: V3 + resource_api_version: V3 +node: + cluster: test-cluster + id: test-id +static_resources: + clusters: + - connect_timeout: 1s + hosts: + - socket_address: + address: HOST_IP + port_value: HOST_PORT + http2_protocol_options: {} + name: xds_cluster