From 73bf3a99c4238b3e0bc7a987d840846b81dc14b3 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Sun, 30 Jan 2022 18:34:36 -0800 Subject: [PATCH 01/25] Start GEP --- site-src/geps/gep-XXXX.md | 94 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 site-src/geps/gep-XXXX.md diff --git a/site-src/geps/gep-XXXX.md b/site-src/geps/gep-XXXX.md new file mode 100644 index 0000000000..b2d624cc50 --- /dev/null +++ b/site-src/geps/gep-XXXX.md @@ -0,0 +1,94 @@ +# GEP-XXXX: GRPCRoute + +* Issue: [TODO](https://github.com/kubernetes-sigs/gateway-api/issues/696) +* Status: Implementable + +(See definitions in [Kubernetes KEP][kep-status]. + +[kep-status]: https://github.com/kubernetes/enhancements/blob/master/keps/NNNN-kep-template/kep.yaml#L9 + +## Goal + +Add an idiomatic GRPCRoute for routing gRPC traffic. + +## Non-Goals + +While certain gRPC implementations support multiple transports and multiple +interface definition languages (IDLs), this proposal limits itself to +[HTTP/2](https://developers.google.com/web/fundamentals/performance/http2) as +the transport and [Protocol Buffers](https://developers.google.com/protocol-buffers) +as the IDL, which makes up the vast majority of gRPC traffic in the wild. + +## Introduction + +At the time of writing, the only official Route resource within the Gateway APIs +is HTTPRoute. It _is_ possible to support other protocols via CRDs and +controllers taking advantage of this have started to pop up. However, in the +long run, this leads to a fragmented ecosystem. + +gRPC is a [popular RPC framework adopted widely across the industry](https://grpc.io/about/#whos-using-grpc-and-why). +The protocol is used pervasively within the Kubernetes project itself as the basis for +many interfaces, including: + +- [the CSI](https://github.com/container-storage-interface/spec/blob/5b0d4540158a260cb3347ef1c87ede8600afb9bf/spec.md), +- [the CRI](https://github.com/kubernetes/cri-api/blob/49fe8b135f4556ea603b1b49470f8365b62f808e/README.md), +- [the device plugin framework](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/) + +Given gRPC's importance in the application-layer networking space and to +the Kubernetes project in particular, we must ensure that the gRPC control plane +configuration landscape does not Balkanize. + +## API + +The API deviates from `HTTPRoute` where it results in a better UX for gRPC +users, while mirroring it in all other cases. + +### Example `GRPCRoute` + +```yaml +kind: GRPCRoute +apiVersion: gateway.networking.k8s.io/v1alpha2 +metadata: + name: foo-grpcroute +spec: + parentRefs: + - name: my-gateway + hostnames: + - foo.com + - bar.com + rules: + - matches: + method: + grpcService: helloworld.Greeter + grpcMethod: SayHello + headers: + - type: Exact + name: magic + value: foo + + filters: + - type: GRPCRequestHeaderModifierFilter + add: + - name: my-header + value: foo + + - type: GRPCRequestMirrorPolicyFilter + destination: + backendRef: + name: mirror-svc + + - type: GRPCTimeoutPolicyFilter + timeout: "30s" + + - type: GRPCRouteRetryPolicyFilter + numRetries: 3 + retryConditions: + - "refused-stream" + - "cancelled" + + backendRefs: + - name: foo-v1 + weight: 90 + - name: foo-v2 + weight: 10 +``` From b76b1a59386e3f30fec9f616972055ac8cc7c2c5 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 31 Jan 2022 11:42:54 -0800 Subject: [PATCH 02/25] Add structs --- site-src/geps/gep-XXXX.md | 172 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 163 insertions(+), 9 deletions(-) diff --git a/site-src/geps/gep-XXXX.md b/site-src/geps/gep-XXXX.md index b2d624cc50..ee9388f31e 100644 --- a/site-src/geps/gep-XXXX.md +++ b/site-src/geps/gep-XXXX.md @@ -77,18 +77,172 @@ spec: backendRef: name: mirror-svc - - type: GRPCTimeoutPolicyFilter - timeout: "30s" - - - type: GRPCRouteRetryPolicyFilter - numRetries: 3 - retryConditions: - - "refused-stream" - - "cancelled" - backendRefs: - name: foo-v1 weight: 90 - name: foo-v2 weight: 10 ``` + +### Structs + +```go +type GRPCRouteSpec struct { + CommonRouteSpec `json:",inline"` + + // Support: Core + // + // +optional + // +kubebuilder:validation:MaxItems=16 + Hostnames []Hostname `json:"hostnames,omitempty"` + + // +optional + // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:default={{matches: {{method: {type: "RegularExpression", service: ".*", method: ".*"}}}}} + Rules []GRPCRouteRule `json:"rules,omitempty"` +} + +type GRPCRouteRule struct { + // +optional + // +kubebuilder:validation:MaxItems=8 + // +kubebuilder:default={{method: {type: "RegularExpression", service: ".*", method: ".*"}}} + Matches []GRPCRouteMatch `json:"matches,omitempty"` + + // Support: Core + // + // +optional + // +kubebuilder:validation:MaxItems=16 + Filters []GRPCRouteFilter `json:"filters,omitempty"` + + // Support: Core for Kubernetes Service + // Support: Custom for any other resource + // + // Support for weight: Core + // + // +optional + // +kubebuilder:validation:MaxItems=16 + BackendRefs []GRPCBackendRef `json:"backendRefs,omitempty"` +} + +type GRPCRouteMatch struct { + // +optional + // +kubebuilder:default={type: "RegularExpression", service: ".*", method: ".*"} + Method *GRPCMethodMatch `json:"path,omitempty"` + + // +listType=map + // +listMapKey=name + // +optional + // +kubebuilder:validation:MaxItems=16 + Headers []GRPCHeaderMatch `json:"headers,omitempty"` +} + +type GRPCMethodMatch struct { + // Support: Core (Exact) + // + // Support: Custom (RegularExpression) + // + // +optional + // +kubebuilder:default=Exact + Type *GRPCMethodMatchType `json:"type,omitempty"` + + // +optional + // +kubebuilder:default="" + // +kubebuilder:validation:MaxLength=1024 + Service *string `json:"value,omitempty"` + + // +optional + // +kubebuilder:default="" + // +kubebuilder:validation:MaxLength=1024 + Method *string `json:"value,omitempty"` + + // +optional + // +kubebuilder:default=true + CaseSensitive *bool `json:"value,omitempty"` +} + +// +kubebuilder:validation:Enum=Exact;RegularExpression +type GRPCMethodMatchType string + +type GRPCHeaderMatch struct { + // +optional + // +kubebuilder:default=Exact + Type *HeaderMatchType `json:"type,omitempty"` + + Name GRPCHeaderName `json:"name"` + + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=4096 + Value string `json:"value"` +} + +// +kubebuilder:validation:Enum=Exact;RegularExpression +type HeaderMatchType string + +// +kubebuilder:validation:MinLength=1 +// +kubebuilder:validation:MaxLength=256 +// +kubebuilder:validation:Pattern=`^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$` +type GRPCHeaderName string + +type GRPCBackendRef struct { + // +optional + BackendRef `json:",inline"` + + // +optional + // +kubebuilder:validation:MaxItems=16 + Filters []GRPCRouteFilter `json:"filters,omitempty"` +} + +type GRPCRouteFilter struct { + // +unionDiscriminator + // +kubebuilder:validation:Enum=RequestHeaderModifier;RequestMirror;ExtensionRef + // + Type GRPCRouteFilterType `json:"type"` + + // Support: Core + // + // +optional + RequestHeaderModifier *GRPCRequestHeaderFilter `json:"requestHeaderModifier,omitempty"` + + // Support: Extended + // + // +optional + RequestMirror *GRPCRequestMirrorFilter `json:"requestMirror,omitempty"` + + // Support: Implementation-specific + // + // +optional + ExtensionRef *LocalObjectReference `json:"extensionRef,omitempty"` +} + +type GRPCRequestHeaderFilter struct { + // +optional + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MaxItems=16 + Set []GRPCHeader `json:"set,omitempty"` + + // +optional + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MaxItems=16 + Add []GRPCHeader `json:"add,omitempty"` + + // +optional + // +kubebuilder:validation:MaxItems=16 + Remove []string `json:"remove,omitempty"` +} + +type GRPCHeader struct { + Name GRPCHeaderName `json:"name"` + + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=4096 + Value string `json:"value"` +} + +type GRPCRequestMirrorFilter struct { + // Support: Extended for Kubernetes Service + // Support: Custom for any other resource + BackendRef BackendObjectReference `json:"backendRef"` +} +``` From 91f0cc0fe1ea429fa3296f073f507f8fd5b90a76 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 31 Jan 2022 12:52:25 -0800 Subject: [PATCH 03/25] Remove unnecessary prefixes --- site-src/geps/gep-XXXX.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/site-src/geps/gep-XXXX.md b/site-src/geps/gep-XXXX.md index ee9388f31e..78068e6a0b 100644 --- a/site-src/geps/gep-XXXX.md +++ b/site-src/geps/gep-XXXX.md @@ -59,20 +59,20 @@ spec: rules: - matches: method: - grpcService: helloworld.Greeter - grpcMethod: SayHello + service: helloworld.Greeter + method: SayHello headers: - type: Exact name: magic value: foo filters: - - type: GRPCRequestHeaderModifierFilter + - type: RequestHeaderModifierFilter add: - name: my-header value: foo - - type: GRPCRequestMirrorPolicyFilter + - type: RequestMirrorPolicyFilter destination: backendRef: name: mirror-svc From b6a7e835225a7e8b5c79a3cc31973c054b64a289 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 10 Feb 2022 10:58:38 -0800 Subject: [PATCH 04/25] Add discussion about encapsulated protocols --- site-src/geps/gep-XXXX.md | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/site-src/geps/gep-XXXX.md b/site-src/geps/gep-XXXX.md index 78068e6a0b..9b8fc139ca 100644 --- a/site-src/geps/gep-XXXX.md +++ b/site-src/geps/gep-XXXX.md @@ -38,6 +38,81 @@ Given gRPC's importance in the application-layer networking space and to the Kubernetes project in particular, we must ensure that the gRPC control plane configuration landscape does not Balkanize. +### Encapsulated Network Protocols + +At the time of writing, the only kind of route officially defined by the Gateway +APIs is `HttpRoute`. This GEP is novel not only in that it introduces a second +protocol to route, but also in that it introduces the first protocol +encapsulated in a protocol already supported by the API. + +That is, it _is_ theoretically possible to route gRPC traffic using only `HTTPRoute` +resources, but there are several serious problems with forcing gRPC users to route traffic at +the level of HTTP. This is why we propose a new resource. + +In setting this precendent, we must also introduce a coherent policy for _when_ +to introduce a custom `Route` resource for an encapsulated protocol for which a +lower layer protocol already exists. We propose the following criteria for such +an addition. + +- Users of the encapsulated protocol would miss out on significant conventional features from their ecosystem if forced to route at a lower layer. +- Users of the enapsulated protocol would experience a degraded user experience if forced to route at a lower layer. +- The encapsulated protocol has a significant user base, particularly in the Kubernetes community. + +gRPC meets _all_ of these criteria and is therefore, we contend, a strong +candidate for inclusion in the Gateway APIs. + +#### HTTP/2 Cleartext + +gRPC allows HTTP/2 cleartext communication (H2C). This is conventionally deployed for +testing. Many control plane implementations do not support this by default and +would require special configuration to work properly. + +#### Content-Based Routing + +While not included in the scope of this initial GEP, a common use case cited for +routing gRPC is payload-aware routing. That is, routing rules which determine a +backend based on the contents of the protocol buffer payload. + +#### User Experience + +The user experience would also degrade significantly if forced to route at the level of HTTP. + +- Encoding services and methods as URIs (an implementation detail of gRPC) +- The [Transfer Encoding header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Transfer-Encoding) for trailers +- Many features supported by HTTP/2 but not by gRPC, such as + - Query parameters + - Methods besides `POST` + - CORS + +### Cross Serving + +A new route type (and especially one encapsulated in an already supported +protocol) raises questions about precedence of routing rules. Today, there is +already a complicated algorithm dictating which routing rule to apply to a +particular HTTP request when two `HTTPRoutes` both apply to the request's +hostname. What semantics should apply when a `GRPCRoute` and an `HTTPRoute` both apply to +a request? + +We propose that, in this case, all rules defined in the `HTTPRoute` should be +applied. Only then will the `GRPCRoute` rules be evaluated. This supports the +common use case of routing gRPC traffic at a particular URI prefix to gRPC +backends while routing all other HTTP traffic to a default HTTP backend. + +More generally, we propose that when a new protocol encapsulated in an already +supported one is added, if traffic applies to both a Route resource of the lower +layer and a Route resource of the higher layer, the rules of the lower layer +will be applied before the rules of the higher layer. + +#### Proxyless Service Mesh + +The gRPC library supports proxyless service mesh, a system by which routing +configuration is received not by an in-line proxy or sidecar proxy but by the client +itself. Eventually, `GRPCRoute` in the Gateway APIs should support this feature. +However, to date, there are no HTTP client libraries capable of participating +in a proxyless service mesh. + +--- + ## API The API deviates from `HTTPRoute` where it results in a better UX for gRPC From ca38d6132e767db428355a7d5ba925fb790a9309 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 10 Feb 2022 11:01:40 -0800 Subject: [PATCH 05/25] Add future enhancements section --- site-src/geps/gep-XXXX.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/site-src/geps/gep-XXXX.md b/site-src/geps/gep-XXXX.md index 9b8fc139ca..588cb20e97 100644 --- a/site-src/geps/gep-XXXX.md +++ b/site-src/geps/gep-XXXX.md @@ -321,3 +321,15 @@ type GRPCRequestMirrorFilter struct { BackendRef BackendObjectReference `json:"backendRef"` } ``` + +## Future Enhancements + +Many more ideas have been discussed for the `GRPCRoute` resource, but in the +interest of keeping this particular proposal tractable, they have been deferred +for future proposals. + +Some of these ideas are: +- Better UX for enabling reflection support +- gRPC Web support +- HTTP/JSON transcoding at the gateway +- Protobuf payload-based routing From b4ccd7f851d1bdecb3835a071ac76d20d676319e Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 10 Feb 2022 11:06:36 -0800 Subject: [PATCH 06/25] Add issue number --- site-src/geps/{gep-XXXX.md => gep-1016.md} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename site-src/geps/{gep-XXXX.md => gep-1016.md} (99%) diff --git a/site-src/geps/gep-XXXX.md b/site-src/geps/gep-1016.md similarity index 99% rename from site-src/geps/gep-XXXX.md rename to site-src/geps/gep-1016.md index 588cb20e97..cb73057e14 100644 --- a/site-src/geps/gep-XXXX.md +++ b/site-src/geps/gep-1016.md @@ -1,6 +1,6 @@ -# GEP-XXXX: GRPCRoute +# GEP-1016: GRPCRoute -* Issue: [TODO](https://github.com/kubernetes-sigs/gateway-api/issues/696) +* Issue: [#1016](https://github.com/kubernetes-sigs/gateway-api/issues/1016) * Status: Implementable (See definitions in [Kubernetes KEP][kep-status]. From aaa8219dabcc7a2f7a566ed3671bb12274307109 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 10 Feb 2022 11:29:57 -0800 Subject: [PATCH 07/25] Add note about listener protocl --- site-src/geps/gep-1016.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index cb73057e14..219c8638d6 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -322,6 +322,10 @@ type GRPCRequestMirrorFilter struct { } ``` +No additional options will be added to the `ProtocolType` string. The `"HTTP"` +and `"HTTPS"` options correspond exactly to the required functionality for the +two main modes of operation in gRPC. + ## Future Enhancements Many more ideas have been discussed for the `GRPCRoute` resource, but in the @@ -329,6 +333,7 @@ interest of keeping this particular proposal tractable, they have been deferred for future proposals. Some of these ideas are: +- Ingtegration with Service Meshes (both sidecar-proxied and proxyless) - Better UX for enabling reflection support - gRPC Web support - HTTP/JSON transcoding at the gateway From 586ad8de670f0afc5b7583809c13f2dab6ce1c1d Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 10 Feb 2022 11:52:17 -0800 Subject: [PATCH 08/25] Add precedence rules --- site-src/geps/gep-1016.md | 105 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 219c8638d6..80265f7073 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -162,9 +162,62 @@ spec: ### Structs ```go +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:resource:categories=gateway-api +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Hostnames",type=string,JSONPath=`.spec.hostnames` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// GRPCRoute is the Schema for the GRPCRoute resource. +type GRPCRoute struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec defines the desired state of GRPCRoute. + Spec GRPCRouteSpec `json:"spec,omitempty"` + + // Status defines the current state of GRPCRoute. + Status GRPCRouteStatus `json:"status,omitempty"` +} + +// GRPCRouteStatus defines the observed state of GRPCRoute. +type GRPCRouteStatus struct { + RouteStatus `json:",inline"` +} + type GRPCRouteSpec struct { CommonRouteSpec `json:",inline"` + // Hostnames defines a set of hostname that should match against + // the Host header to select a GRPCRoute to process the request. + // Hostname is the fully qualified domain name of a network host, + // as defined by RFC 3986. Note the following deviations from the + // "host" part of the URI as defined in the RFC: + // + // 1. IPs are not allowed. + // 2. The `:` delimiter is not respected because ports are not allowed. + // + // Incoming requests are matched against the hostnames before the + // GRCRoute rules. If no hostname is specified, traffic is routed + // based on the GRPCRouteRules. + // + // Hostname can be "precise" which is a domain name without the terminating + // dot of a network host (e.g. "foo.example.com") or "wildcard", which is + // a domain name prefixed with a single wildcard label (e.g. `*.example.com`). + // The wildcard character `*` must appear by itself as the first DNS + // label and matches only a single label. + // You cannot have a wildcard label by itself (e.g. Host == `*`). + // Requests will be matched against the Host field in the following order: + // + // 1. If Host is precise, the request matches this rule if + // the HTTP Host header is equal to Host. + // 2. If Host is a wildcard, then the request matches this rule if + // the HTTP Host header is to equal to the suffix + // (removing the first label) of the wildcard rule. + // + // Support: Core + // // Support: Core // // +optional @@ -178,6 +231,58 @@ type GRPCRouteSpec struct { } type GRPCRouteRule struct { + // Matches define conditions used for matching the rule against incoming + // gRPC requests. Each match is independent, i.e. this rule will be matched + // if **any** one of the matches is satisfied. + // + // For example, take the following matches configuration: + // + // ``` + // matches: + // - method: + // method: foo.bar + // headers: + // values: + // version: 2 + // - method: + // service: foo.bar.v2 + // ``` + // + // For a request to match against this rule, a request should satisfy + // EITHER of the two conditions: + // + // - service of foo.bar AND contains the header `version: 2` + // - service of foo.bar.v2 + // + // See the documentation for GRPCRouteMatch on how to specify multiple + // match conditions that should be ANDed together. + // + // If no matches are specified, the matches every gRPC request. + // + // Each client request MUST map to a maximum of one route rule. If a request + // matches multiple rules, matching precedence MUST be determined in order + // of the following criteria, continuing on ties: + // + // * The longest matching hostname. + // * HTTPRoutes take precedence over GRPCRoutes + // * The longest matching service. + // * The longest matching method. + // * The largest number of header matches. + // + // If ties still exist across multiple Routes, matching precedence MUST be + // determined in order of the following criteria, continuing on ties: + // + // * The oldest Route based on creation timestamp. For example, a Route with + // a creation timestamp of "2020-09-08 01:02:03" is given precedence over + // a Route with a creation timestamp of "2020-09-08 01:02:04". + // * The Route appearing first in alphabetical order by + // "{namespace}/{name}". For example, foo/bar is given precedence over + // foo/baz. + // + // If ties still exist within the Route that has been given precedence, + // matching precedence MUST be granted to the first matching rule meeting + // the above criteria. + // // +optional // +kubebuilder:validation:MaxItems=8 // +kubebuilder:default={{method: {type: "RegularExpression", service: ".*", method: ".*"}}} From adfeba77fee59ade657fe10d83e6b90b9f8ae1c2 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 10 Feb 2022 12:06:03 -0800 Subject: [PATCH 09/25] Start adding docstrings --- site-src/geps/gep-1016.md | 108 +++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 80265f7073..018080ca9d 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -166,10 +166,14 @@ spec: // +kubebuilder:object:root=true // +kubebuilder:resource:categories=gateway-api // +kubebuilder:subresource:status +// +kubebuilder:storageversion // +kubebuilder:printcolumn:name="Hostnames",type=string,JSONPath=`.spec.hostnames` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` -// GRPCRoute is the Schema for the GRPCRoute resource. +// GRPCRoute provides a way to route gRPC requests. This includes the capability +// to match requests by hostname, service, method, or header. Filters can be +// used to specify additional processing steps. Backends specify where matching +// requests should be routed. type GRPCRoute struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -186,37 +190,41 @@ type GRPCRouteStatus struct { RouteStatus `json:",inline"` } +// GRPCRouteSpec defines the desired state of GRPCRoute type GRPCRouteSpec struct { CommonRouteSpec `json:",inline"` - // Hostnames defines a set of hostname that should match against - // the Host header to select a GRPCRoute to process the request. - // Hostname is the fully qualified domain name of a network host, - // as defined by RFC 3986. Note the following deviations from the - // "host" part of the URI as defined in the RFC: + // Hostnames defines a set of hostname that should match against the GRPC + // Host header to select a GRPCRoute to process the request. This matches + // the RFC 1123 definition of a hostname with 2 notable exceptions: // // 1. IPs are not allowed. - // 2. The `:` delimiter is not respected because ports are not allowed. + // 2. A hostname may be prefixed with a wildcard label (`*.`). The wildcard + // label must appear by itself as the first label. // - // Incoming requests are matched against the hostnames before the - // GRCRoute rules. If no hostname is specified, traffic is routed - // based on the GRPCRouteRules. + // If a hostname is specified by both the Listener and GRPCRoute, there + // must be at least one intersecting hostname for the GRPCRoute to be + // attached to the Listener. For example: // - // Hostname can be "precise" which is a domain name without the terminating - // dot of a network host (e.g. "foo.example.com") or "wildcard", which is - // a domain name prefixed with a single wildcard label (e.g. `*.example.com`). - // The wildcard character `*` must appear by itself as the first DNS - // label and matches only a single label. - // You cannot have a wildcard label by itself (e.g. Host == `*`). - // Requests will be matched against the Host field in the following order: + // * A Listener with `test.example.com` as the hostname matches GRPCRoutes + // that have either not specified any hostnames, or have specified at + // least one of `test.example.com` or `*.example.com`. + // * A Listener with `*.example.com` as the hostname matches GRPCRoutes + // that have either not specified any hostnames or have specified at least + // one hostname that matches the Listener hostname. For example, + // `test.example.com` and `*.example.com` would both match. On the other + // hand, `example.com` and `test.example.net` would not match. // - // 1. If Host is precise, the request matches this rule if - // the HTTP Host header is equal to Host. - // 2. If Host is a wildcard, then the request matches this rule if - // the HTTP Host header is to equal to the suffix - // (removing the first label) of the wildcard rule. + // If both the Listener and GRPCRoute have specified hostnames, any + // GRPCRoute hostnames that do not match the Listener hostname MUST be + // ignored. For example, if a Listener specified `*.example.com`, and the + // GRPCRoute specified `test.example.com` and `test.example.net`, + // `test.example.net` must not be considered for a match. // - // Support: Core + // If both the Listener and GRPCRoute have specified hostnames, and none + // match with the criteria above, then the GRPCRoute is not accepted. The + // implementation must raise an 'Accepted' Condition with a status of + // `False` in the corresponding RouteParentStatus. // // Support: Core // @@ -224,12 +232,17 @@ type GRPCRouteSpec struct { // +kubebuilder:validation:MaxItems=16 Hostnames []Hostname `json:"hostnames,omitempty"` + // Rules are a list of GRPC matchers, filters and actions. + // // +optional // +kubebuilder:validation:MaxItems=16 // +kubebuilder:default={{matches: {{method: {type: "RegularExpression", service: ".*", method: ".*"}}}}} Rules []GRPCRouteRule `json:"rules,omitempty"` } +// GRPCRouteRule defines semantics for matching an gRPC request based on +// conditions (matches), processing it (filters), and forwarding the request to +// an API object (backendRefs). type GRPCRouteRule struct { // Matches define conditions used for matching the rule against incoming // gRPC requests. Each match is independent, i.e. this rule will be matched @@ -259,25 +272,24 @@ type GRPCRouteRule struct { // // If no matches are specified, the matches every gRPC request. // - // Each client request MUST map to a maximum of one route rule. If a request - // matches multiple rules, matching precedence MUST be determined in order - // of the following criteria, continuing on ties: + // Proxy or Load Balancer routing configuration generated from HTTPRoutes + // MUST prioritize rules based on the following criteria, continuing on + // ties. Precedence must be given to the the Rule with the largest number + // of: // - // * The longest matching hostname. - // * HTTPRoutes take precedence over GRPCRoutes - // * The longest matching service. - // * The longest matching method. - // * The largest number of header matches. + // * Characters in a matching non-wildcard hostname. + // * Characters in a matching hostname. + // * HTTPRoute over GRPCRoute + // * Characters in a matching service. + // * Characters in a matching method. + // * Header matches. // // If ties still exist across multiple Routes, matching precedence MUST be // determined in order of the following criteria, continuing on ties: // - // * The oldest Route based on creation timestamp. For example, a Route with - // a creation timestamp of "2020-09-08 01:02:03" is given precedence over - // a Route with a creation timestamp of "2020-09-08 01:02:04". + // * The oldest Route based on creation timestamp. // * The Route appearing first in alphabetical order by - // "{namespace}/{name}". For example, foo/bar is given precedence over - // foo/baz. + // "{namespace}/{name}". // // If ties still exist within the Route that has been given precedence, // matching precedence MUST be granted to the first matching rule meeting @@ -288,12 +300,36 @@ type GRPCRouteRule struct { // +kubebuilder:default={{method: {type: "RegularExpression", service: ".*", method: ".*"}}} Matches []GRPCRouteMatch `json:"matches,omitempty"` + // Filters define the filters that are applied to requests that match + // this rule. + // + // The effects of ordering of multiple behaviors are currently unspecified. + // This can change in the future based on feedback during the alpha stage. + // + // Conformance-levels at this level are defined based on the type of filter: + // + // - ALL core filters MUST be supported by all implementations. + // - Implementers are encouraged to support extended filters. + // - Implementation-specific custom filters have no API guarantees across + // implementations. + // + // Specifying a core filter multiple times has unspecified or custom + // conformance. // Support: Core // // +optional // +kubebuilder:validation:MaxItems=16 Filters []GRPCRouteFilter `json:"filters,omitempty"` + // BackendRefs defines the backend(s) where matching requests should be + // sent. + + // If unspecified or invalid (refers to a non-existent resource or a Service + // with no endpoints), the rule performs no forwarding. If there are also no + // filters specified that would result in a response being sent, a gRPC `UNAVAILABLE` + // status is returned. `UNAVAILABLE` responses must be sent so that the overall + // weight is respected; if an invalid backend is requested to have 80% of + // requests, then 80% of requests must get a `UNAVAILABLE` instead. // Support: Core for Kubernetes Service // Support: Custom for any other resource // From b644e3bd180a768089a3ca62e7edcb87fb76ba7d Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 10 Feb 2022 13:34:26 -0800 Subject: [PATCH 10/25] Typo --- site-src/geps/gep-1016.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 018080ca9d..f05a6de7e0 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -100,8 +100,8 @@ backends while routing all other HTTP traffic to a default HTTP backend. More generally, we propose that when a new protocol encapsulated in an already supported one is added, if traffic applies to both a Route resource of the lower -layer and a Route resource of the higher layer, the rules of the lower layer -will be applied before the rules of the higher layer. +layer and a Route resource of the higher layer, the rules of the higher layer +will be applied before the rules of the lower layer. #### Proxyless Service Mesh @@ -279,7 +279,7 @@ type GRPCRouteRule struct { // // * Characters in a matching non-wildcard hostname. // * Characters in a matching hostname. - // * HTTPRoute over GRPCRoute + // * GRPCRoute over HTTPRoute // * Characters in a matching service. // * Characters in a matching method. // * Header matches. From 5c60683da5c81c02e4a5545e97a2a9fd9758d2a2 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 14 Feb 2022 12:12:11 -0800 Subject: [PATCH 11/25] Update merge semantics --- site-src/geps/gep-1016.md | 117 +++++++++++++++++++++++++++++++++++--- 1 file changed, 108 insertions(+), 9 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index f05a6de7e0..d5ead1dfae 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -93,15 +93,113 @@ particular HTTP request when two `HTTPRoutes` both apply to the request's hostname. What semantics should apply when a `GRPCRoute` and an `HTTPRoute` both apply to a request? -We propose that, in this case, all rules defined in the `HTTPRoute` should be -applied. Only then will the `GRPCRoute` rules be evaluated. This supports the -common use case of routing gRPC traffic at a particular URI prefix to gRPC -backends while routing all other HTTP traffic to a default HTTP backend. +We propose that the GRPCRoute's matchers be translated into the equivalent +HTTPRoute matchers and that the normal HTTPRoute precedence rules apply. +This translation is mostly straightforward, but one significant deviation between +GRPCRoute and HTTPRoute poses difficulties: the gRPC method matcher. -More generally, we propose that when a new protocol encapsulated in an already -supported one is added, if traffic applies to both a Route resource of the lower -layer and a Route resource of the higher layer, the rules of the higher layer -will be applied before the rules of the lower layer. +gRPC methods are encoded in the URI of an HTTP/2 message in the format +`/SERVICE/METHOD`. The matcher allows the user to supply these independently, so +that the user may leave either blank, to be interpreted as an unconditional +match. Further, this field may either be an exact match or a regex match. + +To translate this field to an HTTPRoute path matcher, we propose the following +scheme: + +- If service and method are supplied and the type is `Exact`, translate to a + path match of `/SERVICE/METHOD` with type `Exact` +- If service is supplied but not method and the type is `Exact`, translate to a + path match of `/SERVICE/` with type `PathPrefix` +- Otherwise, translate the service and the method individually into regular + expressions (with an empty field interpreted as `.*`), and translate to an + HTTPRoute path matcher of type `RegularExpression` with value `/SERVICE_REGEX/MATCHER_REGEX` + + +#### An Example Implementation + +In the interest of proving the feasibility of implementation of this proposal, +we include a sketch of an implementation for an xDS-based control plane. + +Suppose the following resources are applied to the cluster: + +```yaml +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: Gateway +metadata: + name: my-gateway +spec: + gatewayClassName: acme-lb + listeners: + - name: http + protocol: HTTP + port: 80 +--- +kind: GRPCRoute +apiVersion: gateway.networking.k8s.io/v1alpha2 +metadata: + name: foo-grpcroute +spec: + parentRefs: + - name: my-gateway + hostnames: + - foo.com + rules: + matches: + - method: + service: foo.bar + backendRefs: + - name: grpc-backend + port: 50051 +--- +kind: HTTPRoute +apiVersion: gateway.networking.k8s.io/v1alpha2 +metadata: + name: foo-httproute +spec: + parentRefs: + - name: my-gateway + hostnames: + - foo.com + rules: + matches: + - path: + type: PathPrefix + value: /foo + backendRefs: + - name: http-backend + port: 80 +``` + +According to the above scheme, we translate the gRPC method matcher to an HTTP +path matcher of type `PathPrefix` and value `/foo.bar/`. Following the HTTPRoute +precedence rules, `/foo.bar/` has more characters than `/foo`, so it has higher +precedence. Hence, the GRPCRoute's rule must be evaluated before the HTTPRoute's +rule. + +Accordingly, an xDS server should create something like the following +[`RouteConfiguration`](https://github.com/envoyproxy/envoy/blob/379a7567b0d15cc9d0806052b82670c8cad76fc5/api/envoy/config/route/v3/route.proto#L26): + +```json +{ + "virtual_hosts": [ + "domains": ["foo.com"], + "routes": [ + { + "prefix": "/foo.bar/", + "route": { + "cluster": "grpc-backend-cluster" + } + }, + { + "prefix": "/foo", + "route": { + "cluster": "http-backend-cluster" + } + } + ] + ] +} +``` #### Proxyless Service Mesh @@ -465,7 +563,8 @@ type GRPCRequestMirrorFilter struct { No additional options will be added to the `ProtocolType` string. The `"HTTP"` and `"HTTPS"` options correspond exactly to the required functionality for the -two main modes of operation in gRPC. +two main modes of operation in gRPC. This also enables a `GRPCRoute` to share +the same listener as an `HTTPRoute`. ## Future Enhancements From 7b32222b7a752e9e4316499283d8e99e8a117a79 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 14 Feb 2022 12:18:56 -0800 Subject: [PATCH 12/25] Fix section location --- site-src/geps/gep-1016.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index d5ead1dfae..dc8ea48c4c 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -84,6 +84,17 @@ The user experience would also degrade significantly if forced to route at the l - Methods besides `POST` - CORS + +#### Proxyless Service Mesh + +The gRPC library supports proxyless service mesh, a system by which routing +configuration is received not by an in-line proxy or sidecar proxy but by the client +itself. Eventually, `GRPCRoute` in the Gateway APIs should support this feature. +However, to date, there are no HTTP client libraries capable of participating +in a proxyless service mesh. + +--- + ### Cross Serving A new route type (and especially one encapsulated in an already supported @@ -201,16 +212,6 @@ Accordingly, an xDS server should create something like the following } ``` -#### Proxyless Service Mesh - -The gRPC library supports proxyless service mesh, a system by which routing -configuration is received not by an in-line proxy or sidecar proxy but by the client -itself. Eventually, `GRPCRoute` in the Gateway APIs should support this feature. -However, to date, there are no HTTP client libraries capable of participating -in a proxyless service mesh. - ---- - ## API The API deviates from `HTTPRoute` where it results in a better UX for gRPC From ac0ebc5eecd7951d359e832f01f56199181321e6 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 14 Feb 2022 13:56:26 -0800 Subject: [PATCH 13/25] Flesh out docstrings a bit more --- site-src/geps/gep-1016.md | 183 +++++++++++++++++++++++++++++++++++--- 1 file changed, 170 insertions(+), 13 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index dc8ea48c4c..ab9044744c 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -117,6 +117,8 @@ match. Further, this field may either be an exact match or a regex match. To translate this field to an HTTPRoute path matcher, we propose the following scheme: +- If neither service nor method is supplied and the type is `Exact`, translate to a + path match of `/` with type `PathPrefix` - If service and method are supplied and the type is `Exact`, translate to a path match of `/SERVICE/METHOD` with type `Exact` - If service is supplied but not method and the type is `Exact`, translate to a @@ -335,7 +337,7 @@ type GRPCRouteSpec struct { // // +optional // +kubebuilder:validation:MaxItems=16 - // +kubebuilder:default={{matches: {{method: {type: "RegularExpression", service: ".*", method: ".*"}}}}} + // +kubebuilder:default={{matches: {{method: {type: "Exact"}}}}} Rules []GRPCRouteRule `json:"rules,omitempty"` } @@ -371,17 +373,35 @@ type GRPCRouteRule struct { // // If no matches are specified, the matches every gRPC request. // - // Proxy or Load Balancer routing configuration generated from HTTPRoutes + // Proxy or Load Balancer routing configuration generated from GRPCRoutes or HTTPRoutes // MUST prioritize rules based on the following criteria, continuing on - // ties. Precedence must be given to the the Rule with the largest number - // of: + // ties. When comparing GRPCRoutes and HTTPRoutes, GRPCRoute matches are first converted + // to HTTPRoute matches according to the following scheme: + // + // * HeaderMatches are translated exactly, since the field are + // one-to-one + // * If `MethodMatch.type` is `Exact` and neither the + // `MethodMatch.service` nor `MethodMatch.method` fields are present, + // translate to a `PathMatch` of `{"type": "PathPrefix", value: "/"}` + // * If `MethodMatch.type` is `Exact` and both the `MethodMatch.service` + // and `MethodMatch.method` fields are present, translate to a `PathMatch` + // of `{"type": "Exact", value: "/" + service + "/" + method}` + // * If `MethodMatch.type` is `Exact` and the `MethodMatch.service` + // field is present but not the `MethodMatch.method` field, translate to a + // `PathMatch` of `{"type": "PathPrefix", value: "/" + service}` + // - Otherwise, translate the service and the method individually into + // regular expressions (with an absent field interpreted as `.*`) and + // translate to a `PathMatch` of + // `{"type": "RegularExpression", value: "/": serviceRegex + "/" + methodRegex}` + // + // Once the translation is applied, precedence must be given to the Rule with + // the largest number of: // // * Characters in a matching non-wildcard hostname. // * Characters in a matching hostname. - // * GRPCRoute over HTTPRoute - // * Characters in a matching service. - // * Characters in a matching method. + // * Characters in a matching path. // * Header matches. + // * Query param matches. // // If ties still exist across multiple Routes, matching precedence MUST be // determined in order of the following criteria, continuing on ties: @@ -393,10 +413,11 @@ type GRPCRouteRule struct { // If ties still exist within the Route that has been given precedence, // matching precedence MUST be granted to the first matching rule meeting // the above criteria. + // // // +optional // +kubebuilder:validation:MaxItems=8 - // +kubebuilder:default={{method: {type: "RegularExpression", service: ".*", method: ".*"}}} + // +kubebuilder:default={{method: {type: "Exact"}}} Matches []GRPCRouteMatch `json:"matches,omitempty"` // Filters define the filters that are applied to requests that match @@ -439,11 +460,34 @@ type GRPCRouteRule struct { BackendRefs []GRPCBackendRef `json:"backendRefs,omitempty"` } +// GRPCRouteMatch defines the predicate used to match requests to a given +// action. Multiple match types are ANDed together, i.e. the match will +// evaluate to true only if all conditions are satisfied. +// +// For example, the match below will match a gRPC request only if its service +// is `foo` AND it contains the `version: v1` header: +// +// ``` +// match: +// method: +// type: Exact +// service: "foo" +// headers: +// - name: "version" +// value "v1" +// ``` type GRPCRouteMatch struct { + // Path specifies a gRPC request service/method matcher. If this field is not + // specified, all services and methods will match. + // // +optional - // +kubebuilder:default={type: "RegularExpression", service: ".*", method: ".*"} + // +kubebuilder:default={type: "Exact"} Method *GRPCMethodMatch `json:"path,omitempty"` + // Headers specifies gRPC request header matchers. Multiple match values are + // ANDed together, meaning, a request must match all the specified headers + // to select the route. + // // +listType=map // +listMapKey=name // +optional @@ -451,7 +495,10 @@ type GRPCRouteMatch struct { Headers []GRPCHeaderMatch `json:"headers,omitempty"` } +// GRPCPathMatch describes how to select a gRPC route by matching the gRPC +// request service and/or method.. type GRPCMethodMatch struct { + // Type specifies how to match against the service and/or method. // Support: Core (Exact) // // Support: Custom (RegularExpression) @@ -460,31 +507,70 @@ type GRPCMethodMatch struct { // +kubebuilder:default=Exact Type *GRPCMethodMatchType `json:"type,omitempty"` + + // Value of the service to match against. If left empty or omitted, will + // match all services. // +optional // +kubebuilder:default="" // +kubebuilder:validation:MaxLength=1024 Service *string `json:"value,omitempty"` + // Value of the method to match against. If left empty or omitted, will + // match all services. // +optional // +kubebuilder:default="" // +kubebuilder:validation:MaxLength=1024 Method *string `json:"value,omitempty"` - - // +optional - // +kubebuilder:default=true - CaseSensitive *bool `json:"value,omitempty"` } +// MethodMatchType specifies the semantics of how gRPC methods and services should be compared. +// Valid MethodMatchType values are: +// +// * "Exact" +// * "RegularExpression" +// +// Exact paths must be syntactically valid: +// +// - Must not contain `/` character +// +// +kubebuilder:validation:Enum=Exact;PathPrefix;RegularExpression // +kubebuilder:validation:Enum=Exact;RegularExpression type GRPCMethodMatchType string +const ( + // Matches the service and/or method exactly and with case sensitivity. + PathMatchExact PathMatchType = "Exact" + + // Matches if the service and/or method matches the given regular expression with + // case sensitivity. + // + // Since `"RegularExpression"` has custom conformance, implementations + // can support POSIX, PCRE, RE2 or any other regular expression dialect. + // Please read the implementation's documentation to determine the supported + // dialect. + PathMatchRegularExpression PathMatchType = "RegularExpression" +) + +// GRPCHeaderMatch describes how to select a gRPC route by matching gRPC request +// headers. type GRPCHeaderMatch struct { + // Type specifies how to match against the value of the header. + // // +optional // +kubebuilder:default=Exact Type *HeaderMatchType `json:"type,omitempty"` + // Name is the name of the gRPC Header to be matched. + // + // If multiple entries specify equivalent header names, only the first + // entry with an equivalent name MUST be considered for a match. Subsequent + // entries with an equivalent header name MUST be ignored. Due to the + // case-insensitivity of header names, "foo" and "Foo" are considered + // equivalent. Name GRPCHeaderName `json:"name"` + // Value is the value of the gRPC Header to be matched. + // // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=4096 Value string `json:"value"` @@ -498,31 +584,102 @@ type HeaderMatchType string // +kubebuilder:validation:Pattern=`^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$` type GRPCHeaderName string +// GRPCBackendRef defines how a GRPCRoute should forward a gRPC request. type GRPCBackendRef struct { + // BackendRef is a reference to a backend to forward matched requests to. + // + // If the referent cannot be found, this GRPCBackendRef is invalid and must + // be dropped from the Gateway. The controller must ensure the + // "ResolvedRefs" condition on the Route is set to `status: False` and not + // configure this backend in the underlying implementation. + // + // If there is a cross-namespace reference to an *existing* object + // that is not covered by a ReferencePolicy, the controller must ensure the + // "ResolvedRefs" condition on the Route is set to `status: False`, + // with the "RefNotPermitted" reason and not configure this backend in the + // underlying implementation. + // + // In either error case, the Message of the `ResolvedRefs` Condition + // should be used to provide more detail about the problem. + // + // Support: Custom + // // +optional BackendRef `json:",inline"` + // Filters defined at this level should be executed if and only if the + // request is being forwarded to the backend defined here. + // + // Support: Custom (For broader support of filters, use the Filters field + // in GRPCRouteRule.) + // // +optional // +kubebuilder:validation:MaxItems=16 Filters []GRPCRouteFilter `json:"filters,omitempty"` } +// GRPCRouteFilter defines processing steps that must be completed during the +// request or response lifecycle. GRPCRouteFilters are meant as an extension +// point to express processing that may be done in Gateway implementations. Some +// examples include request or response modification, implementing +// authentication strategies, rate-limiting, and traffic shaping. API +// guarantee/conformance is defined based on the type of the filter. type GRPCRouteFilter struct { + // Type identifies the type of filter to apply. As with other API fields, + // types are classified into three conformance levels: + // + // - Core: Filter types and their corresponding configuration defined by + // "Support: Core" in this package, e.g. "RequestHeaderModifier". All + // implementations must support core filters. + // + // - Extended: Filter types and their corresponding configuration defined by + // "Support: Extended" in this package, e.g. "RequestMirror". Implementers + // are encouraged to support extended filters. + // + // - Custom: Filters that are defined and supported by specific vendors. + // In the future, filters showing convergence in behavior across multiple + // implementations will be considered for inclusion in extended or core + // conformance levels. Filter-specific configuration for such filters + // is specified using the ExtensionRef field. `Type` should be set to + // "ExtensionRef" for custom filters. + // + // Implementers are encouraged to define custom implementation types to + // extend the core API with implementation-specific behavior. + // + // If a reference to a custom filter type cannot be resolved, the filter + // MUST NOT be skipped. Instead, requests that would have been processed by + // that filter MUST receive a HTTP error response. + // // +unionDiscriminator // +kubebuilder:validation:Enum=RequestHeaderModifier;RequestMirror;ExtensionRef // Type GRPCRouteFilterType `json:"type"` + // RequestHeaderModifier defines a schema for a filter that modifies request + // headers. + // + // Support: Core + // // Support: Core // // +optional RequestHeaderModifier *GRPCRequestHeaderFilter `json:"requestHeaderModifier,omitempty"` + // RequestMirror defines a schema for a filter that mirrors requests. + // Requests are sent to the specified destination, but responses from + // that destination are ignored. + // // Support: Extended // // +optional RequestMirror *GRPCRequestMirrorFilter `json:"requestMirror,omitempty"` + // ExtensionRef is an optional, implementation-specific extension to the + // "filter" behavior. For example, resource "myroutefilter" in group + // "networking.example.net"). ExtensionRef MUST NOT be used for core and + // extended filters. + // + // Support: Implementation-specific // Support: Implementation-specific // // +optional From a91c6b463cc102f79d95fdd235c9f0c6ec836c18 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 14 Feb 2022 14:07:49 -0800 Subject: [PATCH 14/25] Yet more docstrings --- site-src/geps/gep-1016.md | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index ab9044744c..ccf57be83a 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -686,33 +686,108 @@ type GRPCRouteFilter struct { ExtensionRef *LocalObjectReference `json:"extensionRef,omitempty"` } +// GPRCRequestHeaderFilter defines configuration for the RequestHeaderModifier +// filter. type GRPCRequestHeaderFilter struct { + // Set overwrites the request with the given header (name, value) + // before the action. + // + // Input: + // my-header: foo + // + // Config: + // set: + // - name: "my-header" + // value: "bar" + // + // Output: + // my-header: bar + // // +optional // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=16 Set []GRPCHeader `json:"set,omitempty"` + // Add adds the given header(s) (name, value) to the request + // before the action. It appends to any existing values associated + // with the header name. + // + // Input: + // my-header: foo + // + // Config: + // add: + // - name: "my-other-header" + // value: "bar" + // + // Output: + // my-header: foo + // my-other-header: bar + // // +optional // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=16 Add []GRPCHeader `json:"add,omitempty"` + // Remove the given header(s) from the HTTP request before the action. The + // value of Remove is a list of gRPC header names. Note that the header + // names are case-insensitive. + // + // Input: + // my-header1: foo + // my-header2: bar + // my-header3: baz + // + // Config: + // remove: ["my-header1", "my-header3"] + // + // Output: + // my-header2: bar + // // +optional // +kubebuilder:validation:MaxItems=16 Remove []string `json:"remove,omitempty"` } +// HTTPHeader represents an HTTP Header name and value. type GRPCHeader struct { + // Name is the name of the gRPC Header to be matched. Name matching MUST be + // case insensitive. + // + // If multiple entries specify equivalent header names, the first entry with + // an equivalent name MUST be considered for a match. Subsequent entries + // with an equivalent header name MUST be ignored. Due to the + // case-insensitivity of header names, "foo" and "Foo" are considered + // equivalent. Name GRPCHeaderName `json:"name"` + // Value is the value of the gRPC Header to be matched. + // // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=4096 Value string `json:"value"` } +// GRPCRequestMirrorFilter defines configuration for the RequestMirror filter. type GRPCRequestMirrorFilter struct { + // BackendRef references a resource where mirrored requests are sent. + // + // If the referent cannot be found, this BackendRef is invalid and must be + // dropped from the Gateway. The controller must ensure the "ResolvedRefs" + // condition on the Route status is set to `status: False` and not configure + // this backend in the underlying implementation. + // + // If there is a cross-namespace reference to an *existing* object + // that is not allowed by a ReferencePolicy, the controller must ensure the + // "ResolvedRefs" condition on the Route is set to `status: False`, + // with the "RefNotPermitted" reason and not configure this backend in the + // underlying implementation. + // + // In either error case, the Message of the `ResolvedRefs` Condition + // should be used to provide more detail about the problem. + // // Support: Extended for Kubernetes Service // Support: Custom for any other resource BackendRef BackendObjectReference `json:"backendRef"` From 527901eb3f1fe1b0cf1cedf63ccaf585806ffb35 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 28 Feb 2022 14:58:22 -0800 Subject: [PATCH 15/25] Update proposal --- site-src/geps/gep-1016.md | 230 ++++++++++++++------------------------ 1 file changed, 83 insertions(+), 147 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index ccf57be83a..be6a2b83cc 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -97,122 +97,33 @@ in a proxyless service mesh. ### Cross Serving -A new route type (and especially one encapsulated in an already supported -protocol) raises questions about precedence of routing rules. Today, there is -already a complicated algorithm dictating which routing rule to apply to a -particular HTTP request when two `HTTPRoutes` both apply to the request's -hostname. What semantics should apply when a `GRPCRoute` and an `HTTPRoute` both apply to -a request? +Occasionally, gRPC users will place gRPC services on the same hostname/port +combination as HTTP services. For example, `foo.com:443/v1` might serve +REST+JSON while `foo.com:443/com.foo.WidgetService/` serves gRPC. Such an +arrangement in the Gateway APIs poses complex technical challenges. How are +GRPCRoutes to be reconciled with HTTPRoutes? And how should invididual +implementations accomplisht this? + +After a long look at the implementations with which the author is familiar, it +was deemed technically infeasible. Furthermore, after surveying the gRPC +community, this was found to be a niche use case to begin with. + +In any case, users wishing to accomplish this always have the option of using +`HTTPRoute` resources to achieve this use case, at the cost of a degraded user +experience. + +If at some point in the future, demand for this use case increases and we have +reason to believe that the feasibility of implementation has improved, this +would be a backward compatible change. + +As such, implementations that support GRPCRoute must enforce uniqueness of +hostnames between `GRPCRoute`s and `HTTPRoute`s. If a route (A) of type `HTTPRoute` or +`GRPCRoute` is attached to a Listener and that listener already has another Route (B) of +the other type attached and the intersection of the hostnames of A and B is +non-empty, then the implementation must reject Route A. That is, the +implementation must raise an 'Accepted' condition with a status of 'False' in +the corresponding RouteParentStatus. -We propose that the GRPCRoute's matchers be translated into the equivalent -HTTPRoute matchers and that the normal HTTPRoute precedence rules apply. -This translation is mostly straightforward, but one significant deviation between -GRPCRoute and HTTPRoute poses difficulties: the gRPC method matcher. - -gRPC methods are encoded in the URI of an HTTP/2 message in the format -`/SERVICE/METHOD`. The matcher allows the user to supply these independently, so -that the user may leave either blank, to be interpreted as an unconditional -match. Further, this field may either be an exact match or a regex match. - -To translate this field to an HTTPRoute path matcher, we propose the following -scheme: - -- If neither service nor method is supplied and the type is `Exact`, translate to a - path match of `/` with type `PathPrefix` -- If service and method are supplied and the type is `Exact`, translate to a - path match of `/SERVICE/METHOD` with type `Exact` -- If service is supplied but not method and the type is `Exact`, translate to a - path match of `/SERVICE/` with type `PathPrefix` -- Otherwise, translate the service and the method individually into regular - expressions (with an empty field interpreted as `.*`), and translate to an - HTTPRoute path matcher of type `RegularExpression` with value `/SERVICE_REGEX/MATCHER_REGEX` - - -#### An Example Implementation - -In the interest of proving the feasibility of implementation of this proposal, -we include a sketch of an implementation for an xDS-based control plane. - -Suppose the following resources are applied to the cluster: - -```yaml -apiVersion: gateway.networking.k8s.io/v1alpha2 -kind: Gateway -metadata: - name: my-gateway -spec: - gatewayClassName: acme-lb - listeners: - - name: http - protocol: HTTP - port: 80 ---- -kind: GRPCRoute -apiVersion: gateway.networking.k8s.io/v1alpha2 -metadata: - name: foo-grpcroute -spec: - parentRefs: - - name: my-gateway - hostnames: - - foo.com - rules: - matches: - - method: - service: foo.bar - backendRefs: - - name: grpc-backend - port: 50051 ---- -kind: HTTPRoute -apiVersion: gateway.networking.k8s.io/v1alpha2 -metadata: - name: foo-httproute -spec: - parentRefs: - - name: my-gateway - hostnames: - - foo.com - rules: - matches: - - path: - type: PathPrefix - value: /foo - backendRefs: - - name: http-backend - port: 80 -``` - -According to the above scheme, we translate the gRPC method matcher to an HTTP -path matcher of type `PathPrefix` and value `/foo.bar/`. Following the HTTPRoute -precedence rules, `/foo.bar/` has more characters than `/foo`, so it has higher -precedence. Hence, the GRPCRoute's rule must be evaluated before the HTTPRoute's -rule. - -Accordingly, an xDS server should create something like the following -[`RouteConfiguration`](https://github.com/envoyproxy/envoy/blob/379a7567b0d15cc9d0806052b82670c8cad76fc5/api/envoy/config/route/v3/route.proto#L26): - -```json -{ - "virtual_hosts": [ - "domains": ["foo.com"], - "routes": [ - { - "prefix": "/foo.bar/", - "route": { - "cluster": "grpc-backend-cluster" - } - }, - { - "prefix": "/foo", - "route": { - "cluster": "http-backend-cluster" - } - } - ] - ] -} -``` ## API @@ -272,9 +183,11 @@ spec: // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // GRPCRoute provides a way to route gRPC requests. This includes the capability -// to match requests by hostname, service, method, or header. Filters can be +// to match requests by hostname, gRPC service, gRPC method, or HTTP/2 header. Filters can be // used to specify additional processing steps. Backends specify where matching // requests should be routed. +// +// Support: Extended type GRPCRoute struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -327,6 +240,13 @@ type GRPCRouteSpec struct { // implementation must raise an 'Accepted' Condition with a status of // `False` in the corresponding RouteParentStatus. // + // If a Route (A) of type HTTPRoute or GRPCRoute is attached to a + // Listener and that listener already has another Route (B) of the other + // type attached and the intersection of the hostnames of A and B is + // non-empty, then the implementation must reject Route A. That is, the + // implementation must raise an 'Accepted' condition with a status of + // 'False' in the corresponding RouteParentStatus. + // // Support: Core // // +optional @@ -371,37 +291,18 @@ type GRPCRouteRule struct { // See the documentation for GRPCRouteMatch on how to specify multiple // match conditions that should be ANDed together. // - // If no matches are specified, the matches every gRPC request. + // If no matches are specified, the implementation must match every gRPC request. // - // Proxy or Load Balancer routing configuration generated from GRPCRoutes or HTTPRoutes + // Proxy or Load Balancer routing configuration generated from GRPCRoutes // MUST prioritize rules based on the following criteria, continuing on - // ties. When comparing GRPCRoutes and HTTPRoutes, GRPCRoute matches are first converted - // to HTTPRoute matches according to the following scheme: - // - // * HeaderMatches are translated exactly, since the field are - // one-to-one - // * If `MethodMatch.type` is `Exact` and neither the - // `MethodMatch.service` nor `MethodMatch.method` fields are present, - // translate to a `PathMatch` of `{"type": "PathPrefix", value: "/"}` - // * If `MethodMatch.type` is `Exact` and both the `MethodMatch.service` - // and `MethodMatch.method` fields are present, translate to a `PathMatch` - // of `{"type": "Exact", value: "/" + service + "/" + method}` - // * If `MethodMatch.type` is `Exact` and the `MethodMatch.service` - // field is present but not the `MethodMatch.method` field, translate to a - // `PathMatch` of `{"type": "PathPrefix", value: "/" + service}` - // - Otherwise, translate the service and the method individually into - // regular expressions (with an absent field interpreted as `.*`) and - // translate to a `PathMatch` of - // `{"type": "RegularExpression", value: "/": serviceRegex + "/" + methodRegex}` - // - // Once the translation is applied, precedence must be given to the Rule with - // the largest number of: + // ties. Merging must not be done between GRPCRoutes and HTTPRoutes. + // Precedence must be given to the rule with the largest number of: // // * Characters in a matching non-wildcard hostname. // * Characters in a matching hostname. - // * Characters in a matching path. + // * Characters in a matching service. + // * Characters in a matching method. // * Header matches. - // * Query param matches. // // If ties still exist across multiple Routes, matching precedence MUST be // determined in order of the following criteria, continuing on ties: @@ -413,7 +314,6 @@ type GRPCRouteRule struct { // If ties still exist within the Route that has been given precedence, // matching precedence MUST be granted to the first matching rule meeting // the above criteria. - // // // +optional // +kubebuilder:validation:MaxItems=8 @@ -499,7 +399,9 @@ type GRPCRouteMatch struct { // request service and/or method.. type GRPCMethodMatch struct { // Type specifies how to match against the service and/or method. - // Support: Core (Exact) + // Support: Core (Exact with method specified) + // + // Support Custom (Exact with no method specified) // // Support: Custom (RegularExpression) // @@ -794,10 +696,44 @@ type GRPCRequestMirrorFilter struct { } ``` -No additional options will be added to the `ProtocolType` string. The `"HTTP"` -and `"HTTPS"` options correspond exactly to the required functionality for the -two main modes of operation in gRPC. This also enables a `GRPCRoute` to share -the same listener as an `HTTPRoute`. +In addition, two new `ProtocolType` options will be added: + +```diff +--- a/apis/v1alpha2/gateway_types.go ++++ b/apis/v1alpha2/gateway_types.go +@@ -165,6 +165,11 @@ type Listener struct { + // protocol layers as described above. If an implementation does not + // ensure that both the SNI and Host header match the Listener hostname, + // it MUST clearly document that. ++ // * H2C: The Listener Hostname MUST match the :authority pseudo-header of the request. ++ // * H2: The Listener Hostname SHOULD match at both the TLS and HTTP ++ // protocol layers as described above. If an implementation does not ++ // ensure that both the SNI and :authority pseudo-header match the Listener hostname, ++ // it MUST clearly document that. + // + // For HTTPRoute and TLSRoute resources, there is an interaction with the + // `spec.hostnames` array. When both listener and route specify hostnames, +@@ -279,6 +284,12 @@ const ( + + // Accepts UDP packets. + UDPProtocolType ProtocolType = "UDP" ++ ++ // Accepts HTTP/2 sessions over TLS. ++ H2ProtocolType ProtocolType = "H2" ++ ++ // Accepts cleartext HTTP/2 sessions. ++ H2CProtocolType ProtocolType = "H2C" + ) + + // GatewayTLSConfig describes a TLS configuration. +``` + +TLS-encryption is the default in HTTP/2, so H2C is the +[marked](https://en.wikipedia.org/wiki/Markedness) option here. + +If the listener supports neither `H2` nor `H2C`, then the implementation MUST +raise an `Accepted` Condition with a status of `False` in the corresponding +RouteParentStatus. ## Future Enhancements From 4e26150d7d55359527674aa043243cfa30a060bd Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 28 Feb 2022 15:01:42 -0800 Subject: [PATCH 16/25] Fix claim --- site-src/geps/gep-1016.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index be6a2b83cc..05a9633f45 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -40,9 +40,7 @@ configuration landscape does not Balkanize. ### Encapsulated Network Protocols -At the time of writing, the only kind of route officially defined by the Gateway -APIs is `HttpRoute`. This GEP is novel not only in that it introduces a second -protocol to route, but also in that it introduces the first protocol +This GEP is novel in that it introduces the first protocol encapsulated in a protocol already supported by the API. That is, it _is_ theoretically possible to route gRPC traffic using only `HTTPRoute` From fdb635b6b932a4368a6ba61f64256f27e463c3ce Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 14 Mar 2022 15:18:16 -0700 Subject: [PATCH 17/25] More fully explain method matcher behavior --- site-src/geps/gep-1016.md | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 05a9633f45..605abb9d61 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -21,10 +21,8 @@ as the IDL, which makes up the vast majority of gRPC traffic in the wild. ## Introduction -At the time of writing, the only official Route resource within the Gateway APIs -is HTTPRoute. It _is_ possible to support other protocols via CRDs and -controllers taking advantage of this have started to pop up. However, in the -long run, this leads to a fragmented ecosystem. +While it would be possible to support gRPC via CRDs, in the long run, this would +lead to a fragmented ecosystem. gRPC is a [popular RPC framework adopted widely across the industry](https://grpc.io/about/#whos-using-grpc-and-why). The protocol is used pervasively within the Kubernetes project itself as the basis for @@ -40,10 +38,7 @@ configuration landscape does not Balkanize. ### Encapsulated Network Protocols -This GEP is novel in that it introduces the first protocol -encapsulated in a protocol already supported by the API. - -That is, it _is_ theoretically possible to route gRPC traffic using only `HTTPRoute` +It is theoretically possible to route gRPC traffic using only `HTTPRoute` resources, but there are several serious problems with forcing gRPC users to route traffic at the level of HTTP. This is why we propose a new resource. @@ -169,6 +164,32 @@ spec: weight: 10 ``` +#### Matcher Types + +`GRPCRoute` method matchers admits two types: `Exact` and `RegularExpression`. +If not specified, the match will be treated as type `Exact`. Method matchers +will act _as if_ a URI match had been used. A full matrix of equivalent behavior +is provided below: + +##### Type Exact + +|Service|Method|URI Matcher| +|----------|----------|-----------| +|Specified|Specified|Exact `/${SERVICE}/${METHOD}`| +|Specified|Unspecified|Prefix `/${SERVICE}/`| +|Unspecified|Specified|Suffix `/${METHOD}/` or Regex `/.+/${METHOD}`| +|Unspecified|Unspecified|Prefix `/`| + +##### Type RegularExpression + +|Service|Method|URI Matcher| +|----------|----------|-----------| +|Specified|Specified|Regex `/${SERVICE}/${METHOD}`| +|Specified|Unspecified|Regex `/${SERVICE}/.+`| +|Unspecified|Specified|Regex `/.+/${METHOD}`| +|Unspecified|Unspecified|Prefix `/`| + + ### Structs ```go From 5c54bf531a4442e1eb9883db073d4c0bb1af3f03 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 22 Mar 2022 13:17:54 -0700 Subject: [PATCH 18/25] Add graduation criteria --- site-src/geps/gep-1016.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 605abb9d61..3297c5b115 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -754,6 +754,19 @@ If the listener supports neither `H2` nor `H2C`, then the implementation MUST raise an `Accepted` Condition with a status of `False` in the corresponding RouteParentStatus. +## Beta Graduation Criteria + +- `GRPCRoute` has been implemented by at least 2 controllers. +- Conformance tests are in place for the majority of the API surface. +- It is known that users of `GRPCRoute` exist. +- An API review has been performed by upstream Kubernetes reviewers. + +## GA Graduation Criteria + +- `GRPCRoute` has been implemented by at least 4 controllers. +- Exhaustive conformance tests are in place. +- Adoption of `GRPCRoute` has been shown to have expanded beyond its initial set of users. + ## Future Enhancements Many more ideas have been discussed for the `GRPCRoute` resource, but in the From e97ba0bd231c1d3fb1eba38a445e458fb0e43f5d Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 31 Mar 2022 15:19:44 -0700 Subject: [PATCH 19/25] Review comments --- site-src/geps/gep-1016.md | 143 ++++++++------------------------------ 1 file changed, 28 insertions(+), 115 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 3297c5b115..389e2790af 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -21,7 +21,7 @@ as the IDL, which makes up the vast majority of gRPC traffic in the wild. ## Introduction -While it would be possible to support gRPC via CRDs, in the long run, this would +While it would be possible to support gRPC via custom, out-of-tree CRDs, in the long run, this would lead to a fragmented ecosystem. gRPC is a [popular RPC framework adopted widely across the industry](https://grpc.io/about/#whos-using-grpc-and-why). @@ -75,16 +75,16 @@ The user experience would also degrade significantly if forced to route at the l - Many features supported by HTTP/2 but not by gRPC, such as - Query parameters - Methods besides `POST` - - CORS + - [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) #### Proxyless Service Mesh The gRPC library supports proxyless service mesh, a system by which routing -configuration is received not by an in-line proxy or sidecar proxy but by the client -itself. Eventually, `GRPCRoute` in the Gateway APIs should support this feature. -However, to date, there are no HTTP client libraries capable of participating -in a proxyless service mesh. +configuration is received and acted upon not by an in-line proxy or sidecar +proxy but by the client itself. Eventually, `GRPCRoute` in the Gateway APIs +should support this feature. However, to date, there are no HTTP client +libraries capable of participating in a proxyless service mesh. --- @@ -164,6 +164,26 @@ spec: weight: 10 ``` +#### Method Matchers + +It's been pointed out that the `method` field above stutters. That is, in order +to specify a method matcher, one must type the string `method` twice in a row. +This is an artifact of less-than-clear nomenclature within gRPC. There +_are_ alternatives for the naming here, but none of them would actually be an +improvement on the stutter. Consider the following URI: + +`/foo.bar.v1.WidgetService/GetWidget` + +- `/foo.bar.v1.WidgetService/GetWidget` is called the method or, less commonly, the _full_ method. +- `foo.bar.v1.WidgetService` is called the service or, less commonly, the _full_ service (since `WidgetService` can reasonably be called the service)] +- `GetWidget` is called the method. + +These terms _could_ be added in, but these names are found almost exclusively +within the various gRPC implementations. And inconsistently across those +implementations. + +Therefore, we opt for the stutter over any of the longer names outlined above. + #### Matcher Types `GRPCRoute` method matchers admits two types: `Exact` and `RegularExpression`. @@ -584,7 +604,7 @@ type GRPCRouteFilter struct { // Support: Core // // +optional - RequestHeaderModifier *GRPCRequestHeaderFilter `json:"requestHeaderModifier,omitempty"` + RequestHeaderModifier *HTTPRequestHeaderFilter `json:"requestHeaderModifier,omitempty"` // RequestMirror defines a schema for a filter that mirrors requests. // Requests are sent to the specified destination, but responses from @@ -593,7 +613,7 @@ type GRPCRouteFilter struct { // Support: Extended // // +optional - RequestMirror *GRPCRequestMirrorFilter `json:"requestMirror,omitempty"` + RequestMirror *HTTPRequestMirrorFilter `json:"requestMirror,omitempty"` // ExtensionRef is an optional, implementation-specific extension to the // "filter" behavior. For example, resource "myroutefilter" in group @@ -606,113 +626,6 @@ type GRPCRouteFilter struct { // +optional ExtensionRef *LocalObjectReference `json:"extensionRef,omitempty"` } - -// GPRCRequestHeaderFilter defines configuration for the RequestHeaderModifier -// filter. -type GRPCRequestHeaderFilter struct { - // Set overwrites the request with the given header (name, value) - // before the action. - // - // Input: - // my-header: foo - // - // Config: - // set: - // - name: "my-header" - // value: "bar" - // - // Output: - // my-header: bar - // - // +optional - // +listType=map - // +listMapKey=name - // +kubebuilder:validation:MaxItems=16 - Set []GRPCHeader `json:"set,omitempty"` - - // Add adds the given header(s) (name, value) to the request - // before the action. It appends to any existing values associated - // with the header name. - // - // Input: - // my-header: foo - // - // Config: - // add: - // - name: "my-other-header" - // value: "bar" - // - // Output: - // my-header: foo - // my-other-header: bar - // - // +optional - // +listType=map - // +listMapKey=name - // +kubebuilder:validation:MaxItems=16 - Add []GRPCHeader `json:"add,omitempty"` - - // Remove the given header(s) from the HTTP request before the action. The - // value of Remove is a list of gRPC header names. Note that the header - // names are case-insensitive. - // - // Input: - // my-header1: foo - // my-header2: bar - // my-header3: baz - // - // Config: - // remove: ["my-header1", "my-header3"] - // - // Output: - // my-header2: bar - // - // +optional - // +kubebuilder:validation:MaxItems=16 - Remove []string `json:"remove,omitempty"` -} - -// HTTPHeader represents an HTTP Header name and value. -type GRPCHeader struct { - // Name is the name of the gRPC Header to be matched. Name matching MUST be - // case insensitive. - // - // If multiple entries specify equivalent header names, the first entry with - // an equivalent name MUST be considered for a match. Subsequent entries - // with an equivalent header name MUST be ignored. Due to the - // case-insensitivity of header names, "foo" and "Foo" are considered - // equivalent. - Name GRPCHeaderName `json:"name"` - - // Value is the value of the gRPC Header to be matched. - // - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=4096 - Value string `json:"value"` -} - -// GRPCRequestMirrorFilter defines configuration for the RequestMirror filter. -type GRPCRequestMirrorFilter struct { - // BackendRef references a resource where mirrored requests are sent. - // - // If the referent cannot be found, this BackendRef is invalid and must be - // dropped from the Gateway. The controller must ensure the "ResolvedRefs" - // condition on the Route status is set to `status: False` and not configure - // this backend in the underlying implementation. - // - // If there is a cross-namespace reference to an *existing* object - // that is not allowed by a ReferencePolicy, the controller must ensure the - // "ResolvedRefs" condition on the Route is set to `status: False`, - // with the "RefNotPermitted" reason and not configure this backend in the - // underlying implementation. - // - // In either error case, the Message of the `ResolvedRefs` Condition - // should be used to provide more detail about the problem. - // - // Support: Extended for Kubernetes Service - // Support: Custom for any other resource - BackendRef BackendObjectReference `json:"backendRef"` -} ``` In addition, two new `ProtocolType` options will be added: From cd26a2bb4e715d65d463da1e51161fb34ad9270b Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 31 Mar 2022 15:43:25 -0700 Subject: [PATCH 20/25] Clarify protocol types --- site-src/geps/gep-1016.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 389e2790af..ad2cffaeee 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -667,6 +667,9 @@ If the listener supports neither `H2` nor `H2C`, then the implementation MUST raise an `Accepted` Condition with a status of `False` in the corresponding RouteParentStatus. +Implementations supporting `H2` or `H2C` _must_ support HTTP/2 without first +upgrading from HTTP/1. + ## Beta Graduation Criteria - `GRPCRoute` has been implemented by at least 2 controllers. From 45ad0f02201f9844bc85e30a9be9e6bcfe608776 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 1 Apr 2022 11:29:13 -0700 Subject: [PATCH 21/25] Remove ProtocolTypes --- site-src/geps/gep-1016.md | 66 ++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 39 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index ad2cffaeee..7178337237 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -209,6 +209,23 @@ is provided below: |Unspecified|Specified|Regex `/.+/${METHOD}`| |Unspecified|Unspecified|Prefix `/`| +#### Transport + +No new `ProtocolType` will be added. While gRPC _does_ have some special +HTTP usage (HTTP/2 cleartext and HTTP/2 without an upgrade from HTTP/1.1), +`GRPCRoute` will be used in conjunction with the existing `HTTP` and `HTTPS` +ProtocolTypes. + +Implementations supporting `GRPCRoute` with the `HTTPS` `ProtocolType` must +accept HTTP/2 connections without an [initial upgrade from HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc7230#section-6.7). If the +implementation does not support this, then it should raise a "Detached" +condition for the affected listener with a reason of "UnsupportedProtocol" + +Implementations supporting `GRPCRoute` with the `HTTP` `ProtocolType` must +support cleartext HTTP/2 connections without an [initial upgrade from HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc7230#section-6.7). If the implementation does not support this, then it +should raise a "Detached" condition for the affected listener with a reason of +"UnsupportedProtocol" + ### Structs @@ -226,6 +243,16 @@ is provided below: // used to specify additional processing steps. Backends specify where matching // requests should be routed. // +// Implementations supporting `GRPCRoute` with the `HTTPS` `ProtocolType` must +// accept HTTP/2 connections without an initial upgrade from HTTP/1.1. If the +// implementation does not support this, then it should raise a "Detached" +// condition for the affected listener with a reason of "UnsupportedProtocol" +// +// Implementations supporting `GRPCRoute` with the `HTTP` `ProtocolType` must +// support cleartext HTTP/2 without an initial upgrade from HTTP/1.1. If the +// implementation does not support this, then it should raise a "Detached" +// condition for the affected listener with a reason of "UnsupportedProtocol" +// // Support: Extended type GRPCRoute struct { metav1.TypeMeta `json:",inline"` @@ -628,47 +655,8 @@ type GRPCRouteFilter struct { } ``` -In addition, two new `ProtocolType` options will be added: - -```diff ---- a/apis/v1alpha2/gateway_types.go -+++ b/apis/v1alpha2/gateway_types.go -@@ -165,6 +165,11 @@ type Listener struct { - // protocol layers as described above. If an implementation does not - // ensure that both the SNI and Host header match the Listener hostname, - // it MUST clearly document that. -+ // * H2C: The Listener Hostname MUST match the :authority pseudo-header of the request. -+ // * H2: The Listener Hostname SHOULD match at both the TLS and HTTP -+ // protocol layers as described above. If an implementation does not -+ // ensure that both the SNI and :authority pseudo-header match the Listener hostname, -+ // it MUST clearly document that. - // - // For HTTPRoute and TLSRoute resources, there is an interaction with the - // `spec.hostnames` array. When both listener and route specify hostnames, -@@ -279,6 +284,12 @@ const ( - - // Accepts UDP packets. - UDPProtocolType ProtocolType = "UDP" -+ -+ // Accepts HTTP/2 sessions over TLS. -+ H2ProtocolType ProtocolType = "H2" -+ -+ // Accepts cleartext HTTP/2 sessions. -+ H2CProtocolType ProtocolType = "H2C" - ) - - // GatewayTLSConfig describes a TLS configuration. -``` - -TLS-encryption is the default in HTTP/2, so H2C is the -[marked](https://en.wikipedia.org/wiki/Markedness) option here. -If the listener supports neither `H2` nor `H2C`, then the implementation MUST -raise an `Accepted` Condition with a status of `False` in the corresponding -RouteParentStatus. -Implementations supporting `H2` or `H2C` _must_ support HTTP/2 without first -upgrading from HTTP/1. ## Beta Graduation Criteria From 3e7de9cc68d80f253fccadef6f0c5ef290421931 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 1 Apr 2022 16:11:11 -0700 Subject: [PATCH 22/25] Reviewer comments --- site-src/geps/gep-1016.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 7178337237..3810af32af 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -340,7 +340,7 @@ type GRPCRouteRule struct { // ``` // matches: // - method: - // method: foo.bar + // service: foo.bar // headers: // values: // version: 2 @@ -463,11 +463,13 @@ type GRPCRouteMatch struct { // GRPCPathMatch describes how to select a gRPC route by matching the gRPC // request service and/or method.. +// +// At least one of Service and Method must be a non-empty string. type GRPCMethodMatch struct { // Type specifies how to match against the service and/or method. - // Support: Core (Exact with method specified) + // Support: Core (Exact) // - // Support Custom (Exact with no method specified) + // Support Custom (Exact with method specified but no service specified) // // Support: Custom (RegularExpression) // @@ -478,6 +480,8 @@ type GRPCMethodMatch struct { // Value of the service to match against. If left empty or omitted, will // match all services. + // + // At least one of Service and Method must be a non-empty string. // +optional // +kubebuilder:default="" // +kubebuilder:validation:MaxLength=1024 @@ -485,6 +489,8 @@ type GRPCMethodMatch struct { // Value of the method to match against. If left empty or omitted, will // match all services. + // + // At least one of Service and Method must be a non-empty string. // +optional // +kubebuilder:default="" // +kubebuilder:validation:MaxLength=1024 @@ -535,7 +541,7 @@ type GRPCHeaderMatch struct { // entries with an equivalent header name MUST be ignored. Due to the // case-insensitivity of header names, "foo" and "Foo" are considered // equivalent. - Name GRPCHeaderName `json:"name"` + Name HeaderName `json:"name"` // Value is the value of the gRPC Header to be matched. // @@ -550,7 +556,7 @@ type HeaderMatchType string // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=256 // +kubebuilder:validation:Pattern=`^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$` -type GRPCHeaderName string +type HeaderName string // GRPCBackendRef defines how a GRPCRoute should forward a gRPC request. type GRPCBackendRef struct { From 818cc539d0b91881f9097d643f65adfd0c33a105 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 1 Apr 2022 16:17:14 -0700 Subject: [PATCH 23/25] Add comment on custom support matcher case --- site-src/geps/gep-1016.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 3810af32af..58274ea103 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -198,7 +198,7 @@ is provided below: |Specified|Specified|Exact `/${SERVICE}/${METHOD}`| |Specified|Unspecified|Prefix `/${SERVICE}/`| |Unspecified|Specified|Suffix `/${METHOD}/` or Regex `/.+/${METHOD}`| -|Unspecified|Unspecified|Prefix `/`| +|Unspecified|Unspecified|Not allowed| ##### Type RegularExpression @@ -209,6 +209,17 @@ is provided below: |Unspecified|Specified|Regex `/.+/${METHOD}`| |Unspecified|Unspecified|Prefix `/`| +##### Method specified but not Service + +In the table above, `Service` unspecified and `Method` specified with type Exact +is listed as being equivalent to a path matcher with type suffix or type regex. +We imagine that many GRPCRoute implementations will be done using translation to +`HTTPRoute`s. `HTTPRoute` does not support a Suffix matcher and its Regex +matcher is specified as "Custom" support. In order to accommodate `GRPCRoute` +implementations built on top of `HTTPRoute` implementations without regex +support, we list this particular case as having custom support within the +context of `GRPCRoute`. + #### Transport No new `ProtocolType` will be added. While gRPC _does_ have some special From 22eb911214133f1bf75915ba4b08afc5d6c3f7f9 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 4 Apr 2022 13:30:50 -0700 Subject: [PATCH 24/25] Add note about backward compatibility for future features --- site-src/geps/gep-1016.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 58274ea103..50fd60dd04 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -692,7 +692,9 @@ type GRPCRouteFilter struct { Many more ideas have been discussed for the `GRPCRoute` resource, but in the interest of keeping this particular proposal tractable, they have been deferred -for future proposals. +for future proposals. Enough thought has been given to these use cases at the +moment, however, that all of the following may be added at a later date in a +backward-compatible manner. Some of these ideas are: - Ingtegration with Service Meshes (both sidecar-proxied and proxyless) From 7753bcef3450026a6e26f8b93c195958d66a920d Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 6 Apr 2022 16:37:42 -0700 Subject: [PATCH 25/25] Update site-src/geps/gep-1016.md Co-authored-by: Harry --- site-src/geps/gep-1016.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-src/geps/gep-1016.md b/site-src/geps/gep-1016.md index 50fd60dd04..8ed0955ad0 100644 --- a/site-src/geps/gep-1016.md +++ b/site-src/geps/gep-1016.md @@ -478,7 +478,7 @@ type GRPCRouteMatch struct { // At least one of Service and Method must be a non-empty string. type GRPCMethodMatch struct { // Type specifies how to match against the service and/or method. - // Support: Core (Exact) + // Support: Core (Exact with service and method specified) // // Support Custom (Exact with method specified but no service specified) //