diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index 9425333636d..56466649d80 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -271,6 +271,11 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe return routeRoutes, nil } +func processRouteTrafficFeatures(irRoute *ir.HTTPRoute, rule gwapiv1.HTTPRouteRule) { + processRouteTimeout(irRoute, rule) + processRouteRetry(irRoute, rule) +} + func processRouteTimeout(irRoute *ir.HTTPRoute, rule gwapiv1.HTTPRouteRule) { if rule.Timeouts != nil { if rule.Timeouts.Request != nil { @@ -293,6 +298,45 @@ func processRouteTimeout(irRoute *ir.HTTPRoute, rule gwapiv1.HTTPRouteRule) { } } +func processRouteRetry(irRoute *ir.HTTPRoute, rule gwapiv1.HTTPRouteRule) { + if rule.Retry == nil { + return + } + + retry := rule.Retry + res := &ir.Retry{} + if retry.Attempts != nil { + res.NumRetries = ptr.To(uint32(*retry.Attempts)) + } + if retry.Backoff != nil { + backoff, err := time.ParseDuration(string(*retry.Backoff)) + if err == nil { + res.PerRetry = &ir.PerRetryPolicy{ + BackOff: &ir.BackOffPolicy{ + BaseInterval: ptr.To(metav1.Duration{Duration: backoff}), + }, + } + // xref: https://gateway-api.sigs.k8s.io/geps/gep-1742/#timeout-values + if rule.Timeouts != nil && rule.Timeouts.BackendRequest != nil { + backendRequestTimeout, err := time.ParseDuration(string(*rule.Timeouts.BackendRequest)) + if err == nil { + res.PerRetry.Timeout = &metav1.Duration{Duration: backendRequestTimeout} + } + } + } + } + if len(retry.Codes) > 0 { + codes := make([]ir.HTTPStatus, 0, len(retry.Codes)) + for _, code := range retry.Codes { + codes = append(codes, ir.HTTPStatus(code)) + } + res.RetryOn = &ir.RetryOn{ + HTTPStatusCodes: codes, + } + } + irRoute.Retry = res +} + func (t *Translator) processHTTPRouteRule(httpRoute *HTTPRouteContext, ruleIdx int, httpFiltersContext *HTTPFiltersContext, rule gwapiv1.HTTPRouteRule) ([]*ir.HTTPRoute, error) { var ruleRoutes []*ir.HTTPRoute @@ -302,7 +346,7 @@ func (t *Translator) processHTTPRouteRule(httpRoute *HTTPRouteContext, ruleIdx i Name: irRouteName(httpRoute, ruleIdx, -1), } irRoute.Metadata = buildRouteMetadata(httpRoute, rule.Name) - processRouteTimeout(irRoute, rule) + processRouteTrafficFeatures(irRoute, rule) applyHTTPFiltersContextToIRRoute(httpFiltersContext, irRoute) ruleRoutes = append(ruleRoutes, irRoute) } @@ -363,7 +407,7 @@ func (t *Translator) processHTTPRouteRule(httpRoute *HTTPRouteContext, ruleIdx i SessionPersistence: sessionPersistence, } irRoute.Metadata = buildRouteMetadata(httpRoute, rule.Name) - processRouteTimeout(irRoute, rule) + processRouteTrafficFeatures(irRoute, rule) if match.Path != nil { switch PathMatchTypeDerefOr(match.Path.Type, gwapiv1.PathMatchPathPrefix) { @@ -751,11 +795,7 @@ func (t *Translator) processHTTPRouteParentRefListener(route RouteContext, route IsHTTP2: routeRoute.IsHTTP2, SessionPersistence: routeRoute.SessionPersistence, Timeout: routeRoute.Timeout, - } - if routeRoute.Traffic != nil { - hostRoute.Traffic = &ir.TrafficFeatures{ - Retry: routeRoute.Traffic.Retry, - } + Retry: routeRoute.Retry, } perHostRoutes = append(perHostRoutes, hostRoute) } diff --git a/internal/gatewayapi/testdata/httproute-retry.in.yaml b/internal/gatewayapi/testdata/httproute-retry.in.yaml new file mode 100644 index 00000000000..a151e311b50 --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-retry.in.yaml @@ -0,0 +1,100 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-2 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-2 + sectionName: http + rules: + - matches: + - path: + value: "/" + retry: + codes: + - 500 + attempts: 3 + backoff: 200ms + timeouts: + backendRequest: 3s + backendRefs: + - name: service-1 + port: 8080 +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + retry: + retryOn: + triggers: + - cancelled + perRetry: + timeout: 250ms + backoff: + baseInterval: 100ms + maxInterval: 10s +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: default + name: policy-for-route + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + retry: + numRetries: 5 + retryOn: + httpStatusCodes: + - 429 + - 503 + triggers: + - connect-failure + - retriable-status-codes + perRetry: + timeout: 250ms + backoff: + baseInterval: 100ms + maxInterval: 10s diff --git a/internal/gatewayapi/testdata/httproute-retry.out.yaml b/internal/gatewayapi/testdata/httproute-retry.out.yaml new file mode 100644 index 00000000000..3b6b246b51e --- /dev/null +++ b/internal/gatewayapi/testdata/httproute-retry.out.yaml @@ -0,0 +1,318 @@ +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-route + namespace: default + spec: + retry: + numRetries: 5 + perRetry: + backOff: + baseInterval: 100ms + maxInterval: 10s + timeout: 250ms + retryOn: + httpStatusCodes: + - 429 + - 503 + triggers: + - connect-failure + - retriable-status-codes + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-2 + namespace: envoy-gateway + sectionName: http + conditions: + - lastTransitionTime: null + message: Policy has been accepted. + reason: Accepted + status: "True" + type: Accepted + controllerName: gateway.envoyproxy.io/gatewayclass-controller +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + retry: + perRetry: + backOff: + baseInterval: 100ms + maxInterval: 10s + timeout: 250ms + retryOn: + triggers: + - cancelled + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: Policy has been accepted. + reason: Accepted + status: "True" + type: Accepted + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 0 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-2 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-2 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: / + retry: + attempts: 3 + backoff: 200ms + codes: + - 500 + timeouts: + backendRequest: 3s + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-2 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 + envoy-gateway/gateway-2: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-2/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-2 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-2 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + envoy-gateway/gateway-2: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-2 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-2/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + hostname: gateway.envoyproxy.io + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: / + retry: + numRetries: 3 + perRetry: + backOff: + baseInterval: 200ms + timeout: 3s + retryOn: + httpStatusCodes: + - 500 + timeout: 3s + traffic: + retry: + numRetries: 5 + perRetry: + backOff: + baseInterval: 100ms + maxInterval: 10s + timeout: 250ms + retryOn: + httpStatusCodes: + - 429 + - 503 + triggers: + - connect-failure + - retriable-status-codes diff --git a/internal/ir/xds.go b/internal/ir/xds.go index bef7ace6cd9..9aedf8920f1 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -714,6 +714,21 @@ type HTTPRoute struct { SessionPersistence *SessionPersistence `json:"sessionPersistence,omitempty" yaml:"sessionPersistence,omitempty"` // Timeout is the time until which entire response is received from the upstream. Timeout *metav1.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` + // Retry defines the retry policy for the route. + // This is derived from the core Gateway API, and should take precedence over Traffic.Retry. + Retry *Retry `json:"retry,omitempty" yaml:"retry,omitempty"` +} + +func (h *HTTPRoute) GetRetry() *Retry { + if h.Retry != nil { + return h.Retry + } + + if h.Traffic != nil { + return h.Traffic.Retry + } + + return nil } // DNS contains configuration options for DNS resolution. @@ -1196,7 +1211,7 @@ type FaultInjectionAbort struct { } // Validate the fields within the HTTPRoute structure -func (h HTTPRoute) Validate() error { +func (h *HTTPRoute) Validate() error { var errs error if h.Name == "" { errs = errors.Join(errs, ErrRouteNameEmpty) @@ -2567,6 +2582,7 @@ const ( Unavailable = TriggerEnum(egv1a1.Unavailable) ) +// RetryOn specifies the retry policy. // +k8s:deepcopy-gen=true type RetryOn struct { // Triggers specifies the retry trigger condition(Http/Grpc). diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index 6bfef2f3ff9..8fae004f384 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -1656,6 +1656,11 @@ func (in *HTTPRoute) DeepCopyInto(out *HTTPRoute) { *out = new(v1.Duration) **out = **in } + if in.Retry != nil { + in, out := &in.Retry, &out.Retry + *out = new(Retry) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPRoute. diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index 036f202b635..369b5011f91 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -106,8 +106,7 @@ func buildXdsRoute(httpRoute *ir.HTTPRoute) (*routev3.Route, error) { // Retries if router.GetRoute() != nil && - httpRoute.Traffic != nil && - httpRoute.Traffic.Retry != nil { + httpRoute.GetRetry() != nil { if rp, err := buildRetryPolicy(httpRoute); err == nil { router.GetRoute().RetryPolicy = rp } else { @@ -573,8 +572,8 @@ func buildHashPolicy(httpRoute *ir.HTTPRoute) []*routev3.RouteAction_HashPolicy } func buildRetryPolicy(route *ir.HTTPRoute) (*routev3.RetryPolicy, error) { - rr := route.Traffic.Retry - any, err := protocov.ToAnyWithValidation(&previoushost.PreviousHostsPredicate{}) + rr := route.GetRetry() + anyCfg, err := protocov.ToAnyWithValidation(&previoushost.PreviousHostsPredicate{}) if err != nil { return nil, err } @@ -586,7 +585,7 @@ func buildRetryPolicy(route *ir.HTTPRoute) (*routev3.RetryPolicy, error) { { Name: "envoy.retry_host_predicates.previous_hosts", ConfigType: &routev3.RetryPolicy_RetryHostPredicate_TypedConfig{ - TypedConfig: any, + TypedConfig: anyCfg, }, }, }, diff --git a/internal/xds/translator/testdata/in/xds-ir/retry.yaml b/internal/xds/translator/testdata/in/xds-ir/retry.yaml new file mode 100644 index 00000000000..22e862eba47 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/retry.yaml @@ -0,0 +1,52 @@ +http: +- name: "first-listener" + address: "::" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "first-route" + hostname: "*" + retry: + numRetries: 3 + perRetry: + backOff: + baseInterval: 200ms + retryOn: + httpStatusCodes: + - 500 + traffic: + retry: + numRetries: 5 + retryOn: + httpStatusCodes: + - 429 + - 503 + triggers: + - reset + - connect-failure + - retriable-status-codes + perRetry: + timeout: 250ms + backoff: + baseInterval: 100ms + maxInterval: 10s + destination: + name: "first-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + - name: "second-route-defaults" + hostname: "foo" + traffic: + retry: {} + destination: + name: "first-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 diff --git a/internal/xds/translator/testdata/out/xds-ir/retry.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/retry.clusters.yaml new file mode 100644 index 00000000000..06e9a8da524 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/retry.clusters.yaml @@ -0,0 +1,17 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: first-route-dest + ignoreHealthOnHostRemoval: true + lbPolicy: LEAST_REQUEST + name: first-route-dest + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/retry.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/retry.endpoints.yaml new file mode 100644 index 00000000000..3b3f2d09076 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/retry.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: first-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: first-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/retry.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/retry.listeners.yaml new file mode 100644 index 00000000000..80ae84fd104 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/retry.listeners.yaml @@ -0,0 +1,34 @@ +- address: + socketAddress: + address: '::' + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: first-listener + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/retry.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/retry.routes.yaml new file mode 100644 index 00000000000..cd767ce7352 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/retry.routes.yaml @@ -0,0 +1,47 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + prefix: / + name: first-route + route: + cluster: first-route-dest + retryPolicy: + hostSelectionRetryMaxAttempts: "5" + numRetries: 3 + retriableStatusCodes: + - 500 + retryBackOff: + baseInterval: 0.200s + retryHostPredicate: + - name: envoy.retry_host_predicates.previous_hosts + typedConfig: + '@type': type.googleapis.com/envoy.extensions.retry.host.previous_hosts.v3.PreviousHostsPredicate + retryOn: connect-failure,refused-stream,unavailable,cancelled,retriable-status-codes + upgradeConfigs: + - upgradeType: websocket + - domains: + - foo + name: first-listener/foo + routes: + - match: + prefix: / + name: second-route-defaults + route: + cluster: first-route-dest + retryPolicy: + hostSelectionRetryMaxAttempts: "5" + numRetries: 2 + retriableStatusCodes: + - 503 + retryHostPredicate: + - name: envoy.retry_host_predicates.previous_hosts + typedConfig: + '@type': type.googleapis.com/envoy.extensions.retry.host.previous_hosts.v3.PreviousHostsPredicate + retryOn: connect-failure,refused-stream,unavailable,cancelled,retriable-status-codes + upgradeConfigs: + - upgradeType: websocket diff --git a/release-notes/current.yaml b/release-notes/current.yaml index cf755ceb6f4..80451174867 100644 --- a/release-notes/current.yaml +++ b/release-notes/current.yaml @@ -22,6 +22,7 @@ new features: | Added support for preserving the user defined HTTPRoute match order in EnvoyProxy API Added support for specifying dynamic metadata namespaces that External Processing services can access read from and write to in EnvoyExtensionPolicy API Added support for API Key Authentication in the SecurityPolicy API + Added support for GEP-1731 (HTTPRoute Retries) bug fixes: | Fixed a nil pointer error that occurs when a SecurityPolicy refers to a UDS backend