From 601218d718f05bc8714bc4cf5e62e7a62fb759a9 Mon Sep 17 00:00:00 2001 From: Clayton Gonsalves <101868649+clayton-gonsalves@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:43:41 +0200 Subject: [PATCH] Add circuit breaker support for extension services (#6539) Closes #6537. Signed-off-by: Clayton Gonsalves --- apis/projectcontour/v1alpha1/contourconfig.go | 8 +- .../v1alpha1/extensionservice.go | 5 + .../v1alpha1/zz_generated.deepcopy.go | 37 +- .../6539-clayton-gonsalves-minor.md | 6 + cmd/contour/serve.go | 10 +- cmd/contour/serve_test.go | 2 +- cmd/contour/servecontext_test.go | 4 +- examples/contour/01-crds.yaml | 45 +++ examples/render/contour-deployment.yaml | 45 +++ .../render/contour-gateway-provisioner.yaml | 45 +++ examples/render/contour-gateway.yaml | 45 +++ examples/render/contour.yaml | 45 +++ internal/dag/accessors.go | 16 +- internal/dag/builder_test.go | 12 +- internal/dag/dag.go | 47 +-- internal/dag/extension_processor.go | 23 ++ internal/dag/gatewayapi_processor.go | 2 +- internal/dag/httpproxy_processor.go | 2 +- internal/dag/ingress_processor.go | 2 +- internal/dag/policy.go | 22 +- internal/dag/policy_test.go | 68 ++-- internal/envoy/v3/cluster.go | 36 +- internal/envoy/v3/cluster_test.go | 20 +- internal/featuretests/v3/cluster_test.go | 8 +- .../featuretests/v3/extensionservice_test.go | 340 +++++++++++++++++- internal/xdscache/v3/cluster_test.go | 2 +- pkg/config/parameters.go | 2 +- .../docs/main/config/api-reference.html | 190 ++++++---- 28 files changed, 884 insertions(+), 205 deletions(-) create mode 100644 changelogs/unreleased/6539-clayton-gonsalves-minor.md diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go index 05545e30493..0af7a50e08a 100644 --- a/apis/projectcontour/v1alpha1/contourconfig.go +++ b/apis/projectcontour/v1alpha1/contourconfig.go @@ -107,7 +107,7 @@ const ( EnvoyServerType XDSServerType = "envoy" ) -type GlobalCircuitBreakerDefaults struct { +type CircuitBreakers struct { // The maximum number of connections that a single Envoy instance allows to the Kubernetes Service; defaults to 1024. // +optional MaxConnections uint32 `json:"maxConnections,omitempty" yaml:"max-connections,omitempty"` @@ -120,6 +120,10 @@ type GlobalCircuitBreakerDefaults struct { // The maximum number of parallel retries a single Envoy instance allows to the Kubernetes Service; defaults to 3. // +optional MaxRetries uint32 `json:"maxRetries,omitempty" yaml:"max-retries,omitempty"` + + // PerHostMaxConnections is the maximum number of connections + // that Envoy will allow to each individual host in a cluster. + PerHostMaxConnections uint32 `json:"perHostMaxConnections,omitempty" yaml:"per-host-max-connections,omitempty"` } // XDSServerConfig holds the config for the Contour xDS server. @@ -707,7 +711,7 @@ type ClusterParameters struct { // If defined, this will be used as the default for all services. // // +optional - GlobalCircuitBreakerDefaults *GlobalCircuitBreakerDefaults `json:"circuitBreakers,omitempty"` + GlobalCircuitBreakerDefaults *CircuitBreakers `json:"circuitBreakers,omitempty"` // UpstreamTLS contains the TLS policy parameters for upstream connections // diff --git a/apis/projectcontour/v1alpha1/extensionservice.go b/apis/projectcontour/v1alpha1/extensionservice.go index 20c53058b54..c074de546e7 100644 --- a/apis/projectcontour/v1alpha1/extensionservice.go +++ b/apis/projectcontour/v1alpha1/extensionservice.go @@ -104,6 +104,11 @@ type ExtensionServiceSpec struct { // +optional // +kubebuilder:validation:Enum=v3 ProtocolVersion ExtensionProtocolVersion `json:"protocolVersion,omitempty"` + + // CircuitBreakerPolicy specifies the circuit breaker budget across the extension service. + // If defined this overrides the global circuit breaker budget. + // +optional + CircuitBreakerPolicy *CircuitBreakers `json:"circuitBreakerPolicy,omitempty"` } // ExtensionServiceStatus defines the observed state of an diff --git a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go index eaa1576bfce..394c22dff6e 100644 --- a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go +++ b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go @@ -47,6 +47,21 @@ func (in AccessLogJSONFields) DeepCopy() AccessLogJSONFields { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CircuitBreakers) DeepCopyInto(out *CircuitBreakers) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CircuitBreakers. +func (in *CircuitBreakers) DeepCopy() *CircuitBreakers { + if in == nil { + return nil + } + out := new(CircuitBreakers) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterParameters) DeepCopyInto(out *ClusterParameters) { *out = *in @@ -62,7 +77,7 @@ func (in *ClusterParameters) DeepCopyInto(out *ClusterParameters) { } if in.GlobalCircuitBreakerDefaults != nil { in, out := &in.GlobalCircuitBreakerDefaults, &out.GlobalCircuitBreakerDefaults - *out = new(GlobalCircuitBreakerDefaults) + *out = new(CircuitBreakers) **out = **in } if in.UpstreamTLS != nil { @@ -813,6 +828,11 @@ func (in *ExtensionServiceSpec) DeepCopyInto(out *ExtensionServiceSpec) { *out = new(v1.TimeoutPolicy) **out = **in } + if in.CircuitBreakerPolicy != nil { + in, out := &in.CircuitBreakerPolicy, &out.CircuitBreakerPolicy + *out = new(CircuitBreakers) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtensionServiceSpec. @@ -897,21 +917,6 @@ func (in *GatewayConfig) DeepCopy() *GatewayConfig { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GlobalCircuitBreakerDefaults) DeepCopyInto(out *GlobalCircuitBreakerDefaults) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalCircuitBreakerDefaults. -func (in *GlobalCircuitBreakerDefaults) DeepCopy() *GlobalCircuitBreakerDefaults { - if in == nil { - return nil - } - out := new(GlobalCircuitBreakerDefaults) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPProxyConfig) DeepCopyInto(out *HTTPProxyConfig) { *out = *in diff --git a/changelogs/unreleased/6539-clayton-gonsalves-minor.md b/changelogs/unreleased/6539-clayton-gonsalves-minor.md new file mode 100644 index 00000000000..6601ec2203d --- /dev/null +++ b/changelogs/unreleased/6539-clayton-gonsalves-minor.md @@ -0,0 +1,6 @@ +## Add Circuit Breaker support for Extension Services + +This change enables the user to configure the Circuit breakers for extension services either via the global Contour config or on an individual Extension Service. + +**NOTE**: The `PerHostMaxConnections` is now also configurable via the global settings. + diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go index 375edb0033b..1c48f9fce88 100644 --- a/cmd/contour/serve.go +++ b/cmd/contour/serve.go @@ -1068,7 +1068,7 @@ type dagBuilderConfig struct { maxRequestsPerConnection *uint32 perConnectionBufferLimitBytes *uint32 globalRateLimitService *contour_v1alpha1.RateLimitServiceConfig - globalCircuitBreakerDefaults *contour_v1alpha1.GlobalCircuitBreakerDefaults + globalCircuitBreakerDefaults *contour_v1alpha1.CircuitBreakers upstreamTLS *dag.UpstreamTLS } @@ -1145,10 +1145,10 @@ func (s *Server) getDAGBuilder(dbc dagBuilderConfig) *dag.Builder { &dag.ExtensionServiceProcessor{ // Note that ExtensionService does not support ExternalName, if it does get added, // need to bring EnableExternalNameService in here too. - FieldLogger: s.log.WithField("context", "ExtensionServiceProcessor"), - ClientCertificate: dbc.clientCert, - ConnectTimeout: dbc.connectTimeout, - UpstreamTLS: dbc.upstreamTLS, + FieldLogger: s.log.WithField("context", "ExtensionServiceProcessor"), + ClientCertificate: dbc.clientCert, + ConnectTimeout: dbc.connectTimeout, + GlobalCircuitBreakerDefaults: dbc.globalCircuitBreakerDefaults, }, &dag.HTTPProxyProcessor{ EnableExternalNameService: dbc.enableExternalNameService, diff --git a/cmd/contour/serve_test.go b/cmd/contour/serve_test.go index 248e9bc17c4..f7846e5dffb 100644 --- a/cmd/contour/serve_test.go +++ b/cmd/contour/serve_test.go @@ -125,7 +125,7 @@ func TestGetDAGBuilder(t *testing.T) { }) t.Run("GlobalCircuitBreakerDefaults specified for all processors", func(t *testing.T) { - g := contour_v1alpha1.GlobalCircuitBreakerDefaults{ + g := contour_v1alpha1.CircuitBreakers{ MaxConnections: 100, } diff --git a/cmd/contour/servecontext_test.go b/cmd/contour/servecontext_test.go index 766a2f0af83..cf205d81a46 100644 --- a/cmd/contour/servecontext_test.go +++ b/cmd/contour/servecontext_test.go @@ -767,7 +767,7 @@ func TestConvertServeContext(t *testing.T) { }, "global circuit breaker defaults": { getServeContext: func(ctx *serveContext) *serveContext { - ctx.Config.Cluster.GlobalCircuitBreakerDefaults = &contour_v1alpha1.GlobalCircuitBreakerDefaults{ + ctx.Config.Cluster.GlobalCircuitBreakerDefaults = &contour_v1alpha1.CircuitBreakers{ MaxConnections: 4, MaxPendingRequests: 5, MaxRequests: 6, @@ -776,7 +776,7 @@ func TestConvertServeContext(t *testing.T) { return ctx }, getContourConfiguration: func(cfg contour_v1alpha1.ContourConfigurationSpec) contour_v1alpha1.ContourConfigurationSpec { - cfg.Envoy.Cluster.GlobalCircuitBreakerDefaults = &contour_v1alpha1.GlobalCircuitBreakerDefaults{ + cfg.Envoy.Cluster.GlobalCircuitBreakerDefaults = &contour_v1alpha1.CircuitBreakers{ MaxConnections: 4, MaxPendingRequests: 5, MaxRequests: 6, diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml index bef93fe89c9..0beece5bc51 100644 --- a/examples/contour/01-crds.yaml +++ b/examples/contour/01-crds.yaml @@ -120,6 +120,12 @@ spec: defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -3896,6 +3902,12 @@ spec: Service; defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -5061,6 +5073,39 @@ spec: description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. properties: + circuitBreakerPolicy: + description: |- + CircuitBreakerPolicy specifies the circuit breaker budget across the extension service. + If defined this overrides the global circuit breaker budget. + properties: + maxConnections: + description: The maximum number of connections that a single Envoy + instance allows to the Kubernetes Service; defaults to 1024. + format: int32 + type: integer + maxPendingRequests: + description: The maximum number of pending requests that a single + Envoy instance allows to the Kubernetes Service; defaults to + 1024. + format: int32 + type: integer + maxRequests: + description: The maximum parallel requests a single Envoy instance + allows to the Kubernetes Service; defaults to 1024 + format: int32 + type: integer + maxRetries: + description: The maximum number of parallel retries a single Envoy + instance allows to the Kubernetes Service; defaults to 3. + format: int32 + type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer + type: object loadBalancerPolicy: description: |- The policy for load balancing GRPC service requests. Note that the diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml index edfe5756ea6..1e085adae27 100644 --- a/examples/render/contour-deployment.yaml +++ b/examples/render/contour-deployment.yaml @@ -340,6 +340,12 @@ spec: defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -4116,6 +4122,12 @@ spec: Service; defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -5281,6 +5293,39 @@ spec: description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. properties: + circuitBreakerPolicy: + description: |- + CircuitBreakerPolicy specifies the circuit breaker budget across the extension service. + If defined this overrides the global circuit breaker budget. + properties: + maxConnections: + description: The maximum number of connections that a single Envoy + instance allows to the Kubernetes Service; defaults to 1024. + format: int32 + type: integer + maxPendingRequests: + description: The maximum number of pending requests that a single + Envoy instance allows to the Kubernetes Service; defaults to + 1024. + format: int32 + type: integer + maxRequests: + description: The maximum parallel requests a single Envoy instance + allows to the Kubernetes Service; defaults to 1024 + format: int32 + type: integer + maxRetries: + description: The maximum number of parallel retries a single Envoy + instance allows to the Kubernetes Service; defaults to 3. + format: int32 + type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer + type: object loadBalancerPolicy: description: |- The policy for load balancing GRPC service requests. Note that the diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml index 327714592e0..b3633a2e0cd 100644 --- a/examples/render/contour-gateway-provisioner.yaml +++ b/examples/render/contour-gateway-provisioner.yaml @@ -131,6 +131,12 @@ spec: defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -3907,6 +3913,12 @@ spec: Service; defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -5072,6 +5084,39 @@ spec: description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. properties: + circuitBreakerPolicy: + description: |- + CircuitBreakerPolicy specifies the circuit breaker budget across the extension service. + If defined this overrides the global circuit breaker budget. + properties: + maxConnections: + description: The maximum number of connections that a single Envoy + instance allows to the Kubernetes Service; defaults to 1024. + format: int32 + type: integer + maxPendingRequests: + description: The maximum number of pending requests that a single + Envoy instance allows to the Kubernetes Service; defaults to + 1024. + format: int32 + type: integer + maxRequests: + description: The maximum parallel requests a single Envoy instance + allows to the Kubernetes Service; defaults to 1024 + format: int32 + type: integer + maxRetries: + description: The maximum number of parallel retries a single Envoy + instance allows to the Kubernetes Service; defaults to 3. + format: int32 + type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer + type: object loadBalancerPolicy: description: |- The policy for load balancing GRPC service requests. Note that the diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml index 067df730e44..0db31e989d5 100644 --- a/examples/render/contour-gateway.yaml +++ b/examples/render/contour-gateway.yaml @@ -156,6 +156,12 @@ spec: defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -3932,6 +3938,12 @@ spec: Service; defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -5097,6 +5109,39 @@ spec: description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. properties: + circuitBreakerPolicy: + description: |- + CircuitBreakerPolicy specifies the circuit breaker budget across the extension service. + If defined this overrides the global circuit breaker budget. + properties: + maxConnections: + description: The maximum number of connections that a single Envoy + instance allows to the Kubernetes Service; defaults to 1024. + format: int32 + type: integer + maxPendingRequests: + description: The maximum number of pending requests that a single + Envoy instance allows to the Kubernetes Service; defaults to + 1024. + format: int32 + type: integer + maxRequests: + description: The maximum parallel requests a single Envoy instance + allows to the Kubernetes Service; defaults to 1024 + format: int32 + type: integer + maxRetries: + description: The maximum number of parallel retries a single Envoy + instance allows to the Kubernetes Service; defaults to 3. + format: int32 + type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer + type: object loadBalancerPolicy: description: |- The policy for load balancing GRPC service requests. Note that the diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml index 49cd4749a20..d78bac68c7b 100644 --- a/examples/render/contour.yaml +++ b/examples/render/contour.yaml @@ -340,6 +340,12 @@ spec: defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -4116,6 +4122,12 @@ spec: Service; defaults to 3. format: int32 type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer type: object dnsLookupFamily: description: |- @@ -5281,6 +5293,39 @@ spec: description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. properties: + circuitBreakerPolicy: + description: |- + CircuitBreakerPolicy specifies the circuit breaker budget across the extension service. + If defined this overrides the global circuit breaker budget. + properties: + maxConnections: + description: The maximum number of connections that a single Envoy + instance allows to the Kubernetes Service; defaults to 1024. + format: int32 + type: integer + maxPendingRequests: + description: The maximum number of pending requests that a single + Envoy instance allows to the Kubernetes Service; defaults to + 1024. + format: int32 + type: integer + maxRequests: + description: The maximum parallel requests a single Envoy instance + allows to the Kubernetes Service; defaults to 1024 + format: int32 + type: integer + maxRetries: + description: The maximum number of parallel retries a single Envoy + instance allows to the Kubernetes Service; defaults to 3. + format: int32 + type: integer + perHostMaxConnections: + description: |- + PerHostMaxConnections is the maximum number of connections + that Envoy will allow to each individual host in a cluster. + format: int32 + type: integer + type: object loadBalancerPolicy: description: |- The policy for load balancing GRPC service requests. Note that the diff --git a/internal/dag/accessors.go b/internal/dag/accessors.go index 20a990b3433..c039d28ccba 100644 --- a/internal/dag/accessors.go +++ b/internal/dag/accessors.go @@ -63,13 +63,15 @@ func (d *DAG) EnsureService(meta types.NamespacedName, port, healthPort int, cac HealthPort: healthSvcPort, Weight: 1, }, - Protocol: upstreamProtocol(svc, svcPort), - MaxConnections: annotation.MaxConnections(svc), - MaxPendingRequests: annotation.MaxPendingRequests(svc), - MaxRequests: annotation.MaxRequests(svc), - MaxRetries: annotation.MaxRetries(svc), - PerHostMaxConnections: annotation.PerHostMaxConnections(svc), - ExternalName: externalName(svc), + Protocol: upstreamProtocol(svc, svcPort), + CircuitBreakers: CircuitBreakers{ + MaxConnections: annotation.MaxConnections(svc), + MaxPendingRequests: annotation.MaxPendingRequests(svc), + MaxRequests: annotation.MaxRequests(svc), + PerHostMaxConnections: annotation.PerHostMaxConnections(svc), + MaxRetries: annotation.MaxRetries(svc), + }, + ExternalName: externalName(svc), }, nil } diff --git a/internal/dag/builder_test.go b/internal/dag/builder_test.go index 98f4525b890..70f9ee3ed0f 100644 --- a/internal/dag/builder_test.go +++ b/internal/dag/builder_test.go @@ -10640,11 +10640,13 @@ func TestDAGInsert(t *testing.T) { ServicePort: s1b.Spec.Ports[0], HealthPort: s1b.Spec.Ports[0], }, - MaxConnections: 9000, - MaxPendingRequests: 4096, - MaxRequests: 404, - MaxRetries: 7, - PerHostMaxConnections: 45, + CircuitBreakers: CircuitBreakers{ + MaxConnections: 9000, + MaxPendingRequests: 4096, + MaxRequests: 404, + MaxRetries: 7, + PerHostMaxConnections: 45, + }, }), ), ), diff --git a/internal/dag/dag.go b/internal/dag/dag.go index 9c67318e0c8..8ec08fea717 100644 --- a/internal/dag/dag.go +++ b/internal/dag/dag.go @@ -971,26 +971,7 @@ type Service struct { Protocol string // Circuit breaking limits - - // Max connections is maximum number of connections - // that Envoy will make to the upstream cluster. - MaxConnections uint32 - - // MaxPendingRequests is maximum number of pending - // requests that Envoy will allow to the upstream cluster. - MaxPendingRequests uint32 - - // MaxRequests is the maximum number of parallel requests that - // Envoy will make to the upstream cluster. - MaxRequests uint32 - - // MaxRetries is the maximum number of parallel retries that - // Envoy will allow to the upstream cluster. - MaxRetries uint32 - - // PerHostMaxConnections is the maximum number of connections - // that Envoy will allow to each individual host in a cluster. - PerHostMaxConnections uint32 + CircuitBreakers CircuitBreakers // ExternalName is an optional field referencing a dns entry for Service type "ExternalName" ExternalName string @@ -1261,6 +1242,9 @@ type ExtensionCluster struct { // UpstreamTLS contains the TLS version and cipher suite configurations for upstream connections UpstreamTLS *UpstreamTLS + + // Circuit breaking limits + CircuitBreakers CircuitBreakers } const singleDNSLabelWildcardRegex = "^[a-z0-9]([-a-z0-9]*[a-z0-9])?" @@ -1299,3 +1283,26 @@ type UpstreamTLS struct { MaximumProtocolVersion string CipherSuites []string } + +// CircuitBreakers holds configuration for circuit breakers. +type CircuitBreakers struct { + // Max connections is maximum number of connections + // that Envoy will make to the upstream cluster. + MaxConnections uint32 + + // MaxPendingRequests is maximum number of pending + // requests that Envoy will allow to the upstream cluster. + MaxPendingRequests uint32 + + // MaxRequests is the maximum number of parallel requests that + // Envoy will make to the upstream cluster. + MaxRequests uint32 + + // MaxRetries is the maximum number of parallel retries that + // Envoy will allow to the upstream cluster. + MaxRetries uint32 + + // PerHostMaxConnections is the maximum number of connections + // that Envoy will allow to each individual host in a cluster. + PerHostMaxConnections uint32 +} diff --git a/internal/dag/extension_processor.go b/internal/dag/extension_processor.go index d954f89d300..8d03f60a360 100644 --- a/internal/dag/extension_processor.go +++ b/internal/dag/extension_processor.go @@ -44,6 +44,9 @@ type ExtensionServiceProcessor struct { // UpstreamTLS defines the TLS settings like min/max version // and cipher suites for upstream connections. UpstreamTLS *UpstreamTLS + + // GlobalCircuitBreakerDefaults defines global circuit breaker defaults. + GlobalCircuitBreakerDefaults *contour_v1alpha1.CircuitBreakers } var _ Processor = &ExtensionServiceProcessor{} @@ -122,6 +125,26 @@ func (p *ExtensionServiceProcessor) buildExtensionService( UpstreamTLS: p.UpstreamTLS, } + if p.GlobalCircuitBreakerDefaults != nil { + extension.CircuitBreakers = CircuitBreakers{ + MaxConnections: p.GlobalCircuitBreakerDefaults.MaxConnections, + MaxPendingRequests: p.GlobalCircuitBreakerDefaults.MaxPendingRequests, + MaxRequests: p.GlobalCircuitBreakerDefaults.MaxRequests, + MaxRetries: p.GlobalCircuitBreakerDefaults.MaxRetries, + PerHostMaxConnections: p.GlobalCircuitBreakerDefaults.PerHostMaxConnections, + } + } + + if ext.Spec.CircuitBreakerPolicy != nil { + extension.CircuitBreakers = CircuitBreakers{ + MaxConnections: ext.Spec.CircuitBreakerPolicy.MaxConnections, + MaxPendingRequests: ext.Spec.CircuitBreakerPolicy.MaxPendingRequests, + MaxRequests: ext.Spec.CircuitBreakerPolicy.MaxRequests, + MaxRetries: ext.Spec.CircuitBreakerPolicy.MaxRetries, + PerHostMaxConnections: ext.Spec.CircuitBreakerPolicy.PerHostMaxConnections, + } + } + lbPolicy := loadBalancerPolicy(ext.Spec.LoadBalancerPolicy) switch lbPolicy { case LoadBalancerPolicyCookie, LoadBalancerPolicyRequestHash: diff --git a/internal/dag/gatewayapi_processor.go b/internal/dag/gatewayapi_processor.go index 638ded0a939..5dd9c1deb6e 100644 --- a/internal/dag/gatewayapi_processor.go +++ b/internal/dag/gatewayapi_processor.go @@ -76,7 +76,7 @@ type GatewayAPIProcessor struct { SetSourceMetadataOnRoutes bool // GlobalCircuitBreakerDefaults defines global circuit breaker defaults. - GlobalCircuitBreakerDefaults *contour_v1alpha1.GlobalCircuitBreakerDefaults + GlobalCircuitBreakerDefaults *contour_v1alpha1.CircuitBreakers // UpstreamTLS defines the TLS settings like min/max version // and cipher suites for upstream connections. diff --git a/internal/dag/httpproxy_processor.go b/internal/dag/httpproxy_processor.go index 7dac1510315..083fd741353 100644 --- a/internal/dag/httpproxy_processor.go +++ b/internal/dag/httpproxy_processor.go @@ -114,7 +114,7 @@ type HTTPProxyProcessor struct { SetSourceMetadataOnRoutes bool // GlobalCircuitBreakerDefaults defines global circuit breaker defaults. - GlobalCircuitBreakerDefaults *contour_v1alpha1.GlobalCircuitBreakerDefaults + GlobalCircuitBreakerDefaults *contour_v1alpha1.CircuitBreakers // UpstreamTLS defines the TLS settings like min/max version // and cipher suites for upstream connections. diff --git a/internal/dag/ingress_processor.go b/internal/dag/ingress_processor.go index d5dd05e5bed..63639e22712 100644 --- a/internal/dag/ingress_processor.go +++ b/internal/dag/ingress_processor.go @@ -68,7 +68,7 @@ type IngressProcessor struct { SetSourceMetadataOnRoutes bool // GlobalCircuitBreakerDefaults defines global circuit breaker defaults. - GlobalCircuitBreakerDefaults *contour_v1alpha1.GlobalCircuitBreakerDefaults + GlobalCircuitBreakerDefaults *contour_v1alpha1.CircuitBreakers // UpstreamTLS defines the TLS settings like min/max version // and cipher suites for upstream connections. diff --git a/internal/dag/policy.go b/internal/dag/policy.go index 9360d641c1e..720d959e534 100644 --- a/internal/dag/policy.go +++ b/internal/dag/policy.go @@ -808,25 +808,29 @@ func loadBalancerRequestHashPolicies(lbp *contour_v1.LoadBalancerPolicy, validCo } } -func serviceCircuitBreakerPolicy(s *Service, cb *contour_v1alpha1.GlobalCircuitBreakerDefaults) *Service { +func serviceCircuitBreakerPolicy(s *Service, cb *contour_v1alpha1.CircuitBreakers) *Service { if s == nil { return nil } - if s.MaxConnections == 0 && cb != nil { - s.MaxConnections = cb.MaxConnections + if s.CircuitBreakers.MaxConnections == 0 && cb != nil { + s.CircuitBreakers.MaxConnections = cb.MaxConnections } - if s.MaxPendingRequests == 0 && cb != nil { - s.MaxPendingRequests = cb.MaxPendingRequests + if s.CircuitBreakers.MaxPendingRequests == 0 && cb != nil { + s.CircuitBreakers.MaxPendingRequests = cb.MaxPendingRequests } - if s.MaxRequests == 0 && cb != nil { - s.MaxRequests = cb.MaxRequests + if s.CircuitBreakers.MaxRequests == 0 && cb != nil { + s.CircuitBreakers.MaxRequests = cb.MaxRequests } - if s.MaxRetries == 0 && cb != nil { - s.MaxRetries = cb.MaxRetries + if s.CircuitBreakers.MaxRetries == 0 && cb != nil { + s.CircuitBreakers.MaxRetries = cb.MaxRetries + } + + if s.CircuitBreakers.PerHostMaxConnections == 0 && cb != nil { + s.CircuitBreakers.PerHostMaxConnections = cb.PerHostMaxConnections } return s diff --git a/internal/dag/policy_test.go b/internal/dag/policy_test.go index 62301bbb1d6..ab006db9b57 100644 --- a/internal/dag/policy_test.go +++ b/internal/dag/policy_test.go @@ -1276,7 +1276,7 @@ func TestValidateHeaderAlteration(t *testing.T) { func TestServiceCircuitBreakerPolicy(t *testing.T) { tests := map[string]struct { in *Service - globalDefault *contour_v1alpha1.GlobalCircuitBreakerDefaults + globalDefault *contour_v1alpha1.CircuitBreakers want *Service }{ "service is nil and globalDefault is nil": { @@ -1286,51 +1286,65 @@ func TestServiceCircuitBreakerPolicy(t *testing.T) { }, "service is nil and globalDefault is not nil": { in: nil, - globalDefault: &contour_v1alpha1.GlobalCircuitBreakerDefaults{}, + globalDefault: &contour_v1alpha1.CircuitBreakers{}, want: nil, }, "service is not nil and globalDefault is nil": { in: &Service{ - MaxConnections: 42, - MaxPendingRequests: 73, - MaxRequests: 89, - MaxRetries: 13, + CircuitBreakers: CircuitBreakers{ + MaxConnections: 42, + MaxPendingRequests: 73, + MaxRequests: 89, + MaxRetries: 13, + PerHostMaxConnections: 23, + }, }, globalDefault: nil, want: &Service{ - MaxConnections: 42, - MaxPendingRequests: 73, - MaxRequests: 89, - MaxRetries: 13, + CircuitBreakers: CircuitBreakers{ + MaxConnections: 42, + MaxPendingRequests: 73, + MaxRequests: 89, + MaxRetries: 13, + PerHostMaxConnections: 23, + }, }, }, "service is not set but global is": { in: &Service{}, - globalDefault: &contour_v1alpha1.GlobalCircuitBreakerDefaults{ - MaxConnections: 42, - MaxPendingRequests: 73, - MaxRequests: 89, - MaxRetries: 13, + globalDefault: &contour_v1alpha1.CircuitBreakers{ + MaxConnections: 42, + MaxPendingRequests: 73, + MaxRequests: 89, + MaxRetries: 13, + PerHostMaxConnections: 23, }, want: &Service{ - MaxConnections: 42, - MaxPendingRequests: 73, - MaxRequests: 89, - MaxRetries: 13, + CircuitBreakers: CircuitBreakers{ + MaxConnections: 42, + MaxPendingRequests: 73, + MaxRequests: 89, + MaxRetries: 13, + PerHostMaxConnections: 23, + }, }, }, "service is not set but global is partial": { in: &Service{}, - globalDefault: &contour_v1alpha1.GlobalCircuitBreakerDefaults{ - MaxConnections: 42, - MaxPendingRequests: 73, - MaxRequests: 89, + globalDefault: &contour_v1alpha1.CircuitBreakers{ + MaxConnections: 42, + MaxPendingRequests: 73, + MaxRequests: 89, + PerHostMaxConnections: 23, }, want: &Service{ - MaxConnections: 42, - MaxPendingRequests: 73, - MaxRequests: 89, - MaxRetries: 0, + CircuitBreakers: CircuitBreakers{ + MaxConnections: 42, + MaxPendingRequests: 73, + MaxRequests: 89, + MaxRetries: 0, + PerHostMaxConnections: 23, + }, }, }, } diff --git a/internal/envoy/v3/cluster.go b/internal/envoy/v3/cluster.go index 40fe15afd52..24346c48421 100644 --- a/internal/envoy/v3/cluster.go +++ b/internal/envoy/v3/cluster.go @@ -79,21 +79,7 @@ func Cluster(c *dag.Cluster) *envoy_config_cluster_v3.Cluster { cluster.IgnoreHealthOnHostRemoval = true } - if envoy.AnyPositive(service.MaxConnections, service.MaxPendingRequests, service.MaxRequests, service.MaxRetries, service.PerHostMaxConnections) { - cluster.CircuitBreakers = &envoy_config_cluster_v3.CircuitBreakers{ - Thresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ - MaxConnections: protobuf.UInt32OrNil(service.MaxConnections), - MaxPendingRequests: protobuf.UInt32OrNil(service.MaxPendingRequests), - MaxRequests: protobuf.UInt32OrNil(service.MaxRequests), - MaxRetries: protobuf.UInt32OrNil(service.MaxRetries), - TrackRemaining: true, - }}, - PerHostThresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ - MaxConnections: protobuf.UInt32OrNil(service.PerHostMaxConnections), - TrackRemaining: true, - }}, - } - } + applyCircuitBreakers(cluster, service.CircuitBreakers) httpVersion := HTTPVersionAuto switch c.Protocol { @@ -198,9 +184,29 @@ func ExtensionCluster(ext *dag.ExtensionCluster) *envoy_config_cluster_v3.Cluste } cluster.TypedExtensionProtocolOptions = protocolOptions(http2Version, ext.ClusterTimeoutPolicy.IdleConnectionTimeout, nil) + applyCircuitBreakers(cluster, ext.CircuitBreakers) + return cluster } +func applyCircuitBreakers(cluster *envoy_config_cluster_v3.Cluster, settings dag.CircuitBreakers) { + if envoy.AnyPositive(settings.MaxConnections, settings.MaxPendingRequests, settings.MaxRequests, settings.MaxRetries, settings.PerHostMaxConnections) { + cluster.CircuitBreakers = &envoy_config_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: protobuf.UInt32OrNil(settings.MaxConnections), + MaxPendingRequests: protobuf.UInt32OrNil(settings.MaxPendingRequests), + MaxRequests: protobuf.UInt32OrNil(settings.MaxRequests), + MaxRetries: protobuf.UInt32OrNil(settings.MaxRetries), + TrackRemaining: true, + }}, + PerHostThresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: protobuf.UInt32OrNil(settings.PerHostMaxConnections), + TrackRemaining: true, + }}, + } + } +} + // DNSNameCluster builds a envoy_config_cluster_v3.Cluster for the given *dag.DNSNameCluster. func DNSNameCluster(c *dag.DNSNameCluster) *envoy_config_cluster_v3.Cluster { cluster := clusterDefaults() diff --git a/internal/envoy/v3/cluster_test.go b/internal/envoy/v3/cluster_test.go index ff6bf79a012..cc9f64bcec5 100644 --- a/internal/envoy/v3/cluster_test.go +++ b/internal/envoy/v3/cluster_test.go @@ -406,7 +406,9 @@ func TestCluster(t *testing.T) { "projectcontour.io/max-connections": { cluster: &dag.Cluster{ Upstream: &dag.Service{ - MaxConnections: 9000, + CircuitBreakers: dag.CircuitBreakers{ + MaxConnections: 9000, + }, Weighted: dag.WeightedService{ Weight: 1, ServiceName: s1.Name, @@ -438,7 +440,9 @@ func TestCluster(t *testing.T) { "projectcontour.io/max-pending-requests": { cluster: &dag.Cluster{ Upstream: &dag.Service{ - MaxPendingRequests: 4096, + CircuitBreakers: dag.CircuitBreakers{ + MaxPendingRequests: 4096, + }, Weighted: dag.WeightedService{ Weight: 1, ServiceName: s1.Name, @@ -470,7 +474,9 @@ func TestCluster(t *testing.T) { "projectcontour.io/max-requests": { cluster: &dag.Cluster{ Upstream: &dag.Service{ - MaxRequests: 404, + CircuitBreakers: dag.CircuitBreakers{ + MaxRequests: 404, + }, Weighted: dag.WeightedService{ Weight: 1, ServiceName: s1.Name, @@ -502,7 +508,9 @@ func TestCluster(t *testing.T) { "projectcontour.io/max-retries": { cluster: &dag.Cluster{ Upstream: &dag.Service{ - MaxRetries: 7, + CircuitBreakers: dag.CircuitBreakers{ + MaxRetries: 7, + }, Weighted: dag.WeightedService{ Weight: 1, ServiceName: s1.Name, @@ -534,7 +542,9 @@ func TestCluster(t *testing.T) { "projectcontour.io/per-host-max-connections": { cluster: &dag.Cluster{ Upstream: &dag.Service{ - PerHostMaxConnections: 45, + CircuitBreakers: dag.CircuitBreakers{ + PerHostMaxConnections: 45, + }, Weighted: dag.WeightedService{ Weight: 1, ServiceName: s1.Name, diff --git a/internal/featuretests/v3/cluster_test.go b/internal/featuretests/v3/cluster_test.go index ff4eb77c07d..2680585ba4b 100644 --- a/internal/featuretests/v3/cluster_test.go +++ b/internal/featuretests/v3/cluster_test.go @@ -391,7 +391,7 @@ func TestCDSResourceFiltering(t *testing.T) { }) } -func circuitBreakerGlobalOpt(t *testing.T, g *contour_v1alpha1.GlobalCircuitBreakerDefaults) func(*dag.Builder) { +func circuitBreakerGlobalOpt(t *testing.T, g *contour_v1alpha1.CircuitBreakers) func(*dag.Builder) { return func(b *dag.Builder) { log := fixture.NewTestLogger(t) log.SetLevel(logrus.DebugLevel) @@ -414,7 +414,7 @@ func circuitBreakerGlobalOpt(t *testing.T, g *contour_v1alpha1.GlobalCircuitBrea } func TestClusterCircuitbreakerAnnotationsIngress(t *testing.T) { - g := &contour_v1alpha1.GlobalCircuitBreakerDefaults{ + g := &contour_v1alpha1.CircuitBreakers{ MaxConnections: 13, MaxPendingRequests: 14, MaxRequests: 15, @@ -549,7 +549,7 @@ func TestClusterCircuitbreakerAnnotationsIngress(t *testing.T) { } func TestClusterCircuitbreakerAnnotationsHTTPProxy(t *testing.T) { - g := &contour_v1alpha1.GlobalCircuitBreakerDefaults{ + g := &contour_v1alpha1.CircuitBreakers{ MaxConnections: 13, MaxPendingRequests: 14, MaxRequests: 15, @@ -692,7 +692,7 @@ func TestClusterCircuitbreakerAnnotationsHTTPProxy(t *testing.T) { } func TestClusterCircuitbreakerAnnotationsGateway(t *testing.T) { - g := &contour_v1alpha1.GlobalCircuitBreakerDefaults{ + g := &contour_v1alpha1.CircuitBreakers{ MaxConnections: 13, MaxPendingRequests: 14, MaxRequests: 15, diff --git a/internal/featuretests/v3/extensionservice_test.go b/internal/featuretests/v3/extensionservice_test.go index 93314db9505..b341882e4d7 100644 --- a/internal/featuretests/v3/extensionservice_test.go +++ b/internal/featuretests/v3/extensionservice_test.go @@ -23,11 +23,13 @@ import ( envoy_transport_socket_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" envoy_service_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" envoy_matcher_v3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "google.golang.org/protobuf/types/known/wrapperspb" core_v1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" contour_v1 "github.com/projectcontour/contour/apis/projectcontour/v1" contour_v1alpha1 "github.com/projectcontour/contour/apis/projectcontour/v1alpha1" + "github.com/projectcontour/contour/internal/dag" envoy_v3 "github.com/projectcontour/contour/internal/envoy/v3" "github.com/projectcontour/contour/internal/featuretests" "github.com/projectcontour/contour/internal/fixture" @@ -402,23 +404,341 @@ func extInvalidLoadBalancerPolicy(t *testing.T, rh ResourceEventHandlerWrapper, }) } +func extCircuitBreakers(t *testing.T, rh ResourceEventHandlerWrapper, c *Contour) { + ext := &contour_v1alpha1.ExtensionService{ + ObjectMeta: fixture.ObjectMeta("ns/ext"), + Spec: contour_v1alpha1.ExtensionServiceSpec{ + Services: []contour_v1alpha1.ExtensionServiceTarget{ + {Name: "svc1", Port: 8081}, + {Name: "svc2", Port: 8082}, + }, + LoadBalancerPolicy: &contour_v1.LoadBalancerPolicy{ + Strategy: "Cookie", + }, + CircuitBreakerPolicy: &contour_v1alpha1.CircuitBreakers{ + MaxConnections: 10000, + MaxPendingRequests: 1048, + MaxRequests: 494, + MaxRetries: 10, + PerHostMaxConnections: 1, + }, + }, + } + + rh.OnAdd(ext) + + c.Request(clusterType).Equals(&envoy_service_discovery_v3.DiscoveryResponse{ + TypeUrl: clusterType, + Resources: resources(t, + DefaultCluster( + // Default load balancer policy should be set as we were passed + // an invalid value, we can assert we get a basic cluster. + h2cCluster(cluster("extension/ns/ext", "extension/ns/ext", "extension_ns_ext")), + &envoy_config_cluster_v3.Cluster{ + TransportSocket: envoy_v3.UpstreamTLSTransportSocket( + &envoy_transport_socket_tls_v3.UpstreamTlsContext{ + CommonTlsContext: &envoy_transport_socket_tls_v3.CommonTlsContext{ + AlpnProtocols: []string{"h2"}, + }, + }, + ), + CircuitBreakers: &envoy_config_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(10000), + MaxPendingRequests: wrapperspb.UInt32(1048), + MaxRequests: wrapperspb.UInt32(494), + MaxRetries: wrapperspb.UInt32(10), + TrackRemaining: true, + }}, + PerHostThresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(1), + TrackRemaining: true, + }}, + }, + }, + ), + ), + }) + + rh.OnUpdate(ext, &contour_v1alpha1.ExtensionService{ + ObjectMeta: fixture.ObjectMeta("ns/ext"), + Spec: contour_v1alpha1.ExtensionServiceSpec{ + Services: []contour_v1alpha1.ExtensionServiceTarget{ + {Name: "svc1", Port: 8081}, + {Name: "svc2", Port: 8082}, + }, + LoadBalancerPolicy: &contour_v1.LoadBalancerPolicy{ + Strategy: "RequestHash", + }, + }, + }) + + c.Request(clusterType).Equals(&envoy_service_discovery_v3.DiscoveryResponse{ + TypeUrl: clusterType, + Resources: resources(t, + DefaultCluster( + // Default load balancer policy should be set as we were passed + // an invalid value, we can assert we get a basic cluster. + h2cCluster(cluster("extension/ns/ext", "extension/ns/ext", "extension_ns_ext")), + &envoy_config_cluster_v3.Cluster{ + TransportSocket: envoy_v3.UpstreamTLSTransportSocket( + &envoy_transport_socket_tls_v3.UpstreamTlsContext{ + CommonTlsContext: &envoy_transport_socket_tls_v3.CommonTlsContext{ + AlpnProtocols: []string{"h2"}, + }, + }, + ), + }, + ), + ), + }) +} + +func extGlobalCircuitBreakers(t *testing.T, rh ResourceEventHandlerWrapper, c *Contour) { + ext := &contour_v1alpha1.ExtensionService{ + ObjectMeta: fixture.ObjectMeta("ns/ext"), + Spec: contour_v1alpha1.ExtensionServiceSpec{ + Services: []contour_v1alpha1.ExtensionServiceTarget{ + {Name: "svc1", Port: 8081}, + {Name: "svc2", Port: 8082}, + }, + LoadBalancerPolicy: &contour_v1.LoadBalancerPolicy{ + Strategy: "Cookie", + }, + }, + } + + rh.OnAdd(ext) + + c.Request(clusterType).Equals(&envoy_service_discovery_v3.DiscoveryResponse{ + TypeUrl: clusterType, + Resources: resources(t, + DefaultCluster( + // Default load balancer policy should be set as we were passed + // an invalid value, we can assert we get a basic cluster. + h2cCluster(cluster("extension/ns/ext", "extension/ns/ext", "extension_ns_ext")), + &envoy_config_cluster_v3.Cluster{ + TransportSocket: envoy_v3.UpstreamTLSTransportSocket( + &envoy_transport_socket_tls_v3.UpstreamTlsContext{ + CommonTlsContext: &envoy_transport_socket_tls_v3.CommonTlsContext{ + AlpnProtocols: []string{"h2"}, + }, + }, + ), + CircuitBreakers: &envoy_config_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(20000), + MaxPendingRequests: wrapperspb.UInt32(2048), + MaxRequests: wrapperspb.UInt32(294), + MaxRetries: wrapperspb.UInt32(20), + TrackRemaining: true, + }}, + PerHostThresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(10), + TrackRemaining: true, + }}, + }, + }, + ), + ), + }) + + rh.OnUpdate(ext, &contour_v1alpha1.ExtensionService{ + ObjectMeta: fixture.ObjectMeta("ns/ext"), + Spec: contour_v1alpha1.ExtensionServiceSpec{ + Services: []contour_v1alpha1.ExtensionServiceTarget{ + {Name: "svc1", Port: 8081}, + {Name: "svc2", Port: 8082}, + }, + LoadBalancerPolicy: &contour_v1.LoadBalancerPolicy{ + Strategy: "RequestHash", + }, + }, + }) + + c.Request(clusterType).Equals(&envoy_service_discovery_v3.DiscoveryResponse{ + TypeUrl: clusterType, + Resources: resources(t, + DefaultCluster( + // Default load balancer policy should be set as we were passed + // an invalid value, we can assert we get a basic cluster. + h2cCluster(cluster("extension/ns/ext", "extension/ns/ext", "extension_ns_ext")), + &envoy_config_cluster_v3.Cluster{ + TransportSocket: envoy_v3.UpstreamTLSTransportSocket( + &envoy_transport_socket_tls_v3.UpstreamTlsContext{ + CommonTlsContext: &envoy_transport_socket_tls_v3.CommonTlsContext{ + AlpnProtocols: []string{"h2"}, + }, + }, + ), + CircuitBreakers: &envoy_config_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(20000), + MaxPendingRequests: wrapperspb.UInt32(2048), + MaxRequests: wrapperspb.UInt32(294), + MaxRetries: wrapperspb.UInt32(20), + TrackRemaining: true, + }}, + PerHostThresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(10), + TrackRemaining: true, + }}, + }, + }, + ), + ), + }) +} + +func overrideExtGlobalCircuitBreakers(t *testing.T, rh ResourceEventHandlerWrapper, c *Contour) { + ext := &contour_v1alpha1.ExtensionService{ + ObjectMeta: fixture.ObjectMeta("ns/ext"), + Spec: contour_v1alpha1.ExtensionServiceSpec{ + Services: []contour_v1alpha1.ExtensionServiceTarget{ + {Name: "svc1", Port: 8081}, + {Name: "svc2", Port: 8082}, + }, + LoadBalancerPolicy: &contour_v1.LoadBalancerPolicy{ + Strategy: "Cookie", + }, + CircuitBreakerPolicy: &contour_v1alpha1.CircuitBreakers{ + MaxConnections: 30000, + MaxPendingRequests: 3048, + MaxRequests: 394, + MaxRetries: 30, + PerHostMaxConnections: 30, + }, + }, + } + + rh.OnAdd(ext) + + c.Request(clusterType).Equals(&envoy_service_discovery_v3.DiscoveryResponse{ + TypeUrl: clusterType, + Resources: resources(t, + DefaultCluster( + // Default load balancer policy should be set as we were passed + // an invalid value, we can assert we get a basic cluster. + h2cCluster(cluster("extension/ns/ext", "extension/ns/ext", "extension_ns_ext")), + &envoy_config_cluster_v3.Cluster{ + TransportSocket: envoy_v3.UpstreamTLSTransportSocket( + &envoy_transport_socket_tls_v3.UpstreamTlsContext{ + CommonTlsContext: &envoy_transport_socket_tls_v3.CommonTlsContext{ + AlpnProtocols: []string{"h2"}, + }, + }, + ), + CircuitBreakers: &envoy_config_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(30000), + MaxPendingRequests: wrapperspb.UInt32(3048), + MaxRequests: wrapperspb.UInt32(394), + MaxRetries: wrapperspb.UInt32(30), + TrackRemaining: true, + }}, + PerHostThresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(30), + TrackRemaining: true, + }}, + }, + }, + ), + ), + }) + + rh.OnUpdate(ext, &contour_v1alpha1.ExtensionService{ + ObjectMeta: fixture.ObjectMeta("ns/ext"), + Spec: contour_v1alpha1.ExtensionServiceSpec{ + Services: []contour_v1alpha1.ExtensionServiceTarget{ + {Name: "svc1", Port: 8081}, + {Name: "svc2", Port: 8082}, + }, + LoadBalancerPolicy: &contour_v1.LoadBalancerPolicy{ + Strategy: "RequestHash", + }, + }, + }) + + c.Request(clusterType).Equals(&envoy_service_discovery_v3.DiscoveryResponse{ + TypeUrl: clusterType, + Resources: resources(t, + DefaultCluster( + // Default load balancer policy should be set as we were passed + // an invalid value, we can assert we get a basic cluster. + h2cCluster(cluster("extension/ns/ext", "extension/ns/ext", "extension_ns_ext")), + &envoy_config_cluster_v3.Cluster{ + TransportSocket: envoy_v3.UpstreamTLSTransportSocket( + &envoy_transport_socket_tls_v3.UpstreamTlsContext{ + CommonTlsContext: &envoy_transport_socket_tls_v3.CommonTlsContext{ + AlpnProtocols: []string{"h2"}, + }, + }, + ), + CircuitBreakers: &envoy_config_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(20000), + MaxPendingRequests: wrapperspb.UInt32(2048), + MaxRequests: wrapperspb.UInt32(294), + MaxRetries: wrapperspb.UInt32(20), + TrackRemaining: true, + }}, + PerHostThresholds: []*envoy_config_cluster_v3.CircuitBreakers_Thresholds{{ + MaxConnections: wrapperspb.UInt32(10), + TrackRemaining: true, + }}, + }, + }, + ), + ), + }) +} + func TestExtensionService(t *testing.T) { subtests := map[string]func(*testing.T, ResourceEventHandlerWrapper, *Contour){ - "Basic": extBasic, - "Cleartext": extCleartext, - "UpstreamValidation": extUpstreamValidation, - "ExternalName": extExternalName, - "IdleConnectionTimeout": extIdleConnectionTimeout, - "MissingService": extMissingService, - "InconsistentProto": extInconsistentProto, - "InvalidTimeout": extInvalidTimeout, - "InvalidLoadBalancerPolicy": extInvalidLoadBalancerPolicy, + "Basic": extBasic, + "Cleartext": extCleartext, + "UpstreamValidation": extUpstreamValidation, + "ExternalName": extExternalName, + "IdleConnectionTimeout": extIdleConnectionTimeout, + "MissingService": extMissingService, + "InconsistentProto": extInconsistentProto, + "InvalidTimeout": extInvalidTimeout, + "InvalidLoadBalancerPolicy": extInvalidLoadBalancerPolicy, + "CircuitBreakers": extCircuitBreakers, + "GlobalCircuitBreakers": extGlobalCircuitBreakers, + "OverrideGlobalCircuitBreakers": overrideExtGlobalCircuitBreakers, } for n, f := range subtests { f := f t.Run(n, func(t *testing.T) { - rh, c, done := setup(t) + var ( + rh ResourceEventHandlerWrapper + c *Contour + done func() + ) + + switch n { + case "GlobalCircuitBreakers", "OverrideGlobalCircuitBreakers": + rh, c, done = setup(t, + func(b *dag.Builder) { + for _, processor := range b.Processors { + if extensionProcessor, ok := processor.(*dag.ExtensionServiceProcessor); ok { + extensionProcessor.GlobalCircuitBreakerDefaults = &contour_v1alpha1.CircuitBreakers{ + MaxConnections: 20000, + MaxPendingRequests: 2048, + MaxRequests: 294, + MaxRetries: 20, + PerHostMaxConnections: 10, + } + } + } + }) + + default: + rh, c, done = setup(t) + } + defer done() // Add common test fixtures. diff --git a/internal/xdscache/v3/cluster_test.go b/internal/xdscache/v3/cluster_test.go index 585c3bfea57..b370948c6e8 100644 --- a/internal/xdscache/v3/cluster_test.go +++ b/internal/xdscache/v3/cluster_test.go @@ -724,7 +724,7 @@ func TestClusterVisit(t *testing.T) { }, ), }, - "circuitbreaker annotations": { + "CircuitBreakers annotations": { objs: []any{ &networking_v1.Ingress{ ObjectMeta: meta_v1.ObjectMeta{ diff --git a/pkg/config/parameters.go b/pkg/config/parameters.go index 51a1ba3eec8..04931c8d349 100644 --- a/pkg/config/parameters.go +++ b/pkg/config/parameters.go @@ -443,7 +443,7 @@ type ClusterParameters struct { // GlobalCircuitBreakerDefaults holds configurable global defaults for the circuit breakers. // // +optional - GlobalCircuitBreakerDefaults *contour_v1alpha1.GlobalCircuitBreakerDefaults `yaml:"circuit-breakers,omitempty"` + GlobalCircuitBreakerDefaults *contour_v1alpha1.CircuitBreakers `yaml:"circuit-breakers,omitempty"` // UpstreamTLS contains the TLS policy parameters for upstream connections UpstreamTLS ProtocolParameters `yaml:"upstream-tls,omitempty"` diff --git a/site/content/docs/main/config/api-reference.html b/site/content/docs/main/config/api-reference.html index 8dba46b5e33..57bc87795fd 100644 --- a/site/content/docs/main/config/api-reference.html +++ b/site/content/docs/main/config/api-reference.html @@ -5522,6 +5522,22 @@

ExtensionService protocol options will be available in future.

+ + +circuitBreakerPolicy +
+ + +CircuitBreakers + + + + +(Optional) +

CircuitBreakerPolicy specifies the circuit breaker budget across the extension service. +If defined this overrides the global circuit breaker budget.

+ + @@ -5610,6 +5626,90 @@

AccessLogType +

CircuitBreakers +

+

+(Appears on: +ClusterParameters, +ExtensionServiceSpec) +

+

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+maxConnections +
+ +uint32 + +		 +(Optional) +

The maximum number of connections that a single Envoy instance allows to the Kubernetes Service; defaults to 1024.

+
+maxPendingRequests +
+ +uint32 + +		 +(Optional) +

The maximum number of pending requests that a single Envoy instance allows to the Kubernetes Service; defaults to 1024.

+
+maxRequests +
+ +uint32 + +		 +(Optional) +

The maximum parallel requests a single Envoy instance allows to the Kubernetes Service; defaults to 1024

+
+maxRetries +
+ +uint32 + +		 +(Optional) +

The maximum number of parallel retries a single Envoy instance allows to the Kubernetes Service; defaults to 3.

+
+perHostMaxConnections +
+ +uint32 + +		 +

PerHostMaxConnections is the maximum number of connections +that Envoy will allow to each individual host in a cluster.

+

ClusterDNSFamilyType (string alias)

@@ -5723,8 +5823,8 @@

ClusterParameters circuitBreakers
- -GlobalCircuitBreakerDefaults + +CircuitBreakers @@ -7529,6 +7629,22 @@

ExtensionServiceSpec protocol options will be available in future.

+ + +circuitBreakerPolicy +
+ + +CircuitBreakers + + + + +(Optional) +

CircuitBreakerPolicy specifies the circuit breaker budget across the extension service. +If defined this overrides the global circuit breaker budget.

+ +

ExtensionServiceStatus @@ -7671,76 +7787,6 @@

GatewayConfig -

GlobalCircuitBreakerDefaults -

-

-(Appears on: -ClusterParameters) -

-

-

- - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
-maxConnections -
- -uint32 - -		 -(Optional) -

The maximum number of connections that a single Envoy instance allows to the Kubernetes Service; defaults to 1024.

-
-maxPendingRequests -
- -uint32 - -		 -(Optional) -

The maximum number of pending requests that a single Envoy instance allows to the Kubernetes Service; defaults to 1024.

-
-maxRequests -
- -uint32 - -		 -(Optional) -

The maximum parallel requests a single Envoy instance allows to the Kubernetes Service; defaults to 1024

-
-maxRetries -
- -uint32 - -		 -(Optional) -

The maximum number of parallel retries a single Envoy instance allows to the Kubernetes Service; defaults to 3.

-

HTTPProxyConfig