Skip to content

Commit 948404e

Browse files
salonichf5salonichf5
authored
Add all load balancing methods (#4325)
Problem: Users want to specify all load balancing methods. Solution: Specify a way for user to be able to specify all combinations of load balancing methods.
1 parent 758a284 commit 948404e

File tree

19 files changed

+827
-27
lines changed

19 files changed

+827
-27
lines changed

‎apis/v1alpha1/upstreamsettingspolicy_types.go‎

Lines changed: 90 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ type UpstreamSettingsPolicyList struct {
3636
}
3737

3838
// UpstreamSettingsPolicySpec defines the desired state of the UpstreamSettingsPolicy.
39+
// +kubebuilder:validation:XValidation:rule="!(has(self.loadBalancingMethod) && (self.loadBalancingMethod == 'hash' || self.loadBalancingMethod == 'hash consistent')) || has(self.hashMethodKey)",message="hashMethodKey is required when loadBalancingMethod is 'hash' or 'hash consistent'"
40+
//
41+
//nolint:lll
3942
type UpstreamSettingsPolicySpec struct {
4043
// ZoneSize is the size of the shared memory zone used by the upstream. This memory zone is used to share
4144
// the upstream configuration between nginx worker processes. The more servers that an upstream has,
@@ -58,6 +61,12 @@ type UpstreamSettingsPolicySpec struct {
5861
// +optional
5962
LoadBalancingMethod *LoadBalancingType `json:"loadBalancingMethod,omitempty"`
6063

64+
// HashMethodKey defines the key used for hash-based load balancing methods.
65+
// This field is required when `LoadBalancingMethod` is set to `hash` or `hash consistent`.
66+
//
67+
// +optional
68+
HashMethodKey *HashMethodKey `json:"hashMethodKey,omitempty"`
69+
6170
// TargetRefs identifies API object(s) to apply the policy to.
6271
// Objects must be in the same namespace as the policy.
6372
// Support: Service
@@ -108,19 +117,96 @@ type UpstreamKeepAlive struct {
108117

109118
// LoadBalancingType defines the supported load balancing methods.
110119
//
111-
// +kubebuilder:validation:Enum=ip_hash;random two least_conn
120+
// +kubebuilder:validation:Enum=round_robin;least_conn;ip_hash;hash;hash consistent;random;random two;random two least_conn;random two least_time=header;random two least_time=last_byte;least_time header;least_time last_byte;least_time header inflight;least_time last_byte inflight
121+
//
122+
//nolint:lll
112123
type LoadBalancingType string
113124

114125
const (
126+
// Combination of NGINX directive
127+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#random
128+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#least_conn
129+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#least_time
130+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream
131+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#ip_hash
132+
// - https://nginx.org/en/docs/http/ngx_http_upstream_module.html#hash
133+
134+
// LoadBalancingMethods supported by NGINX OSS and NGINX Plus.
135+
136+
// LoadBalancingTypeRoundRobin enables round-robin load balancing,
137+
// distributing requests evenly across all upstream servers.
138+
LoadBalancingTypeRoundRobin LoadBalancingType = "round_robin"
139+
140+
// LoadBalancingTypeLeastConnection enables least-connections load balancing,
141+
// routing requests to the upstream server with the fewest active connections.
142+
LoadBalancingTypeLeastConnection LoadBalancingType = "least_conn"
143+
115144
// LoadBalancingTypeIPHash enables IP hash-based load balancing,
116145
// ensuring requests from the same client IP are routed to the same upstream server.
117-
// NGINX directive: https://nginx.org/en/docs/http/ngx_http_upstream_module.html#ip_hash
118146
LoadBalancingTypeIPHash LoadBalancingType = "ip_hash"
119147

148+
// LoadBalancingTypeHash enables generic hash-based load balancing,
149+
// routing requests to upstream servers based on a hash of a specified key
150+
// HashMethodKey field must be set when this method is selected.
151+
// Example configuration: hash $binary_remote_addr;.
152+
LoadBalancingTypeHash LoadBalancingType = "hash"
153+
154+
// LoadBalancingTypeHashConsistent enables consistent hash-based load balancing,
155+
// which minimizes the number of keys remapped when a server is added or removed.
156+
// HashMethodKey field must be set when this method is selected.
157+
// Example configuration: hash $binary_remote_addr consistent;.
158+
LoadBalancingTypeHashConsistent LoadBalancingType = "hash consistent"
159+
160+
// LoadBalancingTypeRandom enables random load balancing,
161+
// routing requests to upstream servers in a random manner.
162+
LoadBalancingTypeRandom LoadBalancingType = "random"
163+
164+
// LoadBalancingTypeRandomTwo enables a variation of random load balancing
165+
// that randomly selects two servers and forwards traffic to one of them.
166+
// The default method is least_conn which passes a request to a server with the least number of active connections.
167+
LoadBalancingTypeRandomTwo LoadBalancingType = "random two"
168+
120169
// LoadBalancingTypeRandomTwoLeastConnection enables a variation of least-connections
121170
// balancing that randomly selects two servers and forwards traffic to the one with
122171
// fewer active connections.
123-
// NGINX directive least_conn: https://nginx.org/en/docs/http/ngx_http_upstream_module.html#least_conn
124-
// NGINX directive random: https://nginx.org/en/docs/http/ngx_http_upstream_module.html#random
125172
LoadBalancingTypeRandomTwoLeastConnection LoadBalancingType = "random two least_conn"
173+
174+
// LoadBalancingMethods supported by NGINX Plus.
175+
176+
// LoadBalancingTypeRandomTwoLeastTimeHeader enables a variation of least-time load balancing
177+
// that randomly selects two servers and forwards traffic to the one with the least
178+
// time to receive the response header.
179+
LoadBalancingTypeRandomTwoLeastTimeHeader LoadBalancingType = "random two least_time=header"
180+
181+
// LoadBalancingTypeRandomTwoLeastTimeLastByte enables a variation of least-time load balancing
182+
// that randomly selects two servers and forwards traffic to the one with the least time
183+
// to receive the full response.
184+
LoadBalancingTypeRandomTwoLeastTimeLastByte LoadBalancingType = "random two least_time=last_byte"
185+
186+
// LoadBalancingTypeLeastTimeHeader enables least-time load balancing,
187+
// routing requests to the upstream server with the least time to receive the response header.
188+
LoadBalancingTypeLeastTimeHeader LoadBalancingType = "least_time header"
189+
190+
// LoadBalancingTypeLeastTimeLastByte enables least-time load balancing,
191+
// routing requests to the upstream server with the least time to receive the full response.
192+
LoadBalancingTypeLeastTimeLastByte LoadBalancingType = "least_time last_byte"
193+
194+
// LoadBalancingTypeLeastTimeHeaderInflight enables least-time load balancing,
195+
// routing requests to the upstream server with the least time to receive the response header,
196+
// considering the incomplete requests.
197+
LoadBalancingTypeLeastTimeHeaderInflight LoadBalancingType = "least_time header inflight"
198+
199+
// LoadBalancingTypeLeastTimeLastByteInflight enables least-time load balancing,
200+
// routing requests to the upstream server with the least time to receive the full response,
201+
// considering the incomplete requests.
202+
LoadBalancingTypeLeastTimeLastByteInflight LoadBalancingType = "least_time last_byte inflight"
126203
)
204+
205+
// HashMethodKey defines the key used for hash-based load balancing methods.
206+
// The key must be a valid NGINX variable name starting with '$' followed by lowercase
207+
// letters and underscores only.
208+
// For a full list of NGINX variables,
209+
// refer to: https://nginx.org/en/docs/http/ngx_http_upstream_module.html#variables
210+
//
211+
// +kubebuilder:validation:Pattern=`^\$[a-z_]+$`
212+
type HashMethodKey string

‎apis/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/gateway.nginx.org_upstreamsettingspolicies.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,12 @@ spec:
5151
spec:
5252
description: Spec defines the desired state of the UpstreamSettingsPolicy.
5353
properties:
54+
hashMethodKey:
55+
description: |-
56+
HashMethodKey defines the key used for hash-based load balancing methods.
57+
This field is required when `LoadBalancingMethod` is set to `hash` or `hash consistent`.
58+
pattern: ^\$[a-z_]+$
59+
type: string
5460
keepAlive:
5561
description: KeepAlive defines the keep-alive settings.
5662
properties:
@@ -91,8 +97,20 @@ spec:
9197
If not specified, NGINX Gateway Fabric defaults to `random two least_conn`,
9298
which differs from the standard NGINX default `round-robin`.
9399
enum:
100+
- round_robin
101+
- least_conn
94102
- ip_hash
103+
- hash
104+
- hash consistent
105+
- random
106+
- random two
95107
- random two least_conn
108+
- random two least_time=header
109+
- random two least_time=last_byte
110+
- least_time header
111+
- least_time last_byte
112+
- least_time header inflight
113+
- least_time last_byte inflight
96114
type: string
97115
targetRefs:
98116
description: |-
@@ -152,6 +170,12 @@ spec:
152170
required:
153171
- targetRefs
154172
type: object
173+
x-kubernetes-validations:
174+
- message: hashMethodKey is required when loadBalancingMethod is 'hash'
175+
or 'hash consistent'
176+
rule: '!(has(self.loadBalancingMethod) && (self.loadBalancingMethod
177+
== ''hash'' || self.loadBalancingMethod == ''hash consistent'')) ||
178+
has(self.hashMethodKey)'
155179
status:
156180
description: Status defines the state of the UpstreamSettingsPolicy.
157181
properties:

‎deploy/crds.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9578,6 +9578,12 @@ spec:
95789578
spec:
95799579
description: Spec defines the desired state of the UpstreamSettingsPolicy.
95809580
properties:
9581+
hashMethodKey:
9582+
description: |-
9583+
HashMethodKey defines the key used for hash-based load balancing methods.
9584+
This field is required when `LoadBalancingMethod` is set to `hash` or `hash consistent`.
9585+
pattern: ^\$[a-z_]+$
9586+
type: string
95819587
keepAlive:
95829588
description: KeepAlive defines the keep-alive settings.
95839589
properties:
@@ -9618,8 +9624,20 @@ spec:
96189624
If not specified, NGINX Gateway Fabric defaults to `random two least_conn`,
96199625
which differs from the standard NGINX default `round-robin`.
96209626
enum:
9627+
- round_robin
9628+
- least_conn
96219629
- ip_hash
9630+
- hash
9631+
- hash consistent
9632+
- random
9633+
- random two
96229634
- random two least_conn
9635+
- random two least_time=header
9636+
- random two least_time=last_byte
9637+
- least_time header
9638+
- least_time last_byte
9639+
- least_time header inflight
9640+
- least_time last_byte inflight
96239641
type: string
96249642
targetRefs:
96259643
description: |-
@@ -9679,6 +9697,12 @@ spec:
96799697
required:
96809698
- targetRefs
96819699
type: object
9700+
x-kubernetes-validations:
9701+
- message: hashMethodKey is required when loadBalancingMethod is 'hash'
9702+
or 'hash consistent'
9703+
rule: '!(has(self.loadBalancingMethod) && (self.loadBalancingMethod
9704+
== ''hash'' || self.loadBalancingMethod == ''hash consistent'')) ||
9705+
has(self.hashMethodKey)'
96829706
status:
96839707
description: Status defines the state of the UpstreamSettingsPolicy.
96849708
properties:

‎internal/controller/manager.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func StartManager(cfg config.Config) error {
124124
mustExtractGVK := kinds.NewMustExtractGKV(scheme)
125125

126126
genericValidator := ngxvalidation.GenericValidator{}
127-
policyManager := createPolicyManager(mustExtractGVK, genericValidator)
127+
policyManager := createPolicyManager(mustExtractGVK, genericValidator, cfg.Plus)
128128

129129
plusSecrets, err := createPlusSecretMetadata(cfg, mgr.GetAPIReader())
130130
if err != nil {
@@ -323,6 +323,7 @@ func StartManager(cfg config.Config) error {
323323
func createPolicyManager(
324324
mustExtractGVK kinds.MustExtractGVK,
325325
validator validation.GenericValidator,
326+
plusEnabled bool,
326327
) *policies.CompositeValidator {
327328
cfgs := []policies.ManagerConfig{
328329
{
@@ -335,7 +336,7 @@ func createPolicyManager(
335336
},
336337
{
337338
GVK: mustExtractGVK(&ngfAPIv1alpha1.UpstreamSettingsPolicy{}),
338-
Validator: upstreamsettings.NewValidator(validator),
339+
Validator: upstreamsettings.NewValidator(validator, plusEnabled),
339340
},
340341
}
341342

‎internal/controller/nginx/config/http/config.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package http
22

33
import (
4+
ngfAPI "github.com/nginx/nginx-gateway-fabric/v2/apis/v1alpha1"
45
"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/nginx/config/shared"
56
)
67

@@ -123,6 +124,7 @@ type Upstream struct {
123124
ZoneSize string // format: 512k, 1m
124125
StateFile string
125126
LoadBalancingMethod string
127+
HashMethodKey string
126128
KeepAlive UpstreamKeepAlive
127129
Servers []UpstreamServer
128130
}
@@ -167,3 +169,33 @@ type ServerConfig struct {
167169
Plus bool
168170
DisableSNIHostValidation bool
169171
}
172+
173+
var (
174+
OSSAllowedLBMethods = map[ngfAPI.LoadBalancingType]struct{}{
175+
ngfAPI.LoadBalancingTypeRoundRobin: {},
176+
ngfAPI.LoadBalancingTypeLeastConnection: {},
177+
ngfAPI.LoadBalancingTypeIPHash: {},
178+
ngfAPI.LoadBalancingTypeRandom: {},
179+
ngfAPI.LoadBalancingTypeHash: {},
180+
ngfAPI.LoadBalancingTypeHashConsistent: {},
181+
ngfAPI.LoadBalancingTypeRandomTwo: {},
182+
ngfAPI.LoadBalancingTypeRandomTwoLeastConnection: {},
183+
}
184+
185+
PlusAllowedLBMethods = map[ngfAPI.LoadBalancingType]struct{}{
186+
ngfAPI.LoadBalancingTypeRoundRobin: {},
187+
ngfAPI.LoadBalancingTypeLeastConnection: {},
188+
ngfAPI.LoadBalancingTypeIPHash: {},
189+
ngfAPI.LoadBalancingTypeRandom: {},
190+
ngfAPI.LoadBalancingTypeHash: {},
191+
ngfAPI.LoadBalancingTypeHashConsistent: {},
192+
ngfAPI.LoadBalancingTypeRandomTwo: {},
193+
ngfAPI.LoadBalancingTypeRandomTwoLeastConnection: {},
194+
ngfAPI.LoadBalancingTypeLeastTimeHeader: {},
195+
ngfAPI.LoadBalancingTypeLeastTimeLastByte: {},
196+
ngfAPI.LoadBalancingTypeLeastTimeHeaderInflight: {},
197+
ngfAPI.LoadBalancingTypeLeastTimeLastByteInflight: {},
198+
ngfAPI.LoadBalancingTypeRandomTwoLeastTimeHeader: {},
199+
ngfAPI.LoadBalancingTypeRandomTwoLeastTimeLastByte: {},
200+
}
201+
)

‎internal/controller/nginx/config/policies/upstreamsettings/processor.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ type UpstreamSettings struct {
1515
ZoneSize string
1616
// LoadBalancingMethod is the load balancing method setting.
1717
LoadBalancingMethod string
18+
// HashMethodKey is the key to be used for hash-based load balancing methods.
19+
HashMethodKey string
1820
// KeepAlive contains the keepalive settings.
1921
KeepAlive http.UpstreamKeepAlive
2022
}
@@ -67,6 +69,10 @@ func processPolicies(pols []policies.Policy) UpstreamSettings {
6769
if usp.Spec.LoadBalancingMethod != nil {
6870
upstreamSettings.LoadBalancingMethod = string(*usp.Spec.LoadBalancingMethod)
6971
}
72+
73+
if usp.Spec.HashMethodKey != nil {
74+
upstreamSettings.HashMethodKey = string(*usp.Spec.HashMethodKey)
75+
}
7076
}
7177

7278
return upstreamSettings

0 commit comments

Comments
 (0)