Skip to content

Commit ec61e87

Browse files
zqztenzqzten
committed
introduce controller concurrency settings to KubeFedConfig
1 parent 65a18ea commit ec61e87

File tree

11 files changed

+97
-6
lines changed

11 files changed

+97
-6
lines changed

‎cmd/controller-manager/app/controller-manager.go

+3
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,9 @@ func setOptionsByKubeFedConfig(opts *options.Options) {
365365
opts.ClusterHealthCheckConfig.FailureThreshold = *spec.ClusterHealthCheck.FailureThreshold
366366
opts.ClusterHealthCheckConfig.SuccessThreshold = *spec.ClusterHealthCheck.SuccessThreshold
367367

368+
opts.Config.MaxConcurrentSyncReconciles = *spec.SyncController.MaxConcurrentReconciles
369+
opts.Config.MaxConcurrentStatusReconciles = *spec.StatusController.MaxConcurrentReconciles
370+
368371
opts.Config.SkipAdoptingResources = *spec.SyncController.AdoptResources == corev1b1.AdoptResourcesDisabled
369372

370373
var featureGates = make(map[string]bool)

‎config/kubefedconfig.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,7 @@ spec:
2626
successThreshold: 1
2727
timeout: 3s
2828
syncController:
29+
maxConcurrentReconciles: 1
2930
adoptResources: Enabled
31+
statusController:
32+
maxConcurrentReconciles: 1

‎pkg/apis/core/v1beta1/defaults/defaults.go

+11
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ const (
3939
DefaultClusterHealthCheckFailureThreshold = 3
4040
DefaultClusterHealthCheckSuccessThreshold = 1
4141
DefaultClusterHealthCheckTimeout = 3 * time.Second
42+
43+
DefaultSyncControllerMaxConcurrentReconciles = 1
44+
DefaultStatusControllerMaxConcurrentReconciles = 1
4245
)
4346

4447
func SetDefaultKubeFedConfig(fedConfig *v1beta1.KubeFedConfig) {
@@ -87,10 +90,18 @@ func SetDefaultKubeFedConfig(fedConfig *v1beta1.KubeFedConfig) {
8790
spec.SyncController = &v1beta1.SyncControllerConfig{}
8891
}
8992

93+
setInt64(&spec.SyncController.MaxConcurrentReconciles, DefaultSyncControllerMaxConcurrentReconciles)
94+
9095
if spec.SyncController.AdoptResources == nil {
9196
spec.SyncController.AdoptResources = new(v1beta1.ResourceAdoption)
9297
*spec.SyncController.AdoptResources = v1beta1.AdoptResourcesEnabled
9398
}
99+
100+
if spec.StatusController == nil {
101+
spec.StatusController = &v1beta1.StatusControllerConfig{}
102+
}
103+
104+
setInt64(&spec.StatusController.MaxConcurrentReconciles, DefaultStatusControllerMaxConcurrentReconciles)
94105
}
95106

96107
func setDefaultKubeFedFeatureGates(fgc []v1beta1.FeatureGatesConfig) []v1beta1.FeatureGatesConfig {

‎pkg/apis/core/v1beta1/defaults/defaults_test.go

+15
Original file line numberDiff line numberDiff line change
@@ -120,12 +120,27 @@ func TestSetDefaultKubeFedConfig(t *testing.T) {
120120
successCases["spec.clusterHealthCheck.timeout is preserved"] = KubeFedConfigComparison{timeoutKFC, modifiedTimeoutKFC}
121121

122122
// SyncController
123+
syncControllerMaxConcurrentReconcilesKFC := defaultKubeFedConfig()
124+
syncControllerMaxConcurrentReconciles := int64(DefaultSyncControllerMaxConcurrentReconciles + 3)
125+
syncControllerMaxConcurrentReconcilesKFC.Spec.SyncController.MaxConcurrentReconciles = &syncControllerMaxConcurrentReconciles
126+
modifiedSyncControllerMaxConcurrentReconcilesKFC := syncControllerMaxConcurrentReconcilesKFC.DeepCopyObject().(*v1beta1.KubeFedConfig)
127+
SetDefaultKubeFedConfig(modifiedSyncControllerMaxConcurrentReconcilesKFC)
128+
successCases["spec.syncController.maxConcurrentReconciles is preserved"] = KubeFedConfigComparison{syncControllerMaxConcurrentReconcilesKFC, modifiedSyncControllerMaxConcurrentReconcilesKFC}
129+
123130
adoptResourcesKFC := defaultKubeFedConfig()
124131
*adoptResourcesKFC.Spec.SyncController.AdoptResources = v1beta1.AdoptResourcesDisabled
125132
modifiedAdoptResourcesKFC := adoptResourcesKFC.DeepCopyObject().(*v1beta1.KubeFedConfig)
126133
SetDefaultKubeFedConfig(modifiedAdoptResourcesKFC)
127134
successCases["spec.leaderElect.adoptResources is preserved"] = KubeFedConfigComparison{adoptResourcesKFC, modifiedAdoptResourcesKFC}
128135

136+
// StatusController
137+
statusControllerMaxConcurrentReconcilesKFC := defaultKubeFedConfig()
138+
statusControllerMaxConcurrentReconciles := int64(DefaultStatusControllerMaxConcurrentReconciles + 3)
139+
statusControllerMaxConcurrentReconcilesKFC.Spec.StatusController.MaxConcurrentReconciles = &statusControllerMaxConcurrentReconciles
140+
modifiedStatusControllerMaxConcurrentReconcilesKFC := statusControllerMaxConcurrentReconcilesKFC.DeepCopyObject().(*v1beta1.KubeFedConfig)
141+
SetDefaultKubeFedConfig(modifiedStatusControllerMaxConcurrentReconcilesKFC)
142+
successCases["spec.statusController.maxConcurrentReconciles is preserved"] = KubeFedConfigComparison{statusControllerMaxConcurrentReconcilesKFC, modifiedStatusControllerMaxConcurrentReconcilesKFC}
143+
129144
for k, v := range successCases {
130145
if !reflect.DeepEqual(v.original, v.modified) {
131146
t.Errorf("[%s] expected success: original=%+v, modified=%+v", k, *v.original, *v.modified)

‎pkg/apis/core/v1beta1/kubefedconfig_types.go

+13
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ type KubeFedConfigSpec struct {
3737
ClusterHealthCheck *ClusterHealthCheckConfig `json:"clusterHealthCheck,omitempty"`
3838
// +optional
3939
SyncController *SyncControllerConfig `json:"syncController,omitempty"`
40+
// +optional
41+
StatusController *StatusControllerConfig `json:"statusController,omitempty"`
4042
}
4143

4244
type DurationConfig struct {
@@ -105,6 +107,10 @@ type ClusterHealthCheckConfig struct {
105107
}
106108

107109
type SyncControllerConfig struct {
110+
// The maximum number of concurrent Reconciles of sync controller which can be run.
111+
// Defaults to 1.
112+
// +optional
113+
MaxConcurrentReconciles *int64 `json:"maxConcurrentReconciles,omitempty"`
108114
// Whether to adopt pre-existing resources in member clusters. Defaults to
109115
// "Enabled".
110116
// +optional
@@ -118,6 +124,13 @@ const (
118124
AdoptResourcesDisabled ResourceAdoption = "Disabled"
119125
)
120126

127+
type StatusControllerConfig struct {
128+
// The maximum number of concurrent Reconciles of status controller which can be run.
129+
// Defaults to 1.
130+
// +optional
131+
MaxConcurrentReconciles *int64 `json:"maxConcurrentReconciles,omitempty"`
132+
}
133+
121134
// +kubebuilder:object:root=true
122135
// +kubebuilder:resource:path=kubefedconfigs
123136

‎pkg/apis/core/v1beta1/validation/validation.go

+9
Original file line numberDiff line numberDiff line change
@@ -356,10 +356,19 @@ func ValidateKubeFedConfig(kubeFedConfig, oldKubeFedConfig *v1beta1.KubeFedConfi
356356
case sync.AdoptResources == nil:
357357
allErrs = append(allErrs, field.Required(adoptPath, ""))
358358
default:
359+
allErrs = append(allErrs, validateIntPtrGreaterThan0(syncPath.Child("maxConcurrentReconciles"), sync.MaxConcurrentReconciles)...)
359360
allErrs = append(allErrs, validateEnumStrings(adoptPath, string(*sync.AdoptResources),
360361
[]string{string(v1beta1.AdoptResourcesEnabled), string(v1beta1.AdoptResourcesDisabled)})...)
361362
}
362363

364+
statusController := spec.StatusController
365+
statusControllerPath := specPath.Child("statusController")
366+
if statusController == nil {
367+
allErrs = append(allErrs, field.Required(statusControllerPath, ""))
368+
} else {
369+
allErrs = append(allErrs, validateIntPtrGreaterThan0(statusControllerPath.Child("maxConcurrentReconciles"), statusController.MaxConcurrentReconciles)...)
370+
}
371+
363372
return allErrs
364373
}
365374

‎pkg/apis/core/v1beta1/validation/validation_test.go

+20
Original file line numberDiff line numberDiff line change
@@ -844,6 +844,14 @@ func TestValidateKubeFedConfig(t *testing.T) {
844844
invalidSyncControllerNil.Spec.SyncController = nil
845845
errorCases["spec.syncController: Required value"] = invalidSyncControllerNil
846846

847+
invalidSyncControllerMaxConcurrentReconcilesNil := testcommon.ValidKubeFedConfig()
848+
invalidSyncControllerMaxConcurrentReconcilesNil.Spec.SyncController.MaxConcurrentReconciles = nil
849+
errorCases["spec.syncController.maxConcurrentReconciles: Required value"] = invalidSyncControllerMaxConcurrentReconcilesNil
850+
851+
invalidSyncControllerMaxConcurrentReconcilesGreaterThan0 := testcommon.ValidKubeFedConfig()
852+
invalidSyncControllerMaxConcurrentReconcilesGreaterThan0.Spec.SyncController.MaxConcurrentReconciles = zeroIntPtr
853+
errorCases["spec.syncController.maxConcurrentReconciles: Invalid value"] = invalidSyncControllerMaxConcurrentReconcilesGreaterThan0
854+
847855
invalidAdoptResourcesNil := testcommon.ValidKubeFedConfig()
848856
invalidAdoptResourcesNil.Spec.SyncController.AdoptResources = nil
849857
errorCases["spec.syncController.adoptResources: Required value"] = invalidAdoptResourcesNil
@@ -853,6 +861,18 @@ func TestValidateKubeFedConfig(t *testing.T) {
853861
invalidAdoptResources.Spec.SyncController.AdoptResources = &invalidAdoptResourcesValue
854862
errorCases["spec.syncController.adoptResources: Unsupported value"] = invalidAdoptResources
855863

864+
invalidStatusControllerNil := testcommon.ValidKubeFedConfig()
865+
invalidStatusControllerNil.Spec.StatusController = nil
866+
errorCases["spec.statusController: Required value"] = invalidStatusControllerNil
867+
868+
invalidStatusControllerMaxConcurrentReconcilesNil := testcommon.ValidKubeFedConfig()
869+
invalidStatusControllerMaxConcurrentReconcilesNil.Spec.StatusController.MaxConcurrentReconciles = nil
870+
errorCases["spec.statusController.maxConcurrentReconciles: Required value"] = invalidStatusControllerMaxConcurrentReconcilesNil
871+
872+
invalidStatusControllerMaxConcurrentReconcilesGreaterThan0 := testcommon.ValidKubeFedConfig()
873+
invalidStatusControllerMaxConcurrentReconcilesGreaterThan0.Spec.StatusController.MaxConcurrentReconciles = zeroIntPtr
874+
errorCases["spec.statusController.maxConcurrentReconciles: Invalid value"] = invalidStatusControllerMaxConcurrentReconcilesGreaterThan0
875+
856876
for k, v := range errorCases {
857877
errs := ValidateKubeFedConfig(v, testcommon.ValidKubeFedConfig())
858878
if len(errs) == 0 {

‎pkg/controller/status/controller.go

+1
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ func newKubeFedStatusController(controllerConfig *util.ControllerConfig, typeCon
127127
WorkerTiming: util.WorkerTiming{
128128
ClusterSyncDelay: s.clusterAvailableDelay,
129129
},
130+
MaxConcurrentReconciles: int(controllerConfig.MaxConcurrentStatusReconciles),
130131
})
131132

132133
// Build deliverer for triggering cluster reconciliations.

‎pkg/controller/sync/controller.go

+1
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ func newKubeFedSyncController(controllerConfig *util.ControllerConfig, typeConfi
136136
WorkerTiming: util.WorkerTiming{
137137
ClusterSyncDelay: s.clusterAvailableDelay,
138138
},
139+
MaxConcurrentReconciles: int(controllerConfig.MaxConcurrentSyncReconciles),
139140
})
140141

141142
// Build deliverer for triggering cluster reconciliations.

‎pkg/controller/testdata/fixtures/crds.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -774,12 +774,25 @@ spec:
774774
`Namespaced` or `Cluster`. `Namespaced` indicates that the KubeFed
775775
namespace will be the only target of the control plane.
776776
type: string
777+
statusController:
778+
properties:
779+
maxConcurrentReconciles:
780+
description: The maximum number of concurrent Reconciles of status
781+
controller which can be run. Defaults to 1.
782+
format: int64
783+
type: integer
784+
type: object
777785
syncController:
778786
properties:
779787
adoptResources:
780788
description: Whether to adopt pre-existing resources in member
781789
clusters. Defaults to "Enabled".
782790
type: string
791+
maxConcurrentReconciles:
792+
description: The maximum number of concurrent Reconciles of sync
793+
controller which can be run. Defaults to 1.
794+
format: int64
795+
type: integer
783796
type: object
784797
required:
785798
- scope

‎pkg/controller/util/controllerconfig.go

+8-6
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,14 @@ type ClusterHealthCheckConfig struct {
6868
// controllers.
6969
type ControllerConfig struct {
7070
KubeFedNamespaces
71-
KubeConfig *restclient.Config
72-
ClusterAvailableDelay time.Duration
73-
ClusterUnavailableDelay time.Duration
74-
MinimizeLatency bool
75-
SkipAdoptingResources bool
76-
RawResourceStatusCollection bool
71+
KubeConfig *restclient.Config
72+
ClusterAvailableDelay time.Duration
73+
ClusterUnavailableDelay time.Duration
74+
MinimizeLatency bool
75+
MaxConcurrentSyncReconciles int64
76+
MaxConcurrentStatusReconciles int64
77+
SkipAdoptingResources bool
78+
RawResourceStatusCollection bool
7779
}
7880

7981
func (c *ControllerConfig) LimitedScope() bool {

0 commit comments

Comments
 (0)