diff --git a/PROJECT b/PROJECT index 95f84f4f6..bd8a7eaf1 100644 --- a/PROJECT +++ b/PROJECT @@ -21,4 +21,12 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + domain: jobset.x-k8s.io + group: jobset + kind: Configuration + path: sigs.k8s.io/jobset/api/config/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/config/v1alpha1/configuration_types.go b/api/config/v1alpha1/configuration_types.go new file mode 100644 index 000000000..2f4a140ae --- /dev/null +++ b/api/config/v1alpha1/configuration_types.go @@ -0,0 +1,133 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" +) + +// +k8s:defaulter-gen=true +// +kubebuilder:object:root=true + +// Configuration is the Schema for the configurations API +type Configuration struct { + metav1.TypeMeta `json:",inline"` + + // Namespace is the namespace in which jobset controller is deployed. + // It is used as part of DNSName of the webhook Service. + // If not set, the value is set from the file /var/run/secrets/kubernetes.io/serviceaccount/namespace + // If the file doesn't exist, default value is kueue-system. + Namespace *string `json:"namespace,omitempty"` + + // ControllerManager returns the configurations for controllers + ControllerManager `json:",inline"` + + // InternalCertManagerment is configuration for internalCertManagerment + InternalCertManagement *InternalCertManagement `json:"internalCertManagement,omitempty"` + + ClientConnection *ClientConnection `json:"clientConnection,omitempty"` +} + +type ControllerManager struct { + // Webhook contains the controllers webhook configuration + // +optional + Webhook ControllerWebhook `json:"webhook,omitempty"` + + // LeaderElection is the LeaderElection config to be used when configuring + // the manager.Manager leader election + // +optional + LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` + + // Metrics contains the controller metrics configuration + // +optional + Metrics ControllerMetrics `json:"metrics,omitempty"` + + // Health contains the controller health configuration + // +optional + Health ControllerHealth `json:"health,omitempty"` +} + +// ControllerWebhook defines the webhook server for the controller. +type ControllerWebhook struct { + // Port is the port that the webhook server serves at. + // It is used to set webhook.Server.Port. + // +optional + Port *int `json:"port,omitempty"` + + // Host is the hostname that the webhook server binds to. + // It is used to set webhook.Server.Host. + // +optional + Host string `json:"host,omitempty"` + + // CertDir is the directory that contains the server key and certificate. + // if not set, webhook server would look up the server key and certificate in + // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate + // must be named tls.key and tls.crt, respectively. + // +optional + CertDir string `json:"certDir,omitempty"` +} + +// ControllerMetrics defines the metrics configs. +type ControllerMetrics struct { + // BindAddress is the TCP address that the controller should bind to + // for serving prometheus metrics. + // It can be set to "0" to disable the metrics serving. + // +optional + BindAddress string `json:"bindAddress,omitempty"` +} + +// ControllerHealth defines the health configs. +type ControllerHealth struct { + // HealthProbeBindAddress is the TCP address that the controller should bind to + // for serving health probes + // It can be set to "0" or "" to disable serving the health probe. + // +optional + HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"` + + // ReadinessEndpointName, defaults to "readyz" + // +optional + ReadinessEndpointName string `json:"readinessEndpointName,omitempty"` + + // LivenessEndpointName, defaults to "healthz" + // +optional + LivenessEndpointName string `json:"livenessEndpointName,omitempty"` +} + +type InternalCertManagement struct { + // Enable controls whether to enable internal cert management or not. + // Defaults to true. If you want to use a third-party management, e.g. cert-manager, + // set it to false. See the user guide for more information. + Enable *bool `json:"enable,omitempty"` + + // WebhookServiceName is the name of the Service used as part of the DNSName. + // Defaults to jobset-webhook-service. + WebhookServiceName *string `json:"webhookServiceName,omitempty"` + + // WebhookSecretName is the name of the Secret used to store CA and server certs. + // Defaults to jobset-webhook-server-cert. + WebhookSecretName *string `json:"webhookSecretName,omitempty"` +} + +type ClientConnection struct { + // QPS controls the number of queries per second allowed for K8S api server + // connection. + QPS *float32 `json:"qps,omitempty"` + + // Burst allows extra queries to accumulate when a client is exceeding its rate. + Burst *int32 `json:"burst,omitempty"` +} diff --git a/api/config/v1alpha1/defaults.go b/api/config/v1alpha1/defaults.go new file mode 100644 index 000000000..d9038ab38 --- /dev/null +++ b/api/config/v1alpha1/defaults.go @@ -0,0 +1,105 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "os" + "strings" + "time" + + configv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/utils/ptr" +) + +const ( + DefaultNamespace = "jobset-system" + DefaultWebhookServiceName = "jobset-webhook-service" + DefaultWebhookSecretName = "jobset-webhook-server-cert" + DefaultWebhookPort = 9443 + DefaultHealthProbeBindAddress = ":8081" + DefaultMetricsBindAddress = ":8080" + DefaultLeaderElectionID = "6d4f6a47.jobset.x-k8s.io" + DefaultLeaderElectionLeaseDuration = 15 * time.Second + DefaultLeaderElectionRenewDeadline = 10 * time.Second + DefaultLeaderElectionRetryPeriod = 2 * time.Second + DefaultResourceLock = "leases" + DefaultClientConnectionQPS float32 = 20.0 + DefaultClientConnectionBurst int32 = 30 +) + +func getOperatorNamespace() string { + if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + if ns := strings.TrimSpace(string(data)); len(ns) > 0 { + return ns + } + } + return DefaultNamespace +} + +// SetDefaults_Configuration sets default values for ComponentConfig. +// +//nolint:revive // format required by generated code for defaulting +func SetDefaults_Configuration(cfg *Configuration) { + if cfg.Namespace == nil { + cfg.Namespace = ptr.To(getOperatorNamespace()) + } + if cfg.Webhook.Port == nil { + cfg.Webhook.Port = ptr.To(DefaultWebhookPort) + } + if len(cfg.Metrics.BindAddress) == 0 { + cfg.Metrics.BindAddress = DefaultMetricsBindAddress + } + if len(cfg.Health.HealthProbeBindAddress) == 0 { + cfg.Health.HealthProbeBindAddress = DefaultHealthProbeBindAddress + } + + if cfg.LeaderElection == nil { + cfg.LeaderElection = &configv1alpha1.LeaderElectionConfiguration{} + } + if len(cfg.LeaderElection.ResourceName) == 0 { + cfg.LeaderElection.ResourceName = DefaultLeaderElectionID + } + if len(cfg.LeaderElection.ResourceLock) == 0 { + cfg.LeaderElection.ResourceLock = DefaultResourceLock + } + // Use the default LeaderElectionConfiguration options + configv1alpha1.RecommendedDefaultLeaderElectionConfiguration(cfg.LeaderElection) + + if cfg.InternalCertManagement == nil { + cfg.InternalCertManagement = &InternalCertManagement{} + } + if cfg.InternalCertManagement.Enable == nil { + cfg.InternalCertManagement.Enable = ptr.To(true) + } + if *cfg.InternalCertManagement.Enable { + if cfg.InternalCertManagement.WebhookServiceName == nil { + cfg.InternalCertManagement.WebhookServiceName = ptr.To(DefaultWebhookServiceName) + } + if cfg.InternalCertManagement.WebhookSecretName == nil { + cfg.InternalCertManagement.WebhookSecretName = ptr.To(DefaultWebhookSecretName) + } + } + if cfg.ClientConnection == nil { + cfg.ClientConnection = &ClientConnection{} + } + if cfg.ClientConnection.QPS == nil { + cfg.ClientConnection.QPS = ptr.To(DefaultClientConnectionQPS) + } + if cfg.ClientConnection.Burst == nil { + cfg.ClientConnection.Burst = ptr.To(DefaultClientConnectionBurst) + } +} diff --git a/api/config/v1alpha1/defaults_test.go b/api/config/v1alpha1/defaults_test.go new file mode 100644 index 000000000..ccc07a3d2 --- /dev/null +++ b/api/config/v1alpha1/defaults_test.go @@ -0,0 +1,289 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/utils/ptr" +) + +const ( + overwriteNamespace = "jobset-tenant-a" + overwriteWebhookPort = 9444 + overwriteMetricBindAddress = ":38081" + overwriteHealthProbeBindAddress = ":38080" + overwriteLeaderElectionID = "foo.jobset.x-k8s.io" +) + +func TestSetDefaults_Configuration(t *testing.T) { + defaultCtrlManagerConfigurationSpec := ControllerManager{ + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: DefaultResourceLock, + ResourceName: DefaultLeaderElectionID, + }, + Webhook: ControllerWebhook{ + Port: ptr.To(DefaultWebhookPort), + }, + Metrics: ControllerMetrics{ + BindAddress: DefaultMetricsBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: DefaultHealthProbeBindAddress, + }, + } + defaultClientConnection := &ClientConnection{ + QPS: ptr.To(DefaultClientConnectionQPS), + Burst: ptr.To(DefaultClientConnectionBurst), + } + + testCases := map[string]struct { + original *Configuration + want *Configuration + }{ + "defaulting namespace": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + }, + }, + "defaulting ControllerManager": { + original: &Configuration{ + ControllerManager: ControllerManager{ + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ + Port: ptr.To(DefaultWebhookPort), + }, + Metrics: ControllerMetrics{ + BindAddress: DefaultMetricsBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: DefaultHealthProbeBindAddress, + }, + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: DefaultResourceLock, + ResourceName: DefaultLeaderElectionID, + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + }, + }, + "should not default ControllerManager": { + original: &Configuration{ + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ + Port: ptr.To(overwriteWebhookPort), + }, + Metrics: ControllerMetrics{ + BindAddress: overwriteMetricBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: overwriteHealthProbeBindAddress, + }, + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: DefaultResourceLock, + ResourceName: overwriteLeaderElectionID, + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ + Port: ptr.To(overwriteWebhookPort), + }, + Metrics: ControllerMetrics{ + BindAddress: overwriteMetricBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: overwriteHealthProbeBindAddress, + }, + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: DefaultResourceLock, + ResourceName: overwriteLeaderElectionID, + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + }, + }, + "should not set LeaderElectionID": { + original: &Configuration{ + ControllerManager: ControllerManager{ + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(false), + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ + Port: ptr.To(DefaultWebhookPort), + }, + Metrics: ControllerMetrics{ + BindAddress: DefaultMetricsBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: DefaultHealthProbeBindAddress, + }, + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(false), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: DefaultResourceLock, + ResourceName: DefaultLeaderElectionID, + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + }, + }, + "defaulting InternalCertManagement": { + original: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + }, + want: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(true), + WebhookServiceName: ptr.To(DefaultWebhookServiceName), + WebhookSecretName: ptr.To(DefaultWebhookSecretName), + }, + ClientConnection: defaultClientConnection, + }, + }, + "should not default InternalCertManagement": { + original: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + }, + }, + "should not default values in custom ClientConnection": { + original: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: &ClientConnection{ + QPS: ptr.To[float32](123.0), + Burst: ptr.To[int32](456), + }, + }, + want: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: &ClientConnection{ + QPS: ptr.To[float32](123.0), + Burst: ptr.To[int32](456), + }, + }, + }, + "should default empty custom ClientConnection": { + original: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: &ClientConnection{}, + }, + want: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + SetDefaults_Configuration(tc.original) + if diff := cmp.Diff(tc.want, tc.original); diff != "" { + t.Errorf("unexpected error (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/api/config/v1alpha1/groupversion_info.go b/api/config/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..f20810980 --- /dev/null +++ b/api/config/v1alpha1/groupversion_info.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the jobset v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=jobset.x-k8s.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "config.jobset.x-k8s.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // localSchemeBuilder is used to register autogenerated conversion and defaults functions + // It is required by ./zz_generated.conversion.go and ./zz_generated.defaults.go + localSchemeBuilder = &SchemeBuilder.SchemeBuilder + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +func init() { + SchemeBuilder.Register(&Configuration{}) + localSchemeBuilder.Register(RegisterDefaults) +} diff --git a/api/config/v1alpha1/zz_generated.deepcopy.go b/api/config/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..aef79ea41 --- /dev/null +++ b/api/config/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,191 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2023 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientConnection) DeepCopyInto(out *ClientConnection) { + *out = *in + if in.QPS != nil { + in, out := &in.QPS, &out.QPS + *out = new(float32) + **out = **in + } + if in.Burst != nil { + in, out := &in.Burst, &out.Burst + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientConnection. +func (in *ClientConnection) DeepCopy() *ClientConnection { + if in == nil { + return nil + } + out := new(ClientConnection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Namespace != nil { + in, out := &in.Namespace, &out.Namespace + *out = new(string) + **out = **in + } + in.ControllerManager.DeepCopyInto(&out.ControllerManager) + if in.InternalCertManagement != nil { + in, out := &in.InternalCertManagement, &out.InternalCertManagement + *out = new(InternalCertManagement) + (*in).DeepCopyInto(*out) + } + if in.ClientConnection != nil { + in, out := &in.ClientConnection, &out.ClientConnection + *out = new(ClientConnection) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Configuration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerHealth) DeepCopyInto(out *ControllerHealth) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerHealth. +func (in *ControllerHealth) DeepCopy() *ControllerHealth { + if in == nil { + return nil + } + out := new(ControllerHealth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerManager) DeepCopyInto(out *ControllerManager) { + *out = *in + in.Webhook.DeepCopyInto(&out.Webhook) + if in.LeaderElection != nil { + in, out := &in.LeaderElection, &out.LeaderElection + *out = new(configv1alpha1.LeaderElectionConfiguration) + (*in).DeepCopyInto(*out) + } + out.Metrics = in.Metrics + out.Health = in.Health +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManager. +func (in *ControllerManager) DeepCopy() *ControllerManager { + if in == nil { + return nil + } + out := new(ControllerManager) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerMetrics) DeepCopyInto(out *ControllerMetrics) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerMetrics. +func (in *ControllerMetrics) DeepCopy() *ControllerMetrics { + if in == nil { + return nil + } + out := new(ControllerMetrics) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerWebhook) DeepCopyInto(out *ControllerWebhook) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerWebhook. +func (in *ControllerWebhook) DeepCopy() *ControllerWebhook { + if in == nil { + return nil + } + out := new(ControllerWebhook) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalCertManagement) DeepCopyInto(out *InternalCertManagement) { + *out = *in + if in.Enable != nil { + in, out := &in.Enable, &out.Enable + *out = new(bool) + **out = **in + } + if in.WebhookServiceName != nil { + in, out := &in.WebhookServiceName, &out.WebhookServiceName + *out = new(string) + **out = **in + } + if in.WebhookSecretName != nil { + in, out := &in.WebhookSecretName, &out.WebhookSecretName + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalCertManagement. +func (in *InternalCertManagement) DeepCopy() *InternalCertManagement { + if in == nil { + return nil + } + out := new(InternalCertManagement) + in.DeepCopyInto(out) + return out +} diff --git a/api/config/v1alpha1/zz_generated.defaults.go b/api/config/v1alpha1/zz_generated.defaults.go new file mode 100644 index 000000000..c26912834 --- /dev/null +++ b/api/config/v1alpha1/zz_generated.defaults.go @@ -0,0 +1,34 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&Configuration{}, func(obj interface{}) { SetObjectDefaults_Configuration(obj.(*Configuration)) }) + return nil +} + +func SetObjectDefaults_Configuration(in *Configuration) { + SetDefaults_Configuration(in) +} diff --git a/config/components/crd/bases/jobset.x-k8s.io_configurations.yaml b/config/components/crd/bases/jobset.x-k8s.io_configurations.yaml new file mode 100644 index 000000000..f10734e33 --- /dev/null +++ b/config/components/crd/bases/jobset.x-k8s.io_configurations.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: configurations.jobset.x-k8s.io +spec: + group: jobset.x-k8s.io + names: + kind: Configuration + listKind: ConfigurationList + plural: configurations + singular: configuration + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Configuration is the Schema for the configurations API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ConfigurationSpec defines the desired state of Configuration + properties: + foo: + description: Foo is an example field of Configuration. Edit configuration_types.go + to remove/update + type: string + type: object + status: + description: ConfigurationStatus defines the observed state of Configuration + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 000000000..1f3f373e4 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,21 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/jobset.x-k8s.io_configurations.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_configurations.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_configurations.yaml +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 000000000..ec5c150a9 --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_configurations.yaml b/config/crd/patches/cainjection_in_configurations.yaml new file mode 100644 index 000000000..7218227ca --- /dev/null +++ b/config/crd/patches/cainjection_in_configurations.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: configurations.jobset.x-k8s.io diff --git a/config/crd/patches/webhook_in_configurations.yaml b/config/crd/patches/webhook_in_configurations.yaml new file mode 100644 index 000000000..69a1168f8 --- /dev/null +++ b/config/crd/patches/webhook_in_configurations.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: configurations.jobset.x-k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/configuration_editor_role.yaml b/config/rbac/configuration_editor_role.yaml new file mode 100644 index 000000000..d36432928 --- /dev/null +++ b/config/rbac/configuration_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit configurations. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: configuration-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/part-of: jobset + app.kubernetes.io/managed-by: kustomize + name: configuration-editor-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - configurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - configurations/status + verbs: + - get diff --git a/config/rbac/configuration_viewer_role.yaml b/config/rbac/configuration_viewer_role.yaml new file mode 100644 index 000000000..45ed5e0a7 --- /dev/null +++ b/config/rbac/configuration_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view configurations. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: configuration-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: jobset + app.kubernetes.io/part-of: jobset + app.kubernetes.io/managed-by: kustomize + name: configuration-viewer-role +rules: +- apiGroups: + - jobset.x-k8s.io + resources: + - configurations + verbs: + - get + - list + - watch +- apiGroups: + - jobset.x-k8s.io + resources: + - configurations/status + verbs: + - get diff --git a/config/samples/jobset_v1alpha1_configuration.yaml b/config/samples/jobset_v1alpha1_configuration.yaml new file mode 100644 index 000000000..602801836 --- /dev/null +++ b/config/samples/jobset_v1alpha1_configuration.yaml @@ -0,0 +1,12 @@ +apiVersion: jobset.x-k8s.io/v1alpha1 +kind: Configuration +metadata: + labels: + app.kubernetes.io/name: configuration + app.kubernetes.io/instance: configuration-sample + app.kubernetes.io/part-of: jobset + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: jobset + name: configuration-sample +spec: + # TODO(user): Add fields here diff --git a/main.go b/main.go index 1a45de09f..5ad09207e 100644 --- a/main.go +++ b/main.go @@ -33,10 +33,13 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" + configapi "sigs.k8s.io/jobset/api/config/v1alpha1" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" + "sigs.k8s.io/jobset/pkg/config" "sigs.k8s.io/jobset/pkg/controllers" "sigs.k8s.io/jobset/pkg/metrics" "sigs.k8s.io/jobset/pkg/util/cert" @@ -53,6 +56,8 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(jobset.AddToScheme(scheme)) + utilruntime.Must(configapi.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme } @@ -63,6 +68,7 @@ func main() { var qps float64 var burst int var featureGates string + var configFile string flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, @@ -71,6 +77,10 @@ func main() { flag.Float64Var(&qps, "kube-api-qps", 500, "Maximum QPS to use while talking with Kubernetes API") flag.IntVar(&burst, "kube-api-burst", 500, "Maximum burst for throttle while talking with Kubernetes API") flag.StringVar(&featureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") + flag.StringVar(&configFile, "config", "", + "The controller will load its initial configuration from this file. "+ + "Once configured, flags other than feature-gates will not take effect"+ + "Omit this flag to use the default configuration values. ") opts := zap.Options{ Development: true, @@ -80,10 +90,6 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - kubeConfig := ctrl.GetConfigOrDie() - kubeConfig.QPS = float32(qps) - kubeConfig.Burst = burst - if err := utilfeature.DefaultMutableFeatureGate.Set(featureGates); err != nil { setupLog.Error(err, "Unable to set flag gates for known features") os.Exit(1) @@ -91,30 +97,48 @@ func main() { metrics.Register() - mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ - Scheme: scheme, - Metrics: server.Options{ - BindAddress: metricsAddr, - }, - WebhookServer: webhook.NewServer( - webhook.Options{ - Port: 9443, - }), - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "6d4f6a47.x-k8s.io", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, - }) + var options manager.Options + + kubeConfig := ctrl.GetConfigOrDie() + if configFile == "" { + options = ctrl.Options{ + Scheme: scheme, + Metrics: server.Options{ + BindAddress: metricsAddr, + }, + WebhookServer: webhook.NewServer( + webhook.Options{ + Port: 9443, + }), + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "6d4f6a47.x-k8s.io", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + } + kubeConfig.QPS = float32(qps) + kubeConfig.Burst = burst + } else { + opts, cfg, err := apply(configFile) + if err != nil { + setupLog.Error(err, "Unable to load the configuration") + os.Exit(1) + } + options = opts + kubeConfig.QPS = *cfg.ClientConnection.QPS + kubeConfig.Burst = int(*cfg.ClientConnection.Burst) + } + + mgr, err := ctrl.NewManager(kubeConfig, options) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) @@ -218,3 +242,16 @@ func setupHealthzAndReadyzCheck(mgr ctrl.Manager, certsReady <-chan struct{}) { os.Exit(1) } } + +func apply(configFile string) (ctrl.Options, configapi.Configuration, error) { + options, cfg, err := config.Load(scheme, configFile) + if err != nil { + return options, cfg, err + } + cfgStr, err := config.Encode(scheme, &cfg) + if err != nil { + return options, cfg, err + } + setupLog.Info("Successfully loaded configuration", "config", cfgStr) + return options, cfg, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 000000000..7c8508c58 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,139 @@ +package config + +import ( + "bytes" + "fmt" + "os" + + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + configapi "sigs.k8s.io/jobset/api/config/v1alpha1" +) + +func fromFile(path string, scheme *runtime.Scheme, cfg *configapi.Configuration) error { + content, err := os.ReadFile(path) + if err != nil { + return err + } + + codecs := serializer.NewCodecFactory(scheme, serializer.EnableStrict) + + // Regardless of if the bytes are of any external version, + // it will be read successfully and converted into the internal version + return runtime.DecodeInto(codecs.UniversalDecoder(), content, cfg) +} + +func addTo(o *ctrl.Options, cfg *configapi.Configuration) { + addLeaderElectionTo(o, cfg) + + if o.Metrics.BindAddress == "" && cfg.Metrics.BindAddress != "" { + o.Metrics.BindAddress = cfg.Metrics.BindAddress + } + + if o.HealthProbeBindAddress == "" && cfg.Health.HealthProbeBindAddress != "" { + o.HealthProbeBindAddress = cfg.Health.HealthProbeBindAddress + } + + if o.ReadinessEndpointName == "" && cfg.Health.ReadinessEndpointName != "" { + o.ReadinessEndpointName = cfg.Health.ReadinessEndpointName + } + + if o.LivenessEndpointName == "" && cfg.Health.LivenessEndpointName != "" { + o.LivenessEndpointName = cfg.Health.LivenessEndpointName + } + + if o.WebhookServer == nil && cfg.Webhook.Port != nil { + wo := webhook.Options{} + if cfg.Webhook.Port != nil { + wo.Port = *cfg.Webhook.Port + } + if cfg.Webhook.Host != "" { + wo.Host = cfg.Webhook.Host + } + + if cfg.Webhook.CertDir != "" { + wo.CertDir = cfg.Webhook.CertDir + } + o.WebhookServer = webhook.NewServer(wo) + } +} + +func addLeaderElectionTo(o *ctrl.Options, cfg *configapi.Configuration) { + if cfg.LeaderElection == nil { + // The source does not have any configuration; noop + return + } + + if !o.LeaderElection && cfg.LeaderElection.LeaderElect != nil { + o.LeaderElection = *cfg.LeaderElection.LeaderElect + } + + if o.LeaderElectionResourceLock == "" && cfg.LeaderElection.ResourceLock != "" { + o.LeaderElectionResourceLock = cfg.LeaderElection.ResourceLock + } + + if o.LeaderElectionNamespace == "" && cfg.LeaderElection.ResourceNamespace != "" { + o.LeaderElectionNamespace = cfg.LeaderElection.ResourceNamespace + } + + if o.LeaderElectionID == "" && cfg.LeaderElection.ResourceName != "" { + o.LeaderElectionID = cfg.LeaderElection.ResourceName + } + + if o.LeaseDuration == nil && !equality.Semantic.DeepEqual(cfg.LeaderElection.LeaseDuration, metav1.Duration{}) { + o.LeaseDuration = &cfg.LeaderElection.LeaseDuration.Duration + } + + if o.RenewDeadline == nil && !equality.Semantic.DeepEqual(cfg.LeaderElection.RenewDeadline, metav1.Duration{}) { + o.RenewDeadline = &cfg.LeaderElection.RenewDeadline.Duration + } + + if o.RetryPeriod == nil && !equality.Semantic.DeepEqual(cfg.LeaderElection.RetryPeriod, metav1.Duration{}) { + o.RetryPeriod = &cfg.LeaderElection.RetryPeriod.Duration + } +} + +func Encode(scheme *runtime.Scheme, cfg *configapi.Configuration) (string, error) { + codecs := serializer.NewCodecFactory(scheme) + const mediaType = runtime.ContentTypeYAML + info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) + if !ok { + return "", fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType) + } + + encoder := codecs.EncoderForVersion(info.Serializer, configapi.GroupVersion) + buf := new(bytes.Buffer) + if err := encoder.Encode(cfg, buf); err != nil { + return "", err + } + return buf.String(), nil +} + +// Load returns a set of controller options and configuration from the given file, if the config file path is empty +// it used the default configapi values. +func Load(scheme *runtime.Scheme, configFile string) (ctrl.Options, configapi.Configuration, error) { + var err error + options := ctrl.Options{ + Scheme: scheme, + } + + cfg := configapi.Configuration{} + if configFile == "" { + scheme.Default(&cfg) + } else { + err := fromFile(configFile, scheme, &cfg) + if err != nil { + return options, cfg, err + } + } + if err := validate(&cfg).ToAggregate(); err != nil { + return options, cfg, err + } + addTo(&options, &cfg) + return options, cfg, err +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go new file mode 100644 index 000000000..b81c9cfd8 --- /dev/null +++ b/pkg/config/config_test.go @@ -0,0 +1,509 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + configapi "sigs.k8s.io/jobset/api/config/v1alpha1" +) + +func TestLoad(t *testing.T) { + testScheme := runtime.NewScheme() + err := configapi.AddToScheme(testScheme) + if err != nil { + t.Fatal(err) + } + + tmpDir := t.TempDir() + + namespaceOverWriteConfig := filepath.Join(tmpDir, "namespace-overwrite.yaml") + if err := os.WriteFile(namespaceOverWriteConfig, []byte(` +apiVersion: config.jobset.x-k8s.io/v1alpha1 +kind: Configuration +namespace: jobset-tenant-a +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: 6d4f6a47.jobset.x-k8s.io +webhook: + port: 9443 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + ctrlManagerConfigSpecOverWriteConfig := filepath.Join(tmpDir, "ctrl-manager-config-spec-overwrite.yaml") + if err := os.WriteFile(ctrlManagerConfigSpecOverWriteConfig, []byte(` +apiVersion: config.jobset.x-k8s.io/v1alpha1 +kind: Configuration +namespace: jobset-system +health: + healthProbeBindAddress: :38081 +metrics: + bindAddress: :38080 +leaderElection: + leaderElect: true + resourceName: test-id +webhook: + port: 9444 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + certOverWriteConfig := filepath.Join(tmpDir, "cert-overwrite.yaml") + if err := os.WriteFile(certOverWriteConfig, []byte(` +apiVersion: config.jobset.x-k8s.io/v1alpha1 +kind: Configuration +namespace: jobset-system +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: 6d4f6a47.jobset.x-k8s.io +webhook: + port: 9443 +internalCertManagement: + enable: true + webhookServiceName: jobset-tenant-a-webhook-service + webhookSecretName: jobset-tenant-a-webhook-server-cert +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + disableCertOverWriteConfig := filepath.Join(tmpDir, "disable-cert-overwrite.yaml") + if err := os.WriteFile(disableCertOverWriteConfig, []byte(` +apiVersion: config.jobset.x-k8s.io/v1alpha1 +kind: Configuration +namespace: jobset-system +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: 6d4f6a47.jobset.x-k8s.io +webhook: + port: 9443 +internalCertManagement: + enable: false +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + leaderElectionDisabledConfig := filepath.Join(tmpDir, "leaderElection-disabled.yaml") + if err := os.WriteFile(leaderElectionDisabledConfig, []byte(` +apiVersion: config.jobset.x-k8s.io/v1alpha1 +kind: Configuration +namespace: jobset-system +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: false +webhook: + port: 9443 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + clientConnectionConfig := filepath.Join(tmpDir, "clientConnection.yaml") + if err := os.WriteFile(clientConnectionConfig, []byte(` +apiVersion: config.jobset.x-k8s.io/v1alpha1 +kind: Configuration +namespace: jobset-system +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: 6d4f6a47.jobset.x-k8s.io +webhook: + port: 9443 +clientConnection: + qps: 50 + burst: 100 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + invalidConfig := filepath.Join(tmpDir, "invalid-config.yaml") + if err := os.WriteFile(invalidConfig, []byte(` +apiVersion: config.jobset.x-k8s.io/v1alpha1 +kind: Configuration +namespaces: jobset-system +invalidField: invalidValue +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: :8080 +leaderElection: + leaderElect: true + resourceName: 6d4f6a47.jobset.x-k8s.io +webhook: + port: 9443 +`), os.FileMode(0600)); err != nil { + t.Fatal(err) + } + + defaultControlOptions := ctrl.Options{ + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + Metrics: metricsserver.Options{ + BindAddress: configapi.DefaultMetricsBindAddress, + }, + LeaderElection: true, + LeaderElectionID: configapi.DefaultLeaderElectionID, + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaseDuration: ptr.To(configapi.DefaultLeaderElectionLeaseDuration), + RenewDeadline: ptr.To(configapi.DefaultLeaderElectionRenewDeadline), + RetryPeriod: ptr.To(configapi.DefaultLeaderElectionRetryPeriod), + WebhookServer: &webhook.DefaultServer{ + Options: webhook.Options{ + Port: configapi.DefaultWebhookPort, + }, + }, + } + + enableDefaultInternalCertManagement := &configapi.InternalCertManagement{ + Enable: ptr.To(true), + WebhookServiceName: ptr.To(configapi.DefaultWebhookServiceName), + WebhookSecretName: ptr.To(configapi.DefaultWebhookSecretName), + } + + ctrlOptsCmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported(ctrl.Options{}), + cmpopts.IgnoreUnexported(webhook.DefaultServer{}), + cmpopts.IgnoreUnexported(ctrlcache.Options{}), + cmpopts.IgnoreFields(ctrl.Options{}, "Scheme", "Logger"), + } + + // Ignore the controller manager section since it's side effect is checked against + // the content of the resulting options + configCmpOpts := []cmp.Option{ + cmpopts.IgnoreFields(configapi.Configuration{}, "ControllerManager"), + } + + defaultClientConnection := &configapi.ClientConnection{ + QPS: ptr.To[float32](configapi.DefaultClientConnectionQPS), + Burst: ptr.To[int32](configapi.DefaultClientConnectionBurst), + } + + testcases := []struct { + name string + configFile string + wantConfiguration configapi.Configuration + wantOptions ctrl.Options + wantError error + }{ + { + name: "default config", + configFile: "", + wantConfiguration: configapi.Configuration{ + Namespace: ptr.To(configapi.DefaultNamespace), + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + }, + wantOptions: ctrl.Options{ + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + Metrics: metricsserver.Options{ + BindAddress: configapi.DefaultMetricsBindAddress, + }, + LeaderElection: true, + LeaderElectionID: configapi.DefaultLeaderElectionID, + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaseDuration: ptr.To(configapi.DefaultLeaderElectionLeaseDuration), + RenewDeadline: ptr.To(configapi.DefaultLeaderElectionRenewDeadline), + RetryPeriod: ptr.To(configapi.DefaultLeaderElectionRetryPeriod), + WebhookServer: &webhook.DefaultServer{ + Options: webhook.Options{ + Port: configapi.DefaultWebhookPort, + }, + }, + }, + }, + { + name: "bad path", + configFile: ".", + wantError: &fs.PathError{ + Op: "read", + Path: ".", + Err: errors.New("is a directory"), + }, + }, + { + name: "namespace overwrite config", + configFile: namespaceOverWriteConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: ptr.To("jobset-tenant-a"), + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + }, + wantOptions: defaultControlOptions, + }, + { + name: "ControllerManagerConfigurationSpec overwrite config", + configFile: ctrlManagerConfigSpecOverWriteConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: ptr.To(configapi.DefaultNamespace), + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + }, + wantOptions: ctrl.Options{ + HealthProbeBindAddress: ":38081", + Metrics: metricsserver.Options{ + BindAddress: ":38080", + }, + LeaderElection: true, + LeaderElectionID: "test-id", + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaseDuration: ptr.To(configapi.DefaultLeaderElectionLeaseDuration), + RenewDeadline: ptr.To(configapi.DefaultLeaderElectionRenewDeadline), + RetryPeriod: ptr.To(configapi.DefaultLeaderElectionRetryPeriod), + WebhookServer: &webhook.DefaultServer{ + Options: webhook.Options{ + Port: 9444, + }, + }, + }, + }, + { + name: "cert options overwrite config", + configFile: certOverWriteConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: ptr.To(configapi.DefaultNamespace), + InternalCertManagement: &configapi.InternalCertManagement{ + Enable: ptr.To(true), + WebhookServiceName: ptr.To("jobset-tenant-a-webhook-service"), + WebhookSecretName: ptr.To("jobset-tenant-a-webhook-server-cert"), + }, + ClientConnection: defaultClientConnection, + }, + wantOptions: defaultControlOptions, + }, + { + name: "disable cert overwrite config", + configFile: disableCertOverWriteConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: ptr.To(configapi.DefaultNamespace), + InternalCertManagement: &configapi.InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + }, + wantOptions: defaultControlOptions, + }, + { + name: "leaderElection disabled config", + configFile: leaderElectionDisabledConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: ptr.To("jobset-system"), + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: defaultClientConnection, + }, + wantOptions: ctrl.Options{ + HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, + Metrics: metricsserver.Options{ + BindAddress: configapi.DefaultMetricsBindAddress, + }, + LeaderElectionID: configapi.DefaultLeaderElectionID, + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaseDuration: ptr.To(configapi.DefaultLeaderElectionLeaseDuration), + RenewDeadline: ptr.To(configapi.DefaultLeaderElectionRenewDeadline), + RetryPeriod: ptr.To(configapi.DefaultLeaderElectionRetryPeriod), + LeaderElection: false, + WebhookServer: &webhook.DefaultServer{ + Options: webhook.Options{ + Port: configapi.DefaultWebhookPort, + }, + }, + }, + }, + { + name: "clientConnection config", + configFile: clientConnectionConfig, + wantConfiguration: configapi.Configuration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: configapi.GroupVersion.String(), + Kind: "Configuration", + }, + Namespace: ptr.To(configapi.DefaultNamespace), + InternalCertManagement: enableDefaultInternalCertManagement, + ClientConnection: &configapi.ClientConnection{ + QPS: ptr.To[float32](50), + Burst: ptr.To[int32](100), + }, + }, + wantOptions: defaultControlOptions, + }, + { + name: "invalid config", + configFile: invalidConfig, + wantError: runtime.NewStrictDecodingError([]error{ + errors.New("unknown field \"invalidField\""), + errors.New("unknown field \"namespaces\""), + }), + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + options, cfg, err := Load(testScheme, tc.configFile) + if tc.wantError == nil { + if err != nil { + t.Errorf("Unexpected error:%s", err) + } + if diff := cmp.Diff(tc.wantConfiguration, cfg, configCmpOpts...); diff != "" { + t.Errorf("Unexpected config (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tc.wantOptions, options, ctrlOptsCmpOpts...); diff != "" { + t.Errorf("Unexpected options (-want +got):\n%s", diff) + } + } else { + if diff := cmp.Diff(tc.wantError.Error(), err.Error()); diff != "" { + t.Errorf("Unexpected error (-want +got):\n%s", diff) + } + } + }) + } +} + +func TestEncode(t *testing.T) { + testScheme := runtime.NewScheme() + err := configapi.AddToScheme(testScheme) + if err != nil { + t.Fatal(err) + } + + defaultConfig := &configapi.Configuration{} + testScheme.Default(defaultConfig) + + testcases := []struct { + name string + scheme *runtime.Scheme + cfg *configapi.Configuration + wantResult map[string]any + }{ + + { + name: "empty", + scheme: testScheme, + cfg: &configapi.Configuration{}, + wantResult: map[string]any{ + "apiVersion": "config.jobset.x-k8s.io/v1alpha1", + "kind": "Configuration", + "health": map[string]any{}, + "metrics": map[string]any{}, + "webhook": map[string]any{}, + }, + }, + { + name: "default", + scheme: testScheme, + cfg: defaultConfig, + wantResult: map[string]any{ + "apiVersion": "config.jobset.x-k8s.io/v1alpha1", + "kind": "Configuration", + "namespace": configapi.DefaultNamespace, + "webhook": map[string]any{ + "port": int64(configapi.DefaultWebhookPort), + }, + "metrics": map[string]any{ + "bindAddress": configapi.DefaultMetricsBindAddress, + }, + "health": map[string]any{ + "healthProbeBindAddress": configapi.DefaultHealthProbeBindAddress, + }, + "leaderElection": map[string]any{ + "leaderElect": true, + "leaseDuration": configapi.DefaultLeaderElectionLeaseDuration.String(), + "renewDeadline": configapi.DefaultLeaderElectionRenewDeadline.String(), + "retryPeriod": configapi.DefaultLeaderElectionRetryPeriod.String(), + "resourceLock": resourcelock.LeasesResourceLock, + "resourceName": configapi.DefaultLeaderElectionID, + "resourceNamespace": "", + }, + "internalCertManagement": map[string]any{ + "enable": true, + "webhookServiceName": configapi.DefaultWebhookServiceName, + "webhookSecretName": configapi.DefaultWebhookSecretName, + }, + "clientConnection": map[string]any{ + "burst": int64(configapi.DefaultClientConnectionBurst), + "qps": int64(configapi.DefaultClientConnectionQPS), + }, + }, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := Encode(tc.scheme, tc.cfg) + if err != nil { + t.Errorf("Unexpected error:%s", err) + } + gotMap := map[string]interface{}{} + err = yaml.Unmarshal([]byte(got), &gotMap) + if err != nil { + t.Errorf("Unable to unmarshal result:%s", err) + } + if diff := cmp.Diff(tc.wantResult, gotMap); diff != "" { + t.Errorf("Unexpected result (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/config/validation.go b/pkg/config/validation.go new file mode 100644 index 000000000..a3c86349a --- /dev/null +++ b/pkg/config/validation.go @@ -0,0 +1,39 @@ +package config + +import ( + "strings" + + apimachineryvalidation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" + + configapi "sigs.k8s.io/jobset/api/config/v1alpha1" +) + +var ( + internalCertManagementPath = field.NewPath("internalCertManagement") +) + +func validate(c *configapi.Configuration) field.ErrorList { + var allErrs field.ErrorList + allErrs = append(allErrs, validateInternalCertManagement(c)...) + return allErrs +} + +func validateInternalCertManagement(c *configapi.Configuration) field.ErrorList { + var allErrs field.ErrorList + if c.InternalCertManagement == nil || !ptr.Deref(c.InternalCertManagement.Enable, false) { + return allErrs + } + if svcName := c.InternalCertManagement.WebhookServiceName; svcName != nil { + if errs := apimachineryvalidation.IsDNS1035Label(*svcName); len(errs) != 0 { + allErrs = append(allErrs, field.Invalid(internalCertManagementPath.Child("webhookServiceName"), svcName, strings.Join(errs, ","))) + } + } + if secName := c.InternalCertManagement.WebhookSecretName; secName != nil { + if errs := apimachineryvalidation.IsDNS1123Subdomain(*secName); len(errs) != 0 { + allErrs = append(allErrs, field.Invalid(internalCertManagementPath.Child("webhookSecretName"), secName, strings.Join(errs, ","))) + } + } + return allErrs +} diff --git a/pkg/config/validation_test.go b/pkg/config/validation_test.go new file mode 100644 index 000000000..44520d09c --- /dev/null +++ b/pkg/config/validation_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" + + configapi "sigs.k8s.io/jobset/api/config/v1alpha1" +) + +func TestValidate(t *testing.T) { + testCases := map[string]struct { + cfg *configapi.Configuration + wantErr field.ErrorList + }{ + "invalid .internalCertManagement.webhookSecretName": { + cfg: &configapi.Configuration{ + InternalCertManagement: &configapi.InternalCertManagement{ + Enable: ptr.To(true), + WebhookSecretName: ptr.To(":)"), + }, + }, + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "internalCertManagement.webhookSecretName", + }, + }, + }, + "invalid .internalCertManagement.webhookServiceName": { + cfg: &configapi.Configuration{ + InternalCertManagement: &configapi.InternalCertManagement{ + Enable: ptr.To(true), + WebhookServiceName: ptr.To("0-invalid"), + }, + }, + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "internalCertManagement.webhookServiceName", + }, + }, + }, + "disabled .internalCertManagement with invalid .internalCertManagement.webhookServiceName": { + cfg: &configapi.Configuration{ + InternalCertManagement: &configapi.InternalCertManagement{ + Enable: ptr.To(false), + WebhookServiceName: ptr.To("0-invalid"), + }, + }, + }, + "valid .internalCertManagement": { + cfg: &configapi.Configuration{ + InternalCertManagement: &configapi.InternalCertManagement{ + Enable: ptr.To(true), + WebhookServiceName: ptr.To("webhook-svc"), + WebhookSecretName: ptr.To("webhook-sec"), + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + if diff := cmp.Diff(tc.wantErr, validate(tc.cfg), cmpopts.IgnoreFields(field.Error{}, "BadValue", "Detail")); diff != "" { + t.Errorf("Unexpected returned error (-want,+got):\n%s", diff) + } + }) + } +}