From fce60873e0c35d36bf9dba3f3ac44818781eb14e Mon Sep 17 00:00:00 2001 From: safronovD Date: Mon, 7 Jun 2021 16:58:41 +0300 Subject: [PATCH 1/7] increase limit Signed-off-by: safronovD --- charts/csi-baremetal-operator/templates/manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/csi-baremetal-operator/templates/manager.yaml b/charts/csi-baremetal-operator/templates/manager.yaml index 69dd6b40..7db25857 100644 --- a/charts/csi-baremetal-operator/templates/manager.yaml +++ b/charts/csi-baremetal-operator/templates/manager.yaml @@ -29,7 +29,7 @@ spec: resources: limits: cpu: 100m - memory: 30Mi + memory: 100Mi requests: cpu: 100m memory: 20Mi From 3e4b4c0fb2f5dc504966d03da12bb770ac3c9ac6 Mon Sep 17 00:00:00 2001 From: safronovD Date: Mon, 7 Jun 2021 17:46:57 +0300 Subject: [PATCH 2/7] fixes Signed-off-by: safronovD --- charts/csi-baremetal-operator/templates/rbac.yaml | 6 ++++++ pkg/scheduler_patcher_openshift.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/csi-baremetal-operator/templates/rbac.yaml b/charts/csi-baremetal-operator/templates/rbac.yaml index 54532380..6cde3fd9 100644 --- a/charts/csi-baremetal-operator/templates/rbac.yaml +++ b/charts/csi-baremetal-operator/templates/rbac.yaml @@ -38,6 +38,12 @@ rules: - "*" verbs: - "*" +- apiGroups: + - config.openshift.io + resources: + - schedulers + verbs: + - "*" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/pkg/scheduler_patcher_openshift.go b/pkg/scheduler_patcher_openshift.go index cc0e8b09..3eb0da02 100644 --- a/pkg/scheduler_patcher_openshift.go +++ b/pkg/scheduler_patcher_openshift.go @@ -21,7 +21,7 @@ const ( "apiVersion" : "v1", "extenders": [ { - "urlPrefix": "http://127.0.0.1:$PORT", + "urlPrefix": "http://127.0.0.1:8889", "filterVerb": "filter", "enableHttps": false, "nodeCacheCapable": false, From e544d901c80dec767738a6113dea23f75e013a6b Mon Sep 17 00:00:00 2001 From: safronovD Date: Tue, 8 Jun 2021 10:51:39 +0300 Subject: [PATCH 3/7] fix comments Signed-off-by: safronovD --- api/v1/components/patcher.go | 1 + .../configmap/patcher-configmap.yaml | 4 +-- .../csi-baremetal_v1_deployment.yaml | 1 + charts/csi-baremetal-deployment/values.yaml | 1 + .../templates/rbac.yaml | 2 ++ pkg/csi_deployment.go | 2 +- pkg/scheduler_patcher_openshift.go | 27 ++++++++++--------- 7 files changed, 23 insertions(+), 15 deletions(-) diff --git a/api/v1/components/patcher.go b/api/v1/components/patcher.go index 02c62aa2..d14f3103 100644 --- a/api/v1/components/patcher.go +++ b/api/v1/components/patcher.go @@ -23,4 +23,5 @@ type Patcher struct { Interval int `json:"interval,omitempty"` RestoreOnShutdown bool `json:"restoreOnShutdown,omitempty"` ConfigMapName string `json:"configMapName,omitempty"` + UrlPrefix string `json:"urlPrefix,omitempty"` } diff --git a/charts/csi-baremetal-deployment/templates/configmap/patcher-configmap.yaml b/charts/csi-baremetal-deployment/templates/configmap/patcher-configmap.yaml index 02af3d5a..82ff60a9 100644 --- a/charts/csi-baremetal-deployment/templates/configmap/patcher-configmap.yaml +++ b/charts/csi-baremetal-deployment/templates/configmap/patcher-configmap.yaml @@ -24,7 +24,7 @@ data: apiVersion: v1 kind: Policy extenders: - - urlPrefix: "http://127.0.0.1:8889" + - urlPrefix: {{ .Values.scheduler.patcher.url_prefix | quote }} filterVerb: filter prioritizeVerb: prioritize weight: 1 @@ -38,7 +38,7 @@ data: apiVersion: kubescheduler.config.k8s.io/v1beta1 kind: KubeSchedulerConfiguration extenders: - - urlPrefix: "http://127.0.0.1:8889" + - urlPrefix: {{ .Values.scheduler.patcher.url_prefix | quote }} filterVerb: filter prioritizeVerb: prioritize weight: 1 diff --git a/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml b/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml index 96b1124e..4ebd3e4b 100644 --- a/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml +++ b/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml @@ -99,6 +99,7 @@ spec: interval: {{ .Values.scheduler.patcher.interval }} restoreOnShutdown: {{ .Values.scheduler.patcher.restore_on_shutdown }} configMapName: {{ .Values.scheduler.patcher.config_map_name }} + urlPrefix: {{ .Values.scheduler.patcher.url_prefix }} storageProvisioner: {{ .Values.scheduler.provisioner }} nodeController: enable: {{ .Values.nodeController.enable }} diff --git a/charts/csi-baremetal-deployment/values.yaml b/charts/csi-baremetal-deployment/values.yaml index fd7cb2a0..7ceb1b98 100644 --- a/charts/csi-baremetal-deployment/values.yaml +++ b/charts/csi-baremetal-deployment/values.yaml @@ -124,6 +124,7 @@ scheduler: image: tag: + url_prefix: http://127.0.0.1:8889 kubeconfig: /etc/kubernetes/scheduler.conf interval: 60 restore_on_shutdown: true diff --git a/charts/csi-baremetal-operator/templates/rbac.yaml b/charts/csi-baremetal-operator/templates/rbac.yaml index 6cde3fd9..05229a3a 100644 --- a/charts/csi-baremetal-operator/templates/rbac.yaml +++ b/charts/csi-baremetal-operator/templates/rbac.yaml @@ -38,12 +38,14 @@ rules: - "*" verbs: - "*" +{{- if eq .Values.platform "openshift" }} - apiGroups: - config.openshift.io resources: - schedulers verbs: - "*" +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/pkg/csi_deployment.go b/pkg/csi_deployment.go index 48e1ac81..1832985a 100644 --- a/pkg/csi_deployment.go +++ b/pkg/csi_deployment.go @@ -73,7 +73,7 @@ func (c *CSIDeployment) Update(ctx context.Context, csi *csibaremetalv1.Deployme // Patching method for the scheduler depends on the platform switch csi.Spec.Platform { case platformOpenshift: - return c.patcher.PatchOpenShift(ctx, scheme) + return c.patcher.PatchOpenShift(ctx, csi) default: return c.patcher.Update(csi, scheme) diff --git a/pkg/scheduler_patcher_openshift.go b/pkg/scheduler_patcher_openshift.go index 3eb0da02..79ee9c1a 100644 --- a/pkg/scheduler_patcher_openshift.go +++ b/pkg/scheduler_patcher_openshift.go @@ -2,36 +2,39 @@ package pkg import ( "context" + "fmt" openshiftv1 "github.com/openshift/api/config/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + + csibaremetalv1 "github.com/dell/csi-baremetal-operator/api/v1" ) const ( openshiftNS = "openshift-config" openshiftConfig = "scheduler-policy" - oshiftpolicyFile = "policy.cfg" - oshiftpolicy = `{ + openshiftPolicyFile = "policy.cfg" +) + +func (p *SchedulerPatcher) PatchOpenShift(ctx context.Context, csi *csibaremetalv1.Deployment) error { + openshiftPolicy := fmt.Sprintf(`{ "kind" : "Policy", "apiVersion" : "v1", "extenders": [ { - "urlPrefix": "http://127.0.0.1:8889", + "urlPrefix": "%s", "filterVerb": "filter", "enableHttps": false, "nodeCacheCapable": false, "ignorable": true } ] -}` -) +}`, csi.Spec.Scheduler.Patcher.UrlPrefix) -func (p *SchedulerPatcher) PatchOpenShift(ctx context.Context, scheme *runtime.Scheme) error { cfClient := p.CoreV1().ConfigMaps(openshiftNS) oscf, err := cfClient.Get(p.ctx, openshiftConfig, metav1.GetOptions{}) if err != nil { @@ -40,8 +43,8 @@ func (p *SchedulerPatcher) PatchOpenShift(ctx context.Context, scheme *runtime.S return err } } else { - if v, ok := oscf.Data[oshiftpolicyFile]; ok { - if v == oshiftpolicy { + if v, ok := oscf.Data[openshiftPolicyFile]; ok { + if v == openshiftPolicy { p.Logger.Info("Configmap is already patched") return nil } @@ -54,7 +57,7 @@ func (p *SchedulerPatcher) PatchOpenShift(ctx context.Context, scheme *runtime.S } } - _, err = cfClient.Create(p.ctx, createOpenshiftConfig(), metav1.CreateOptions{}) + _, err = cfClient.Create(p.ctx, createOpenshiftConfig(openshiftPolicy), metav1.CreateOptions{}) if err != nil { p.Logger.Error(err, "Failed to create configmap") return err @@ -79,14 +82,14 @@ func (p *SchedulerPatcher) UnPatchOpenShift(ctx context.Context) error { return p.unpatchSheduler(ctx) } -func createOpenshiftConfig() *corev1.ConfigMap { +func createOpenshiftConfig(policy string) *corev1.ConfigMap { return &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Name: openshiftConfig, Namespace: openshiftNS, }, - Data: map[string]string{oshiftpolicyFile: oshiftpolicy}, + Data: map[string]string{openshiftPolicyFile: policy}, } } From c19eef759a23b02a82a1e55c6875f5ed5936eee6 Mon Sep 17 00:00:00 2001 From: safronovD Date: Tue, 8 Jun 2021 11:33:05 +0300 Subject: [PATCH 4/7] fix comment Signed-off-by: safronovD --- api/v1/components/patcher.go | 2 +- .../templates/configmap/patcher-configmap.yaml | 4 ++-- .../templates/csi-baremetal_v1_deployment.yaml | 2 +- charts/csi-baremetal-deployment/values.yaml | 2 +- pkg/scheduler_patcher_openshift.go | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/v1/components/patcher.go b/api/v1/components/patcher.go index d14f3103..5b256af6 100644 --- a/api/v1/components/patcher.go +++ b/api/v1/components/patcher.go @@ -23,5 +23,5 @@ type Patcher struct { Interval int `json:"interval,omitempty"` RestoreOnShutdown bool `json:"restoreOnShutdown,omitempty"` ConfigMapName string `json:"configMapName,omitempty"` - UrlPrefix string `json:"urlPrefix,omitempty"` + ExtenderPort string `json:"extenderPort,omitempty"` } diff --git a/charts/csi-baremetal-deployment/templates/configmap/patcher-configmap.yaml b/charts/csi-baremetal-deployment/templates/configmap/patcher-configmap.yaml index 82ff60a9..b58ff12a 100644 --- a/charts/csi-baremetal-deployment/templates/configmap/patcher-configmap.yaml +++ b/charts/csi-baremetal-deployment/templates/configmap/patcher-configmap.yaml @@ -24,7 +24,7 @@ data: apiVersion: v1 kind: Policy extenders: - - urlPrefix: {{ .Values.scheduler.patcher.url_prefix | quote }} + - urlPrefix: "http://127.0.0.1:{{ .Values.scheduler.patcher.extender_port }}" filterVerb: filter prioritizeVerb: prioritize weight: 1 @@ -38,7 +38,7 @@ data: apiVersion: kubescheduler.config.k8s.io/v1beta1 kind: KubeSchedulerConfiguration extenders: - - urlPrefix: {{ .Values.scheduler.patcher.url_prefix | quote }} + - urlPrefix: "http://127.0.0.1:{{ .Values.scheduler.patcher.extender_port }}" filterVerb: filter prioritizeVerb: prioritize weight: 1 diff --git a/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml b/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml index 4ebd3e4b..fe4c403c 100644 --- a/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml +++ b/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml @@ -99,7 +99,7 @@ spec: interval: {{ .Values.scheduler.patcher.interval }} restoreOnShutdown: {{ .Values.scheduler.patcher.restore_on_shutdown }} configMapName: {{ .Values.scheduler.patcher.config_map_name }} - urlPrefix: {{ .Values.scheduler.patcher.url_prefix }} + extenderPort: {{ .Values.scheduler.patcher.extender_port | quote}} storageProvisioner: {{ .Values.scheduler.provisioner }} nodeController: enable: {{ .Values.nodeController.enable }} diff --git a/charts/csi-baremetal-deployment/values.yaml b/charts/csi-baremetal-deployment/values.yaml index 7ceb1b98..001a7961 100644 --- a/charts/csi-baremetal-deployment/values.yaml +++ b/charts/csi-baremetal-deployment/values.yaml @@ -124,7 +124,7 @@ scheduler: image: tag: - url_prefix: http://127.0.0.1:8889 + extender_port: 8889 kubeconfig: /etc/kubernetes/scheduler.conf interval: 60 restore_on_shutdown: true diff --git a/pkg/scheduler_patcher_openshift.go b/pkg/scheduler_patcher_openshift.go index 79ee9c1a..200e19da 100644 --- a/pkg/scheduler_patcher_openshift.go +++ b/pkg/scheduler_patcher_openshift.go @@ -26,14 +26,14 @@ func (p *SchedulerPatcher) PatchOpenShift(ctx context.Context, csi *csibaremetal "apiVersion" : "v1", "extenders": [ { - "urlPrefix": "%s", + "urlPrefix": "http://127.0.0.1:%s", "filterVerb": "filter", "enableHttps": false, "nodeCacheCapable": false, "ignorable": true } ] -}`, csi.Spec.Scheduler.Patcher.UrlPrefix) +}`, csi.Spec.Scheduler.Patcher.ExtenderPort) cfClient := p.CoreV1().ConfigMaps(openshiftNS) oscf, err := cfClient.Get(p.ctx, openshiftConfig, metav1.GetOptions{}) From 0b40b97f393c1325f7fb0b60453945a2ad6995e0 Mon Sep 17 00:00:00 2001 From: safronovD Date: Tue, 8 Jun 2021 12:02:44 +0300 Subject: [PATCH 5/7] fix rbac Signed-off-by: safronovD --- charts/csi-baremetal-operator/templates/rbac.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/csi-baremetal-operator/templates/rbac.yaml b/charts/csi-baremetal-operator/templates/rbac.yaml index 05229a3a..c3eb71be 100644 --- a/charts/csi-baremetal-operator/templates/rbac.yaml +++ b/charts/csi-baremetal-operator/templates/rbac.yaml @@ -38,14 +38,13 @@ rules: - "*" verbs: - "*" -{{- if eq .Values.platform "openshift" }} +# Required for Openshift only - apiGroups: - config.openshift.io resources: - schedulers verbs: - "*" -{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From d582b5afe2d1c910f72aecdbe219af149e05b339 Mon Sep 17 00:00:00 2001 From: safronovD Date: Tue, 8 Jun 2021 18:07:07 +0300 Subject: [PATCH 6/7] fix rbac Signed-off-by: safronovD --- api/v1/components/patcher.go | 1 - api/v1/components/scheduler.go | 1 + .../templates/csi-baremetal_v1_deployment.yaml | 2 +- charts/csi-baremetal-deployment/values.yaml | 4 +++- charts/csi-baremetal-operator/templates/manager.yaml | 1 + pkg/scheduler_patcher_openshift.go | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/v1/components/patcher.go b/api/v1/components/patcher.go index 5b256af6..02c62aa2 100644 --- a/api/v1/components/patcher.go +++ b/api/v1/components/patcher.go @@ -23,5 +23,4 @@ type Patcher struct { Interval int `json:"interval,omitempty"` RestoreOnShutdown bool `json:"restoreOnShutdown,omitempty"` ConfigMapName string `json:"configMapName,omitempty"` - ExtenderPort string `json:"extenderPort,omitempty"` } diff --git a/api/v1/components/scheduler.go b/api/v1/components/scheduler.go index 3ec032b2..f13d77ab 100644 --- a/api/v1/components/scheduler.go +++ b/api/v1/components/scheduler.go @@ -23,5 +23,6 @@ type Scheduler struct { Log *Log `json:"log,omitempty"` Metrics *Metrics `json:"metrics,omitempty"` Patcher *Patcher `json:"patcher,omitempty"` + ExtenderPort string `json:"extenderPort,omitempty"` StorageProvisioner string `json:"storageProvisioner"` } diff --git a/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml b/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml index fe4c403c..603cda08 100644 --- a/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml +++ b/charts/csi-baremetal-deployment/templates/csi-baremetal_v1_deployment.yaml @@ -83,6 +83,7 @@ spec: metrics: path: {{ .Values.scheduler.metrics.path }} port: {{ .Values.scheduler.metrics.port }} + extenderPort: {{ .Values.scheduler.extender.port | quote }} patcher: enable: {{ .Values.scheduler.patcher.enable }} image: @@ -99,7 +100,6 @@ spec: interval: {{ .Values.scheduler.patcher.interval }} restoreOnShutdown: {{ .Values.scheduler.patcher.restore_on_shutdown }} configMapName: {{ .Values.scheduler.patcher.config_map_name }} - extenderPort: {{ .Values.scheduler.patcher.extender_port | quote}} storageProvisioner: {{ .Values.scheduler.provisioner }} nodeController: enable: {{ .Values.nodeController.enable }} diff --git a/charts/csi-baremetal-deployment/values.yaml b/charts/csi-baremetal-deployment/values.yaml index 001a7961..83c30cc9 100644 --- a/charts/csi-baremetal-deployment/values.yaml +++ b/charts/csi-baremetal-deployment/values.yaml @@ -117,6 +117,9 @@ scheduler: path: /metrics port: 8787 + extender: + port: 8889 + # Patcher settings patcher: enable: false @@ -124,7 +127,6 @@ scheduler: image: tag: - extender_port: 8889 kubeconfig: /etc/kubernetes/scheduler.conf interval: 60 restore_on_shutdown: true diff --git a/charts/csi-baremetal-operator/templates/manager.yaml b/charts/csi-baremetal-operator/templates/manager.yaml index 7db25857..b50a7ee1 100644 --- a/charts/csi-baremetal-operator/templates/manager.yaml +++ b/charts/csi-baremetal-operator/templates/manager.yaml @@ -27,6 +27,7 @@ spec: name: manager imagePullPolicy: {{ .Values.image.pullPolicy }} resources: + # Memory limit is increased due to bug in OpenShift 4.6 - https://bugzilla.redhat.com/show_bug.cgi?id=1904558 limits: cpu: 100m memory: 100Mi diff --git a/pkg/scheduler_patcher_openshift.go b/pkg/scheduler_patcher_openshift.go index 200e19da..9f2b466c 100644 --- a/pkg/scheduler_patcher_openshift.go +++ b/pkg/scheduler_patcher_openshift.go @@ -33,7 +33,7 @@ func (p *SchedulerPatcher) PatchOpenShift(ctx context.Context, csi *csibaremetal "ignorable": true } ] -}`, csi.Spec.Scheduler.Patcher.ExtenderPort) +}`, csi.Spec.Scheduler.ExtenderPort) cfClient := p.CoreV1().ConfigMaps(openshiftNS) oscf, err := cfClient.Get(p.ctx, openshiftConfig, metav1.GetOptions{}) From 9b61d75c15f7c4bf48d6064d7d3e334323ecf482 Mon Sep 17 00:00:00 2001 From: safronovD Date: Wed, 9 Jun 2021 11:46:25 +0300 Subject: [PATCH 7/7] add TODO Signed-off-by: safronovD --- charts/csi-baremetal-deployment/values.yaml | 1 + charts/csi-baremetal-operator/templates/rbac.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/charts/csi-baremetal-deployment/values.yaml b/charts/csi-baremetal-deployment/values.yaml index 83c30cc9..03e91b2e 100644 --- a/charts/csi-baremetal-deployment/values.yaml +++ b/charts/csi-baremetal-deployment/values.yaml @@ -154,4 +154,5 @@ nodeController: path: /metrics port: 8787 +# TODO Move parameter to Operator chart - https://github.com/dell/csi-baremetal/issues/422 platform: "vanilla" diff --git a/charts/csi-baremetal-operator/templates/rbac.yaml b/charts/csi-baremetal-operator/templates/rbac.yaml index c3eb71be..fbf74d3e 100644 --- a/charts/csi-baremetal-operator/templates/rbac.yaml +++ b/charts/csi-baremetal-operator/templates/rbac.yaml @@ -39,6 +39,7 @@ rules: verbs: - "*" # Required for Openshift only +# TODO Deploy only when OpenShift - https://github.com/dell/csi-baremetal/issues/422 - apiGroups: - config.openshift.io resources: