From 6f4fd1825e318ba9327aab40b16d31d3d5cba51f Mon Sep 17 00:00:00 2001 From: dnskr Date: Mon, 12 Dec 2022 22:58:35 +0100 Subject: [PATCH 1/8] [K8S][HELM] Fix image tag assignment and compact Chart.yaml --- docker/helm/Chart.yaml | 27 +++----------------- docker/helm/templates/kyuubi-deployment.yaml | 2 +- docker/helm/values.yaml | 3 +-- 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/docker/helm/Chart.yaml b/docker/helm/Chart.yaml index 9c1a216a13d..d72530cca2d 100644 --- a/docker/helm/Chart.yaml +++ b/docker/helm/Chart.yaml @@ -17,30 +17,11 @@ apiVersion: v2 name: kyuubi -description: A Helm chart for kyuubi server - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. +description: A Helm chart for Kyuubi server type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.5.2" - +appVersion: "master-snapshot" +home: https://kyuubi.apache.org icon: https://raw.githubusercontent.com/apache/incubator-kyuubi/master/docs/imgs/logo.png - sources: - - https://github.com/apache/incubator-kyuubi/tree/master/docker \ No newline at end of file + - https://github.com/apache/incubator-kyuubi diff --git a/docker/helm/templates/kyuubi-deployment.yaml b/docker/helm/templates/kyuubi-deployment.yaml index eca757fffa2..aa3645399e7 100644 --- a/docker/helm/templates/kyuubi-deployment.yaml +++ b/docker/helm/templates/kyuubi-deployment.yaml @@ -40,7 +40,7 @@ spec: {{- end}} containers: - name: kyuubi-server - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: frontend-port diff --git a/docker/helm/values.yaml b/docker/helm/values.yaml index 59c29bfb81d..f1984615da6 100644 --- a/docker/helm/values.yaml +++ b/docker/helm/values.yaml @@ -25,8 +25,7 @@ replicaCount: 2 image: repository: apache/kyuubi pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - tag: "master-snapshot" + tag: ~ # ServiceAccount used for Kyuubi create/list/delete pod in kubernetes serviceAccount: From e860a2c6f695a6f7b014c035196029f4cf18611d Mon Sep 17 00:00:00 2001 From: dnskr Date: Mon, 12 Dec 2022 23:07:11 +0100 Subject: [PATCH 2/8] [K8S][HELM] Add imagePullSecrets support to helm chart --- docker/helm/templates/kyuubi-deployment.yaml | 3 +++ docker/helm/values.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/docker/helm/templates/kyuubi-deployment.yaml b/docker/helm/templates/kyuubi-deployment.yaml index aa3645399e7..712d76f07ec 100644 --- a/docker/helm/templates/kyuubi-deployment.yaml +++ b/docker/helm/templates/kyuubi-deployment.yaml @@ -35,6 +35,9 @@ spec: app: {{ template "kyuubi.name" . }} release: {{ .Release.Name }} spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.serviceAccount }} serviceAccountName: {{ .Values.serviceAccount.name }} {{- end}} diff --git a/docker/helm/values.yaml b/docker/helm/values.yaml index f1984615da6..0299c0338b6 100644 --- a/docker/helm/values.yaml +++ b/docker/helm/values.yaml @@ -27,6 +27,8 @@ image: pullPolicy: Always tag: ~ +imagePullSecrets: [] + # ServiceAccount used for Kyuubi create/list/delete pod in kubernetes serviceAccount: name: default From 95db61bae29c0b746402342bef01518496ac1f51 Mon Sep 17 00:00:00 2001 From: dnskr Date: Mon, 12 Dec 2022 23:16:34 +0100 Subject: [PATCH 3/8] [K8S][HELM] Add more options for liveness and readiness probes to helm chart --- docker/helm/templates/kyuubi-deployment.yaml | 4 ++++ docker/helm/values.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docker/helm/templates/kyuubi-deployment.yaml b/docker/helm/templates/kyuubi-deployment.yaml index 712d76f07ec..674a9f2f963 100644 --- a/docker/helm/templates/kyuubi-deployment.yaml +++ b/docker/helm/templates/kyuubi-deployment.yaml @@ -55,7 +55,9 @@ spec: port: {{ .Values.server.bind.port }} initialDelaySeconds: {{ .Values.probe.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.probe.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.probe.liveness.timeoutSeconds }} failureThreshold: {{ .Values.probe.liveness.failureThreshold }} + successThreshold: {{ .Values.probe.liveness.successThreshold }} {{- end }} {{- if .Values.probe.readiness.enabled }} readinessProbe: @@ -63,7 +65,9 @@ spec: port: {{ .Values.server.bind.port }} initialDelaySeconds: {{ .Values.probe.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.probe.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.probe.readiness.timeoutSeconds }} failureThreshold: {{ .Values.probe.readiness.failureThreshold }} + successThreshold: {{ .Values.probe.readiness.successThreshold }} {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} diff --git a/docker/helm/values.yaml b/docker/helm/values.yaml index 0299c0338b6..cce9fca6552 100644 --- a/docker/helm/values.yaml +++ b/docker/helm/values.yaml @@ -38,12 +38,16 @@ probe: enabled: true initialDelaySeconds: 30 periodSeconds: 10 + timeoutSeconds: 2 failureThreshold: 10 + successThreshold: 1 readiness: enabled: true initialDelaySeconds: 30 periodSeconds: 10 + timeoutSeconds: 2 failureThreshold: 10 + successThreshold: 1 server: bind: From 1073e03fdb2ca2715ba9ac9dcb3da4a5c65a366f Mon Sep 17 00:00:00 2001 From: dnskr Date: Tue, 13 Dec 2022 00:00:02 +0100 Subject: [PATCH 4/8] [K8S][HELM] Add env and envFrom support to helm chart and delete redundant $KYUUBI_HOME variable set --- docker/helm/templates/kyuubi-deployment.yaml | 13 +++++++------ docker/helm/values.yaml | 5 +++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docker/helm/templates/kyuubi-deployment.yaml b/docker/helm/templates/kyuubi-deployment.yaml index 674a9f2f963..de98fd5f33e 100644 --- a/docker/helm/templates/kyuubi-deployment.yaml +++ b/docker/helm/templates/kyuubi-deployment.yaml @@ -45,6 +45,12 @@ spec: - name: kyuubi-server image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.env }} + env: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.envFrom }} + envFrom: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} ports: - name: frontend-port containerPort: {{ .Values.server.bind.port }} @@ -74,12 +80,7 @@ spec: volumeMounts: - name: kyuubi-defaults mountPath: {{ .Values.server.conf.mountPath }} - env: - {{- range $key, $val := .Values.environments }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end }} volumes: - name: kyuubi-defaults configMap: - name: kyuubi-defaults \ No newline at end of file + name: kyuubi-defaults diff --git a/docker/helm/values.yaml b/docker/helm/values.yaml index cce9fca6552..e561087cccd 100644 --- a/docker/helm/values.yaml +++ b/docker/helm/values.yaml @@ -56,8 +56,9 @@ server: conf: mountPath: /opt/kyuubi/conf -environments: - KYUUBI_HOME: /opt/kyuubi +# Environment variables (templated) +env: [] +envFrom: [] service: type: NodePort From 53677ae6f4b2dd60783ef1ae2d1ad5af12e9df50 Mon Sep 17 00:00:00 2001 From: dnskr Date: Tue, 13 Dec 2022 00:08:24 +0100 Subject: [PATCH 5/8] [K8S][HELM] Do not render Deployment resources if the value is not set --- docker/helm/templates/kyuubi-deployment.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/helm/templates/kyuubi-deployment.yaml b/docker/helm/templates/kyuubi-deployment.yaml index de98fd5f33e..e936280f7b4 100644 --- a/docker/helm/templates/kyuubi-deployment.yaml +++ b/docker/helm/templates/kyuubi-deployment.yaml @@ -75,8 +75,9 @@ spec: failureThreshold: {{ .Values.probe.readiness.failureThreshold }} successThreshold: {{ .Values.probe.readiness.successThreshold }} {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - name: kyuubi-defaults mountPath: {{ .Values.server.conf.mountPath }} From 00337f055374e37629ca33c46892f343554c38db Mon Sep 17 00:00:00 2001 From: dnskr Date: Tue, 13 Dec 2022 00:16:48 +0100 Subject: [PATCH 6/8] [K8S][HELM] Use {{ .Release.Name }} for resource names in helm chart --- docker/helm/templates/kyuubi-configmap.yaml | 2 +- docker/helm/templates/kyuubi-deployment.yaml | 4 ++-- docker/helm/templates/kyuubi-service.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/helm/templates/kyuubi-configmap.yaml b/docker/helm/templates/kyuubi-configmap.yaml index b4fd82d4d27..dfe064cad50 100644 --- a/docker/helm/templates/kyuubi-configmap.yaml +++ b/docker/helm/templates/kyuubi-configmap.yaml @@ -18,7 +18,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: kyuubi-defaults + name: {{ .Release.Name }}-kyuubi-defaults labels: app: {{ template "kyuubi.name" . }} data: diff --git a/docker/helm/templates/kyuubi-deployment.yaml b/docker/helm/templates/kyuubi-deployment.yaml index e936280f7b4..46b8c22a899 100644 --- a/docker/helm/templates/kyuubi-deployment.yaml +++ b/docker/helm/templates/kyuubi-deployment.yaml @@ -18,7 +18,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: kyuubi-server + name: {{ .Release.Name }} labels: app: {{ template "kyuubi.name" . }} release: {{ .Release.Name }} @@ -84,4 +84,4 @@ spec: volumes: - name: kyuubi-defaults configMap: - name: kyuubi-defaults + name: {{ .Release.Name }}-kyuubi-defaults diff --git a/docker/helm/templates/kyuubi-service.yaml b/docker/helm/templates/kyuubi-service.yaml index 8d3423e7b3c..860477496c7 100644 --- a/docker/helm/templates/kyuubi-service.yaml +++ b/docker/helm/templates/kyuubi-service.yaml @@ -18,7 +18,7 @@ apiVersion: v1 kind: Service metadata: - name: kyuubi-svc + name: {{ .Release.Name }} labels: app: {{ template "kyuubi.name" . }} {{- with .Values.service.annotations }} From f677247c5b7f35ee3b67f2c4e47e31e97cace2f4 Mon Sep 17 00:00:00 2001 From: dnskr Date: Tue, 13 Dec 2022 01:02:00 +0100 Subject: [PATCH 7/8] [K8S][HELM] Add nodeSelector, tolerations, affinity and securityContext support to helm chart --- docker/helm/templates/kyuubi-deployment.yaml | 12 ++++++++++++ docker/helm/values.yaml | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/docker/helm/templates/kyuubi-deployment.yaml b/docker/helm/templates/kyuubi-deployment.yaml index 46b8c22a899..18f9fd0fa0e 100644 --- a/docker/helm/templates/kyuubi-deployment.yaml +++ b/docker/helm/templates/kyuubi-deployment.yaml @@ -85,3 +85,15 @@ spec: - name: kyuubi-defaults configMap: name: {{ .Release.Name }}-kyuubi-defaults + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/docker/helm/values.yaml b/docker/helm/values.yaml index e561087cccd..6930891aa1a 100644 --- a/docker/helm/values.yaml +++ b/docker/helm/values.yaml @@ -80,3 +80,10 @@ resources: {} # requests: # cpu: 2 # memory: 4Gi + +# Constrain Kyuubi server pods to specific nodes +nodeSelector: {} +tolerations: [] +affinity: {} + +securityContext: {} From b2896138631a2ec9263f65ee6bed595fed1ccb1f Mon Sep 17 00:00:00 2001 From: dnskr Date: Tue, 13 Dec 2022 01:02:30 +0100 Subject: [PATCH 8/8] [K8S][HELM] Add serviceAccount and rbac creation support to helm chart --- docker/helm/templates/kyuubi-deployment.yaml | 4 +-- docker/helm/templates/kyuubi-role.yaml | 26 +++++++++++++++ docker/helm/templates/kyuubi-rolebinding.yaml | 32 +++++++++++++++++++ .../helm/templates/kyuubi-serviceaccount.yaml | 25 +++++++++++++++ docker/helm/values.yaml | 10 +++++- 5 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 docker/helm/templates/kyuubi-role.yaml create mode 100644 docker/helm/templates/kyuubi-rolebinding.yaml create mode 100644 docker/helm/templates/kyuubi-serviceaccount.yaml diff --git a/docker/helm/templates/kyuubi-deployment.yaml b/docker/helm/templates/kyuubi-deployment.yaml index 18f9fd0fa0e..3dbdc06e3e7 100644 --- a/docker/helm/templates/kyuubi-deployment.yaml +++ b/docker/helm/templates/kyuubi-deployment.yaml @@ -38,9 +38,7 @@ spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.serviceAccount }} - serviceAccountName: {{ .Values.serviceAccount.name }} - {{- end}} + serviceAccountName: {{ .Values.serviceAccount.name | default .Release.Name }} containers: - name: kyuubi-server image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/docker/helm/templates/kyuubi-role.yaml b/docker/helm/templates/kyuubi-role.yaml new file mode 100644 index 00000000000..203a41cd9c4 --- /dev/null +++ b/docker/helm/templates/kyuubi-role.yaml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }} + labels: + app: {{ template "kyuubi.name" . }} +rules: {{- toYaml .Values.rbac.rules | nindent 2 }} +{{- end }} diff --git a/docker/helm/templates/kyuubi-rolebinding.yaml b/docker/helm/templates/kyuubi-rolebinding.yaml new file mode 100644 index 00000000000..26f395f300d --- /dev/null +++ b/docker/helm/templates/kyuubi-rolebinding.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }} + labels: + app: {{ template "kyuubi.name" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.name | default .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }} +{{- end }} diff --git a/docker/helm/templates/kyuubi-serviceaccount.yaml b/docker/helm/templates/kyuubi-serviceaccount.yaml new file mode 100644 index 00000000000..56f5ff5a1c8 --- /dev/null +++ b/docker/helm/templates/kyuubi-serviceaccount.yaml @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.name | default .Release.Name }} + labels: + app: {{ template "kyuubi.name" . }} +{{- end }} diff --git a/docker/helm/values.yaml b/docker/helm/values.yaml index 6930891aa1a..48909da52b7 100644 --- a/docker/helm/values.yaml +++ b/docker/helm/values.yaml @@ -31,7 +31,15 @@ imagePullSecrets: [] # ServiceAccount used for Kyuubi create/list/delete pod in kubernetes serviceAccount: - name: default + create: true + name: ~ + +rbac: + create: true + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "list", "delete"] probe: liveness: