diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml index 7365997e..5507e4dd 100644 --- a/.github/.licenserc.yaml +++ b/.github/.licenserc.yaml @@ -18,5 +18,6 @@ header: - gradlew - 'build/**' - '**/*.json' + - '**/.helmignore' comment: on-failure diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0b1d636e..46ceb911 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -44,6 +44,9 @@ jobs: - name: Build with Gradle run: | ./gradlew build + - name: Validate helm chart linting + run: | + helm lint --strict build-tools/helm/spark-kubernetes-operator build-image: name: "Build Operator Image CI" runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 5e0e9b6b..b174a96f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .vscode /lib/ target/ +build-tools/helm/spark-kubernetes-operator/crds # Gradle Files # ################ diff --git a/build-tools/helm/spark-kubernetes-operator/.helmignore b/build-tools/helm/spark-kubernetes-operator/.helmignore new file mode 100644 index 00000000..59203923 --- /dev/null +++ b/build-tools/helm/spark-kubernetes-operator/.helmignore @@ -0,0 +1,12 @@ +.DS_Store +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/build-tools/helm/spark-kubernetes-operator/Chart.yaml b/build-tools/helm/spark-kubernetes-operator/Chart.yaml new file mode 100644 index 00000000..deed565f --- /dev/null +++ b/build-tools/helm/spark-kubernetes-operator/Chart.yaml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v2 +name: spark-kubernetes-operator +description: A Helm chart for the Apache Spark Kubernetes Operator +type: application +version: 0.1.0-SNAPSHOT +appVersion: 0.1.0 +icon: https://spark.apache.org/favicon.ico diff --git a/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl b/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl new file mode 100644 index 00000000..e8ee9010 --- /dev/null +++ b/build-tools/helm/spark-kubernetes-operator/templates/_helpers.tpl @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{/* +Expand the name of the chart. +*/}} +{{- define "spark-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "spark-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "spark-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "spark-operator.commonLabels" -}} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ include "spark-operator.chart" . }} +{{- end }} + +{{/* +Dynamic config (hot properties) labels +*/}} +{{- define "spark-operator.dynamicConfigLabels" -}} +app.kubernetes.io/name: {{ include "spark-operator.name" . }} +app.kubernetes.io/component: "operator-dynamic-config-overrides" +{{- include "spark-operator.commonLabels" . }} +{{- end }} + +{{/* +Bootstrap config labels +*/}} +{{- define "spark-operator.configLabels" -}} +app.kubernetes.io/name: {{ include "spark-operator.name" . }} +app.kubernetes.io/component: "operator-config" +{{- include "spark-operator.commonLabels" . }} +{{- end }} + +{{/* +Deployment selector labels +*/}} +{{- define "spark-operator.deploymentSelectorLabels" -}} +app.kubernetes.io/name: {{ include "spark-operator.name" . }} +app.kubernetes.io/component: "operator-deployment" +{{- end }} + +{{/* +Create the path of the operator image to use +*/}} +{{- define "spark-operator.imagePath" -}} +{{- if .Values.image.digest }} +{{- .Values.image.repository }}@{{ .Values.image.digest }} +{{- else }} +{{- .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} +{{- end }} +{{- end }} + +{{/* +List of Spark app namespaces. If not provied in values, use the same namespace as operator +*/}} +{{- define "spark-operator.appNamespacesStr" -}} +{{- if index (.Values.appResources.namespaces) "data" }} +{{- $ns_list := join "," .Values.appResources.namespaces.data }} +{{- printf "%s" $ns_list }} +{{- else }} +{{- printf "%s" .Release.Namespace }} +{{- end }} +{{- end }} + +{{/* +Default property overrides +*/}} +{{- define "spark-operator.defaultPropertyOverrides" -}} +# Runtime resolved properties +spark.kubernetes.operator.namespace={{ .Release.Namespace }} +spark.kubernetes.operator.name={{- include "spark-operator.name" . }} +spark.kubernetes.operator.dynamicConfig.enabled={{ .Values.operatorConfiguration.dynamicConfig.enable }} +{{- if .Values.appResources.namespaces.overrideWatchedNamespaces }} +spark.kubernetes.operator.watchedNamespaces={{ include "spark-operator.appNamespacesStr" . | trim }} +{{- end }} +{{- end }} + +{{/* +Readiness Probe properties overrides +*/}} +{{- define "spark-operator.readinessProbe.failureThreshold" -}} +{{- default 30 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.failureThreshold }} +{{- end }} +{{- define "spark-operator.readinessProbe.periodSeconds" -}} +{{- default 10 .Values.operatorDeployment.operatorPod.operatorContainer.probes.startupProbe.periodSeconds }} +{{- end }} + +{{/* +Liveness Probe properties override +*/}} +{{- define "spark-operator.livenessProbe.initialDelaySeconds" -}} +{{- default 30 .Values.operatorDeployment.operatorPod.operatorContainer.probes.livenessProbe.initialDelaySeconds }} +{{- end }} +{{- define "spark-operator.livenessProbe.periodSeconds" -}} +{{- default 10 .Values.operatorDeployment.operatorPod.operatorContainer.probes.livenessProbe.periodSeconds }} +{{- end }} + +{{/* +Readiness Probe property overrides +*/}} +{{- define "spark-operator.probePort" -}} +{{- default 19091 .Values.operatorDeployment.operatorPod.operatorContainer.probes.port }} +{{- end }} diff --git a/build-tools/helm/spark-kubernetes-operator/templates/app-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/app-rbac.yaml new file mode 100644 index 00000000..f76d56bf --- /dev/null +++ b/build-tools/helm/spark-kubernetes-operator/templates/app-rbac.yaml @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{/* +RBAC rules used to create the app (cluster)role based on the scope +*/}} +{{- define "spark-operator.appRbacRules" }} +rules: + - apiGroups: + - "" + resources: + - pods + - services + - configmaps + - persistentvolumeclaims + verbs: + - '*' +{{- end }} + +{{/* +RoleRef for app service account rolebindings +*/}} +{{- define "spark-operator.appRoleRef" }} +roleRef: + apiGroup: rbac.authorization.k8s.io +{{- if .Values.appResources.clusterRole.create }} + kind: ClusterRole + name: {{ .Values.appResources.clusterRole.name }} +{{- else }} + kind: Role + name: {{ .Values.appResources.role.name }} +{{- end }} +{{- end }} + +{{/* +Labels and annotations to be applied +*/}} +{{- define "spark-operator.appLabels" -}} + {{- with .Values.appResources.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- include "spark-operator.commonLabels" . | nindent 4 }} +{{- end }} + +{{- define "spark-operator.appAnnotations" -}} + {{- with .Values.appResources.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- define "spark-operator.appLabelsAnnotations" }} + labels: + {{ template "spark-operator.appLabels" $ }} + annotations: + {{ template "spark-operator.appAnnotations" $ }} +{{- end }} +--- +{{- $appResources := .Values.appResources -}} +{{- $systemNs := .Release.Namespace -}} +{{- $operatorRbac := .Values.operatorRbac -}} +{{- if index (.Values.appResources.namespaces) "data" }} +{{- range $appNs := .Values.appResources.namespaces.data }} +{{- if $appResources.namespaces.create }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $appNs }} +{{- template "spark-operator.appLabelsAnnotations" $ }} +--- +{{- end }} +{{- if $appResources.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $appResources.serviceAccount.name }} + namespace: {{ $appNs }} +{{- template "spark-operator.appLabelsAnnotations" $ }} +--- +{{- end }} +{{- if $appResources.role.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $appResources.role.name }} + namespace: {{ $appNs }} +{{- template "spark-operator.appLabelsAnnotations" $ }} +{{- template "spark-operator.appRbacRules" $ }} +--- +{{- end }} +{{- if $appResources.roleBinding.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $appResources.roleBinding.name }} + namespace: {{ $appNs }} +{{- template "spark-operator.appLabelsAnnotations" $ }} +{{- template "spark-operator.appRoleRef" $ }} +subjects: + - kind: ServiceAccount + name: {{ $appResources.serviceAccount.name }} + namespace: {{ $appNs }} +--- +{{- end }} +{{- end }} +{{- else }} +{{- if $appResources.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $appResources.serviceAccount.name }} + namespace: {{ $systemNs }} +{{- template "spark-operator.appLabelsAnnotations" $ }} +--- +{{- end }} +{{- if $appResources.role.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $appResources.role.name }} + namespace: {{ $systemNs }} +{{- template "spark-operator.appLabelsAnnotations" $ }} +{{- template "spark-operator.appRbacRules" $ }} +--- +{{- end }} +{{- if $appResources.roleBinding.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $appResources.serviceAccount.name }} + namespace: {{ $systemNs }} +{{- template "spark-operator.appLabelsAnnotations" $ }} +{{- template "spark-operator.appRoleRef" $ }} +subjects: + - kind: ServiceAccount + name: {{ $appResources.serviceAccount.name }} + namespace: {{ $systemNs }} +--- +{{- end }} +{{- end }} + +{{- if $appResources.clusterRole.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $appResources.clusterRole.name }} +{{- template "spark-operator.appLabelsAnnotations" $ }} +{{- template "spark-operator.appRbacRules" $ }} +--- +{{- end }} +{{- if $appResources.sparkApplicationSentinel.create }} +{{- range $sentinelNs := .Values.appResources.sparkApplicationSentinel.sentinelNamespaces.data }} +apiVersion: org.apache.spark/v1alpha1 +kind: SparkApplication +metadata: + name: {{ $appResources.sparkApplicationSentinel.name }} + namespace: {{ $sentinelNs }} + labels: + "spark.operator/sentinel": "true" + {{- template "spark-operator.appLabels" $ }} + annotations: + {{- template "spark-operator.appAnnotations" $ }} +{{- end }} +--- +{{- end }} diff --git a/build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml new file mode 100644 index 00000000..f8364e69 --- /dev/null +++ b/build-tools/helm/spark-kubernetes-operator/templates/operator-rbac.yaml @@ -0,0 +1,160 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{/* +RBAC rules used to create the operator (cluster)role +*/}} +{{- define "spark-operator.operatorRbacRules" }} +rules: + - apiGroups: + - "" + resources: + - pods + - services + - configmaps + - persistentvolumeclaims + - persistentvolumes + - events + verbs: + - '*' + - apiGroups: + - "apps" + resources: + - statefulsets + verbs: + - '*' + - apiGroups: + - "org.apache.spark" + resources: + - '*' + verbs: + - '*' +{{- end }} + +{{/* +Labels and annotations to be applied on rbacResources +*/}} +{{- define "spark-operator.rbacLabelsAnnotations" }} + labels: + {{- with .Values.operatorRbac.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- include "spark-operator.commonLabels" . | nindent 4 }} + {{- with .Values.operatorRbac.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +--- +# Service account and rolebindings for operator +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.operatorRbac.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +{{- template "spark-operator.rbacLabelsAnnotations" $ }} +--- +{{- if .Values.operatorRbac.clusterRoleBinding.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Values.operatorRbac.clusterRoleBinding.name }} +{{- template "spark-operator.rbacLabelsAnnotations" $ }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Values.operatorRbac.clusterRole.name }} +subjects: + - kind: ServiceAccount + name: {{ .Values.operatorRbac.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +{{- end }} +--- +{{- if .Values.operatorRbac.clusterRole.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Values.operatorRbac.clusterRole.name }} +{{- template "spark-operator.rbacLabelsAnnotations" $ }} +{{- template "spark-operator.operatorRbacRules" $ }} +--- +{{- end }} +{{- if .Values.operatorConfiguration.dynamicConfig.enable }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Values.operatorRbac.configManagement.roleName }} + namespace: {{ .Release.Namespace }} +{{- template "spark-operator.rbacLabelsAnnotations" $ }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Values.operatorRbac.configManagement.roleBindingName }} + namespace: {{ .Release.Namespace }} +{{- template "spark-operator.rbacLabelsAnnotations" $ }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Values.operatorRbac.configManagement.roleName }} +subjects: + - kind: ServiceAccount + name: {{ .Values.operatorRbac.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +--- +{{- end }} + +{{- $appResources := .Values.appResources -}} +{{- $systemNs := .Release.Namespace -}} +{{- $operatorRbac := .Values.operatorRbac -}} +{{- if index (.Values.appResources.namespaces) "data" }} +{{- range $appNs := .Values.appResources.namespaces.data }} +{{- if $operatorRbac.role.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $operatorRbac.role.name }} + namespace: {{ $appNs }} +{{- template "spark-operator.rbacLabelsAnnotations" $ }} +{{- template "spark-operator.operatorRbacRules" $ }} +--- +{{- end }} +{{- if $operatorRbac.roleBinding.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $operatorRbac.roleBinding.name }} + namespace: {{ $appNs }} +{{- template "spark-operator.rbacLabelsAnnotations" $ }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: {{ $operatorRbac.roleBinding.roleRef.kind }} + name: {{ $operatorRbac.roleBinding.roleRef.name }} +subjects: + - kind: ServiceAccount + name: {{ $operatorRbac.serviceAccount.name }} + namespace: {{ $systemNs }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml b/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml new file mode 100644 index 00000000..5edd2287 --- /dev/null +++ b/build-tools/helm/spark-kubernetes-operator/templates/spark-operator.yaml @@ -0,0 +1,205 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "spark-operator.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-operator.deploymentSelectorLabels" . | nindent 4 }} + {{- include "spark-operator.commonLabels" . | nindent 4 }} +spec: + replicas: {{ .Values.operatorDeployment.replicas }} + strategy: + {{- toYaml .Values.operatorDeployment.strategy | nindent 4 }} + selector: + matchLabels: + {{- include "spark-operator.deploymentSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "spark-operator.deploymentSelectorLabels" . | nindent 8 }} + {{- if index (.Values.operatorDeployment.operatorPod) "labels" }} + {{- with .Values.operatorDeployment.operatorPod.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + annotations: + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + {{- if index (.Values.operatorDeployment.operatorPod) "annotations" }} + {{- with .Values.operatorDeployment.operatorPod.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + spec: + {{- with .Values.operatorDeployment.operatorPod.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.operatorDeployment.operatorPod.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.operatorDeployment.operatorPod.nodeSelector }} + nodeSelector: {{ toYaml .Values.operatorDeployment.operatorPod.nodeSelector | nindent 8 }} + {{- end }} + {{- with .Values.operatorDeployment.operatorPod.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operatorDeployment.operatorPod.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.operatorRbac.serviceAccount.name }} + {{- if .Values.operatorDeployment.operatorPod.topologySpreadConstraints }} + topologySpreadConstraints: {{ toYaml .Values.operatorDeployment.operatorPod.topologySpreadConstraints | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ include "spark-operator.imagePath" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: [ "./docker-entrypoint.sh", "operator" ] + ports: + - containerPort: {{ include "spark-operator.probePort" . }} + name: probe-port + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: {{ include "spark-operator.name" . }} + - name: LOG_CONFIG + value: -Dlog4j.configurationFile=/opt/spark-operator/conf/log4j2.properties + - name: OPERATOR_JAVA_OPTS + value: {{ .Values.operatorDeployment.operatorPod.operatorContainer.jvmArgs }} + {{- with .Values.operatorDeployment.operatorPod.operatorContainer.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.operatorDeployment.operatorPod.operatorContainer.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.operatorDeployment.operatorPod.operatorContainer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + readinessProbe: + httpGet: + port: probe-port + path: /readyz + failureThreshold: {{ include "spark-operator.readinessProbe.failureThreshold" . }} + periodSeconds: {{ include "spark-operator.readinessProbe.periodSeconds" . }} + livenessProbe: + httpGet: + port: probe-port + path: /healthz + initialDelaySeconds: {{ include "spark-operator.livenessProbe.initialDelaySeconds" . }} + periodSeconds: {{ include "spark-operator.livenessProbe.periodSeconds" . }} + {{- with .Values.operatorDeployment.operatorPod.operatorContainer.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: spark-operator-config-volume + mountPath: /opt/spark-operator/conf + - name: logs-volume + mountPath: /opt/spark-operator/logs + {{- with .Values.operatorDeployment.operatorPod.operatorContainer.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.operatorDeployment.operatorPod.additionalContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if index (.Values.operatorDeployment.operatorPod) "dnsPolicy" }} + dnsPolicy: {{ .Values.operatorDeployment.operatorPod.dnsPolicy | quote }} + {{- end }} + {{- if index (.Values.operatorDeployment.operatorPod) "dnsConfig" }} + dnsConfig: + {{- with .Values.operatorDeployment.operatorPod.dnsConfig }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + restartPolicy: Always + volumes: + - name: spark-operator-config-volume + configMap: + name: spark-kubernetes-operator-configuration + - name: logs-volume + emptyDir: { } + {{- with .Values.operatorDeployment.operatorPod.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: spark-kubernetes-operator-configuration + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-operator.configLabels" . | nindent 4 }} +data: + log4j2.properties: |+ +{{- if .Values.operatorConfiguration.append }} + {{- $.Files.Get "conf/log4j2.properties" | nindent 4 -}} +{{- end }} +{{- if index (.Values.operatorConfiguration) "log4j2.properties" }} + {{- index (.Values.operatorConfiguration) "log4j2.properties" | nindent 4 -}} +{{- end }} + spark-operator.properties: |+ + {{- include "spark-operator.defaultPropertyOverrides" . | nindent 4 }} +{{- if .Values.operatorConfiguration.append }} + {{- $.Files.Get "conf/spark-operator.properties" | nindent 4 -}} +{{- end }} +{{- if index (.Values.operatorConfiguration) "spark-operator.properties" }} + {{- index (.Values.operatorConfiguration) "spark-operator.properties" | nindent 4 -}} +{{- end }} + metrics.properties: |+ +{{- if index (.Values.operatorConfiguration) "metrics.properties" }} + {{- index (.Values.operatorConfiguration) "metrics.properties" | nindent 4 -}} +{{- end }} +--- +{{- if .Values.operatorConfiguration.dynamicConfig.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: spark-kubernetes-operator-dynamic-configuration + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-operator.dynamicConfigLabels" . | nindent 4 }} + annotations: + {{- toYaml .Values.operatorConfiguration.dynamicConfig.annotations | nindent 4 }} +{{- with .Values.operatorConfiguration.dynamicConfig.data }} +data: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} diff --git a/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml b/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml new file mode 100644 index 00000000..33d8fa32 --- /dev/null +++ b/build-tools/helm/spark-kubernetes-operator/templates/tests/test-rbac.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "spark-operator.name" . }}-test-operator-rbac" + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-operator.commonLabels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +spec: + containers: + - name: kubectl + image: bitnami/kubectl:latest + command: ['bash', '-c' ] + args: [ + 'kubectl auth can-i list sparkapplications --all-namespaces', + 'kubectl auth can-i create pods --all-namespaces', + 'kubectl auth can-i create services --all-namespaces', + 'kubectl auth can-i create configmaps --all-namespaces', + 'kubectl auth can-i create persistentvolumeclaims', + 'kubectl auth can-i create events --all-namespaces' + ] + serviceAccountName: {{ .Values.operatorRbac.serviceAccount.name }} + restartPolicy: Never +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "spark-operator.name" . }}-test-app-rbac" + namespace: {{ .Release.Namespace }} + labels: + {{- include "spark-operator.commonLabels" . | nindent 4 }} + annotations: + "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed +spec: + containers: + - name: kubectl + image: bitnami/kubectl:latest + command: ['bash', '-c' ] + args: [ + 'kubectl auth can-i create pods -n {{ .Release.Namespace }}', + 'kubectl auth can-i create configmaps -n {{ .Release.Namespace }}', + 'kubectl auth can-i create services -n {{ .Release.Namespace }}', + 'kubectl auth can-i create persistentvolumeclaims -n {{ .Release.Namespace }}' + ] + serviceAccountName: {{ .Values.appResources.serviceAccount.name }} + restartPolicy: Never diff --git a/build-tools/helm/spark-kubernetes-operator/values.yaml b/build-tools/helm/spark-kubernetes-operator/values.yaml new file mode 100644 index 00000000..87fda1e7 --- /dev/null +++ b/build-tools/helm/spark-kubernetes-operator/values.yaml @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +image: + repository: spark-kubernetes-operator + pullPolicy: IfNotPresent + tag: 0.1.0 + # If image digest is set then it takes precedence and the image tag will be ignored + # digest: "" + +imagePullSecrets: [ ] + +operatorDeployment: + # Replicas must be 1 unless leader election is enabled + replicas: 1 + # Strategy type must be 'Recreate' unless leader election is enabled + strategy: + type: Recreate + operatorPod: + priorityClassName: null + annotations: { } + labels: { } + affinity: { } + nodeSelector: { } + # Node tolerations for operator pod assignment + # https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [ ] + # Topology spread constrains + # https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: [ ] + operatorContainer: + jvmArgs: "-Dfile.encoding=UTF8" + env: + envFrom: + volumeMounts: { } + resources: + limits: + cpu: "1" + ephemeral-storage: 2Gi + memory: 2Gi + requests: + cpu: "1" + ephemeral-storage: 2Gi + memory: 2Gi + probes: + port: 19091 + livenessProbe: + periodSeconds: 10 + initialDelaySeconds: 30 + startupProbe: + failureThreshold: 30 + periodSeconds: 10 + # By default, operator container is configured to comply restricted standard + # https://kubernetes.io/docs/concepts/security/pod-security-standards/ + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 185 + seccompProfile: + type: RuntimeDefault + additionalContainers: { } + volumes: { } + securityContext: { } + dnsPolicy: + dnsConfig: + +operatorRbac: + serviceAccount: + create: true + name: "spark-operator" + clusterRole: + create: true + name: "spark-operator-clusterrole" + clusterRoleBinding: + create: true + name: "spark-operator-clusterrolebinding" + role: + # If enabled, a Role would be created inside each app namespace, as configured + # in {appResources.namespaces.data} for operator. Please enable *at least one* of + # {operatorRbac.clusterRole.create} or {operatorRbac.role.create} for operator + # unless permission is provided separately from the chart + create: false + name: "spark-operator-role" + roleBinding: + # If enabled, a RoleBinding would be created inside each app namespace, as configured + # in {appResources.namespaces.data} for operator. Please enable *at least one* of + # {operatorRbac.clusterRoleBinding.create} or {operatorRbac.roleBinding.create} for + # operator unless permission is provided separately from the chart + create: false + name: "spark-operator-rolebinding" + # It's possible to make the rolebinding refers to role / clusterrole as applicable + roleRef: + kind: Role + name: "spark-operator-role" + configManagement: + # Role to enable operator loading hot properties from config map + create: true + roleName: "spark-operator-config-monitor" + roleBindingName: "spark-operator-config-monitor-role-binding" + labels: + "app.kubernetes.io/component": "operator-rbac" + +appResources: + # Create namespace(s), service account(s), clusterrole, role(s) and rolebinding(s) for SparkApps + namespaces: + create: true + # When enabled, value for conf property `spark.operator.watched.namespaces` would be set to + # the values provided in {appResources.namespaces.data} field as well + overrideWatchedNamespaces: true + data: + # - "spark-1" + # - "spark-2" + serviceAccount: + create: true + name: "spark" + role: + # When enabled, a role would be created in each app namespace, as configured in + # {appResources.namespaces.data} for Spark apps. Please enable *at least one* of + # {appResources.roles.create} or {appResources.clusterRole.create} for + # Spark app unless permission is provided separately from the chart + create: false + name: "spark-app-role" + clusterRole: + # When enabled, a clusterrole would be created for Spark apps. Please enable *at least one* of + # {appResources.roles.create} or {appResources.clusterRole.create} for Spark app unless + # permission is provided separately from the chart + create: true + name: "spark-app-clusterrole" + roleBinding: + create: true + name: "spark-app-rolebinding" + sparkApplicationSentinel: + create: false + name: "spark-app-sentinel" + sentinelNamespaces: + data: + # - "spark-1" + # - "spark-2" + # When enabled, a sentinel resources will be deployed to namespace(s) provided/ + # Note that sentinelNamespaces list should be a subset of {appResources.namespaces.data} + # - "default" + # App resources are by default annotated to avoid app abort due to operator upgrade + annotations: + "helm.sh/resource-policy": keep + # labels to be added on app resources + labels: + "app.kubernetes.io/component": "spark-app" + +operatorConfiguration: + # If set to true, below properties would be appended to default conf files under conf/ + # Otherwise, below would override default conf files + append: true + log4j2.properties: |+ + # Logging Overrides + # rootLogger.level=DEBUG + spark-operator.properties: |+ + # Property Overrides. e.g. + # spark.kubernetes.operator.reconciler.intervalSeconds=60 + metrics.properties: |+ + # Metrics Properties Overrides + dynamicConfig: + # Enable this for hot properties loading. + enable: false + # Enable this to create a config map for hot property loading + create: false + annotations: + "helm.sh/resource-policy": keep + data: + # Spark Operator Config Runtime Properties Overrides. e.g. + spark.kubernetes.operator.reconciler.intervalSeconds: 60 diff --git a/dev/.rat-excludes b/dev/.rat-excludes index 57dc552b..4234634f 100644 --- a/dev/.rat-excludes +++ b/dev/.rat-excludes @@ -15,3 +15,4 @@ RELEASE build .*gradle .*json +.helmignore diff --git a/spark-operator-api/build.gradle b/spark-operator-api/build.gradle index 25c7069c..8b1dfc3d 100644 --- a/spark-operator-api/build.gradle +++ b/spark-operator-api/build.gradle @@ -32,3 +32,11 @@ tasks.register('finalizeGeneratedCRD', Exec) { println "Updating PrinterColumns for generated CRD" commandLine 'sh', './src/main/resources/printer-columns.sh' } + +// Copy generated yaml to Helm charts +tasks.register('relocateGeneratedCRD', Copy) { + dependsOn finalizeGeneratedCRD + from "build/classes/java/main/META-INF/fabric8/sparkapplications.org.apache.spark-v1.yml" + into "../build-tools/helm/spark-kubernetes-operator/crds" + rename '(.+).yml', '$1.yaml' +}