Separate workers service accounts

chart/files/pod-template-file.kubernetes-helm-yaml

@@ -211,7 +211,11 @@ spec:
   terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }}
   tolerations: {{- toYaml $tolerations | nindent 4 }}
   topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 4 }}
+  {{- if .Values.workers.useWorkerDedicatedServiceAccounts }}
+  serviceAccountName: {{ include "worker.kubernetes.serviceAccountName" . }}
+  {{- else }}
   serviceAccountName: {{ include "worker.serviceAccountName" . }}
+  {{- end }}
   volumes:
   {{- if .Values.dags.persistence.enabled }}
   - name: dags

chart/templates/_helpers.yaml

@@ -636,13 +636,23 @@ server_tls_key_file = /etc/pgbouncer/server.key
   {{- end }}
 {{- end }}
 
-{{/* Helper to generate service account name respecting .Values.$section.serviceAccount flags */}}
+{{/* Helper for service account name generation */}}
+{{- define "_serviceAccountNameGen" -}}
+  {{- if .sa.create }}
+    {{- default (printf "%s-%s" (include "airflow.serviceAccountName" .) (default .key .nameSuffix )) .sa.name | quote }}
+  {{- else }}
+    {{- default "default" .sa.name | quote }}
+  {{- end }}
+{{- end }}
+
+{{/* Helper to generate service account name respecting .Values.$section.serviceAccount or .Values.$section.$subSection.serviceAccount flags */}}
 {{- define "_serviceAccountName" -}}
-  {{- $sa := get (get .Values .key) "serviceAccount" }}
-  {{- if $sa.create }}
-    {{- default (printf "%s-%s" (include "airflow.serviceAccountName" .) (default .key .nameSuffix )) $sa.name | quote }}
+  {{- if .subKey }}
+    {{- $sa := get (get (get .Values .key) .subKey) "serviceAccount" -}}
+    {{- include "_serviceAccountNameGen" (merge (dict "sa" $sa "key" .key "nameSuffix" .nameSuffix) .) }}
   {{- else }}
-    {{- default "default" $sa.name | quote }}
+    {{- $sa := get (get .Values .key) "serviceAccount" }}
+    {{- include "_serviceAccountNameGen" (merge (dict "sa" $sa "key" .key "nameSuffix" .nameSuffix) .) }}
   {{- end }}
 {{- end }}
 
@@ -692,6 +702,16 @@ server_tls_key_file = /etc/pgbouncer/server.key
   {{- include "_serviceAccountName" (merge (dict "key" "workers" "nameSuffix" "worker") .) -}}
 {{- end }}
 
+{{/* Create the name of the worker celery service account to use */}}
+{{- define "worker.celery.serviceAccountName" -}}
+  {{- include "_serviceAccountName" (merge (dict "key" "workers" "subKey" "celery" "nameSuffix" "worker-celery") .) -}}
+{{- end }}
+
+{{/* Create the name of the worker kubernetes service account to use */}}
+{{- define "worker.kubernetes.serviceAccountName" -}}
+  {{- include "_serviceAccountName" (merge (dict "key" "workers" "subKey" "kubernetes" "nameSuffix" "worker-kubernetes") .) -}}
+{{- end }}
+
 {{/* Create the name of the triggerer service account to use */}}
 {{- define "triggerer.serviceAccountName" -}}
   {{- include "_serviceAccountName" (merge (dict "key" "triggerer") .) -}}

chart/templates/rbac/pod-launcher-rolebinding.yaml

@@ -68,12 +68,22 @@ subjects:
   {{- end }}
   {{- end }}
   {{- $workerAdded := false }}
+  {{- $workersDedicatedSA := .Values.workers.useWorkerDedicatedServiceAccounts -}}
   {{- range $executor := $executors }}
   {{- if and (has $executor $workerLaunchExecutors) (not $workerAdded) }}
   {{- $workerAdded = true }}
+  {{- if $workersDedicatedSA }}
+  - kind: ServiceAccount
+    name: {{ include "worker.celery.serviceAccountName" $ }}
+    namespace: "{{ $.Release.Namespace }}"
+  - kind: ServiceAccount
+    name: {{ include "worker.kubernetes.serviceAccountName" $ }}
+    namespace: "{{ $.Release.Namespace }}"
+  {{- else }}
   - kind: ServiceAccount
     name: {{ include "worker.serviceAccountName" $ }}
     namespace: "{{ $.Release.Namespace }}"
   {{- end }}
   {{- end }}
+  {{- end }}
 {{- end }}

chart/templates/rbac/security-context-constraint-rolebinding.yaml

@@ -55,10 +55,19 @@ subjects:
     name: {{ include "webserver.serviceAccountName" . }}
     namespace: "{{ .Release.Namespace }}"
   {{- if $hasWorkers }}
+  {{- if .Values.workers.useWorkerDedicatedServiceAccounts }}
+  - kind: ServiceAccount
+    name: {{ include "worker.celery.serviceAccountName" . }}
+    namespace: "{{ .Release.Namespace }}"
+  - kind: ServiceAccount
+    name: {{ include "worker.kubernetes.serviceAccountName" . }}
+    namespace: "{{ .Release.Namespace }}"
+  {{- else }}
   - kind: ServiceAccount
     name: {{ include "worker.serviceAccountName" . }}
     namespace: "{{ .Release.Namespace }}"
   {{- end }}
+  {{- end }}
   - kind: ServiceAccount
     name: {{ include "scheduler.serviceAccountName" . }}
     namespace: "{{ .Release.Namespace }}"

chart/templates/workers/worker-celery-serviceaccount.yaml

@@ -0,0 +1,41 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+#######################################
+## Airflow Worker Celery ServiceAccount
+#######################################
+{{- if and .Values.workers.celery.serviceAccount.create .Values.workers.useWorkerDedicatedServiceAccounts (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor)) }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.workers.celery.serviceAccount.automountServiceAccountToken }}
+metadata:
+  name: {{ include "worker.celery.serviceAccountName" . }}
+  labels:
+    tier: airflow
+    component: worker
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- if or .Values.labels .Values.workers.labels .Values.workers.celery.labels }}
+      {{- mustMerge .Values.workers.celery.labels .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+  {{- with .Values.workers.celery.serviceAccount.annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

chart/templates/workers/worker-deployment.yaml

@@ -135,7 +135,11 @@ spec:
       {{- end }}
       terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }}
       restartPolicy: Always
+      {{- if .Values.workers.useWorkerDedicatedServiceAccounts }}
+      serviceAccountName: {{ include "worker.celery.serviceAccountName" . }}
+      {{- else }}
       serviceAccountName: {{ include "worker.serviceAccountName" . }}
+      {{- end }}
       securityContext: {{ $securityContext | nindent 8 }}
       {{- if or .Values.registry.secretName .Values.registry.connection }}
       imagePullSecrets:

chart/templates/workers/worker-kubernetes-serviceaccount.yaml

@@ -0,0 +1,41 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+###########################################
+## Airflow Worker Kubernetes ServiceAccount
+###########################################
+{{- if and .Values.workers.kubernetes.serviceAccount.create .Values.workers.useWorkerDedicatedServiceAccounts (or (contains "CeleryKubernetesExecutor" .Values.executor) (contains "KubernetesExecutor" .Values.executor) (contains "LocalKubernetesExecutor" .Values.executor)) }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{ .Values.workers.kubernetes.serviceAccount.automountServiceAccountToken }}
+metadata:
+  name: {{ include "worker.kubernetes.serviceAccountName" . }}
+  labels:
+    tier: airflow
+    component: worker
+    release: {{ .Release.Name }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+    {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }}
+      {{- mustMerge .Values.workers.kubernetes.labels .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
+    {{- end }}
+  {{- with .Values.workers.kubernetes.serviceAccount.annotations }}
+  annotations: {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

chart/templates/workers/worker-serviceaccount.yaml

@@ -20,7 +20,7 @@
 ################################
 ## Airflow Worker ServiceAccount
 #################################
-{{- if and .Values.workers.serviceAccount.create (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) (contains "KubernetesExecutor" .Values.executor) (contains "LocalKubernetesExecutor" .Values.executor)) }}
+{{- if and .Values.workers.serviceAccount.create (not .Values.workers.useWorkerDedicatedServiceAccounts) (or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) (contains "KubernetesExecutor" .Values.executor) (contains "LocalKubernetesExecutor" .Values.executor)) }}
 apiVersion: v1
 kind: ServiceAccount
 automountServiceAccountToken: {{ .Values.workers.serviceAccount.automountServiceAccountToken }}

chart/values.schema.json

@@ -2530,6 +2530,89 @@
                             }
                         }
                     ]
+                },
+                "useWorkerDedicatedServiceAccounts": {
+                    "description": "One common Service Account for all workers will be created if flag is set to false. If true, dedicated Service Accounts for every worker type will be created.",
+                    "type": "boolean",
+                    "default": false
+                },
+                "celery": {
+                    "description": "Airflow Celery Workers configuration.",
+                    "type": "object",
+                    "x-docsSection": "Workers",
+                    "properties": {
+                        "serviceAccount": {
+                            "description": "Create ServiceAccount.",
+                            "type": "object",
+                            "properties": {
+                                "automountServiceAccountToken": {
+                                    "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.",
+                                    "type": "boolean",
+                                    "default": true
+                                },
+                                "create": {
+                                    "description": "Specifies whether a ServiceAccount should be created.",
+                                    "type": "boolean",
+                                    "default": true
+                                },
+                                "name": {
+                                    "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.",
+                                    "type": [
+                                        "string",
+                                        "null"
+                                    ],
+                                    "default": null
+                                },
+                                "annotations": {
+                                    "description": "Annotations to add to the Airflow Celery worker Kubernetes ServiceAccount.",
+                                    "type": "object",
+                                    "default": {},
+                                    "additionalProperties": {
+                                        "type": "string"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                },
+                "kubernetes": {
+                    "description": "Airflow pod-template-file configuration.",
+                    "type": "object",
+                    "x-docsSection": "Workers",
+                    "properties": {
+                        "serviceAccount": {
+                            "description": "Create ServiceAccount.",
+                            "type": "object",
+                            "properties": {
+                                "automountServiceAccountToken": {
+                                    "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.",
+                                    "type": "boolean",
+                                    "default": true
+                                },
+                                "create": {
+                                    "description": "Specifies whether a ServiceAccount should be created.",
+                                    "type": "boolean",
+                                    "default": true
+                                },
+                                "name": {
+                                    "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.",
+                                    "type": [
+                                        "string",
+                                        "null"
+                                    ],
+                                    "default": null
+                                },
+                                "annotations": {
+                                    "description": "Annotations to add to the worker Kubernetes ServiceAccount.",
+                                    "type": "object",
+                                    "default": {},
+                                    "additionalProperties": {
+                                        "type": "string"
+                                    }
+                                }
+                            }
+                        }
+                    }
                 }
             }
         },

chart/values.yaml

@@ -985,6 +985,38 @@ workers:
   #         requests:
   #           storage: "20Gi"
 
+  # One common Service Account for all workers will be created if flag is set to false.
+  # If true, dedicated Service Accounts for every worker type will be created.
+  useWorkerDedicatedServiceAccounts: false
+
+  celery:
+    # Create ServiceAccount for Airflow Celery workers
+    serviceAccount:
+      # default value is true
+      # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+      automountServiceAccountToken: true
+      # Specifies whether a ServiceAccount should be created
+      create: true
+      # The name of the ServiceAccount to use.
+      # If not set and create is true, a name is generated using the release name
+      name: ~
+      # Annotations to add to worker kubernetes service account.
+      annotations: {}
+
+  kubernetes:
+    # Create ServiceAccount for pods created with pod-template-file
+    serviceAccount:
+      # Auto mount service account token into the pod. Default value is true.
+      # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+      automountServiceAccountToken: true
+      # Specifies whether a ServiceAccount should be created
+      create: true
+      # The name of the ServiceAccount to use.
+      # If not set and create is true, a name is generated using the release name.
+      name: ~
+      # Annotations to add to worker kubernetes service account
+      annotations: {}
+
 # Airflow scheduler settings
 scheduler:
   enabled: true