From d2e07d6e81a1a7df4717af73dadcf739edb99b44 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Tue, 29 Apr 2025 03:55:27 +0300 Subject: [PATCH 01/10] Generate JWT secret during HELM install --- chart/templates/_helpers.yaml | 11 +++++++ chart/templates/secrets/jwt-secret.yaml | 41 +++++++++++++++++++++++++ chart/values.schema.json | 23 ++++++++++++++ chart/values.yaml | 5 +++ 4 files changed, 80 insertions(+) create mode 100644 chart/templates/secrets/jwt-secret.yaml diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 4e9b9c787b5aa..2833d5114c451 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -98,6 +98,13 @@ If release name contains chart name it will be used as a full name. name: {{ template "webserver_secret_key_secret" . }} key: webserver-secret-key {{- end }} + {{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__API_AUTH__JWT_SECRET }} + - name: AIRFLOW__API_AUTH__JWT_SECRET + valueFrom: + secretKeyRef: + name: {{ template "jwt_secret" . }} + key: jwt-secret + {{- end }} {{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} {{- if or (semverCompare "<2.4.0" .Values.airflowVersion) (.Values.data.resultBackendSecretName) (.Values.data.resultBackendConnection) }} {{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__CELERY_RESULT_BACKEND }} @@ -396,6 +403,10 @@ If release name contains chart name it will be used as a full name. {{- default (printf "%s-fernet-key" .Release.Name) .Values.fernetKeySecretName }} {{- end }} +{{- define "jwt_secret" -}} + {{- default (printf "%s-jwt-secret" .Release.Name) .Values.jwtSecretName }} +{{- end }} + {{- define "webserver_secret_key_secret" -}} {{- default (printf "%s-webserver-secret-key" (include "airflow.fullname" .)) .Values.webserverSecretKeySecretName }} {{- end }} diff --git a/chart/templates/secrets/jwt-secret.yaml b/chart/templates/secrets/jwt-secret.yaml new file mode 100644 index 0000000000000..aee391f33b4cb --- /dev/null +++ b/chart/templates/secrets/jwt-secret.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +############################################ +## Airflow JWT Secret +############################################ +{{- if not .Values.jwtSecretName }} +{{ $generated_secret_key := (randAlphaNum 32 | b64enc) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "airflow.fullname" . }}-jwt-secret + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +type: Opaque +data: + jwt-secret: {{ (default $generated_secret_key .Values.jwtSecret) | b64enc | quote }} +{{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 93036cac30e42..3c0f636c2d785 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1064,6 +1064,11 @@ "type": "boolean", "default": true }, + "AIRFLOW__API_AUTH__JWT_SECRET": { + "description": "Enable ``AIRFLOW__API_AUTH__JWT_SECRET`` variable to be read from the JWT Secret", + "type": "boolean", + "default": true + }, "AIRFLOW__WEBSERVER__SECRET_KEY": { "description": "Enable ``AIRFLOW__WEBSERVER__SECRET_KEY`` variable to be read from the Webserver Secret Key Secret", "type": "boolean", @@ -1426,6 +1431,24 @@ "x-docsSection": "Airflow", "default": null }, + "jwtSecret": { + "description": "Secret key used to encode and decode JWTs to authenticate to public and private APIs (can only be set during install, not upgrade).", + "type": [ + "string", + "null" + ], + "x-docsSection": "Common", + "default": null + }, + "jwtSecretName": { + "description": "The JWT secret name.", + "type": [ + "string", + "null" + ], + "x-docsSection": "Airflow", + "default": null + }, "webserverSecretKey": { "description": "The Flask secret key for Airflow Webserver to encrypt browser session.", "type": [ diff --git a/chart/values.yaml b/chart/values.yaml index 9066cbbb7f2c7..28eb4b5992b05 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -360,6 +360,7 @@ enableBuiltInSecretEnvVars: AIRFLOW__CORE__SQL_ALCHEMY_CONN: true AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: true AIRFLOW_CONN_AIRFLOW_DB: true + AIRFLOW__API_AUTH__JWT_SECRET: true AIRFLOW__WEBSERVER__SECRET_KEY: true AIRFLOW__CELERY__CELERY_RESULT_BACKEND: true AIRFLOW__CELERY__RESULT_BACKEND: true @@ -499,6 +500,10 @@ fernetKeySecretName: ~ webserverSecretKey: ~ webserverSecretKeySecretName: ~ +# Secret key used to encode and decode JWTs: `[api_auth] jwt_secret` in airflow.cfg +jwtSecret: ~ +jwtSecretName: ~ + # In order to use kerberos you need to create secret containing the keytab file # The secret name should follow naming convention of the application where resources are # name {{ .Release-name }}-. In case of the keytab file, the postfix is "kerberos-keytab" From e3e35c2ce69fdf8a0b254dff2eb3874367c41a35 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Tue, 29 Apr 2025 07:25:18 +0300 Subject: [PATCH 02/10] =?UTF-8?q?=F0=9F=A7=AA=20Fix=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- helm-tests/tests/helm_tests/security/test_rbac.py | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-tests/tests/helm_tests/security/test_rbac.py b/helm-tests/tests/helm_tests/security/test_rbac.py index 6271e7cfafcf0..fcd578daa608e 100644 --- a/helm-tests/tests/helm_tests/security/test_rbac.py +++ b/helm-tests/tests/helm_tests/security/test_rbac.py @@ -46,6 +46,7 @@ ("StatefulSet", "test-rbac-worker"), ("Secret", "test-rbac-broker-url"), ("Secret", "test-rbac-fernet-key"), + ("Secret", "test-rbac-jwt-secret"), ("Secret", "test-rbac-redis-password"), ("Secret", "test-rbac-webserver-secret-key"), ("Job", "test-rbac-create-user"), From 5b49fa9a90f4ff4b241195e2dddef2e6ff1d0232 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Tue, 29 Apr 2025 07:38:36 +0300 Subject: [PATCH 03/10] =?UTF-8?q?=F0=9F=A7=AA=20Fix=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py index 00ff40799e3d0..96825f0818223 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -38,6 +38,7 @@ ("Secret", "test-basic-airflow-metadata"), ("Secret", "test-basic-broker-url"), ("Secret", "test-basic-fernet-key"), + ("Secret", "test-basic-jwt-secret"), ("Secret", "test-basic-airflow-webserver-secret-key"), ("Secret", "test-basic-redis-password"), ("Secret", "test-basic-postgresql"), From fb85daa317a95c57345b999c2e40ad126423509b Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Tue, 29 Apr 2025 07:55:25 +0300 Subject: [PATCH 04/10] =?UTF-8?q?=F0=9F=A7=AA=20Fix=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/helm_tests/airflow_aux/test_basic_helm_chart.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py index 96825f0818223..0817655ac1662 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -137,6 +137,7 @@ def test_basic_deployments(self, version): ("Secret", "test-basic-metadata"), ("Secret", "test-basic-broker-url"), ("Secret", "test-basic-fernet-key"), + ("Secret", "test-basic-jwt-secret"), ("Secret", "test-basic-webserver-secret-key"), ("Secret", "test-basic-postgresql"), ("Secret", "test-basic-redis-password"), @@ -237,6 +238,7 @@ def test_basic_deployment_with_standalone_dag_processor(self, version): ("Secret", "test-basic-metadata"), ("Secret", "test-basic-broker-url"), ("Secret", "test-basic-fernet-key"), + ("Secret", "test-basic-jwt-secret"), ("Secret", "test-basic-webserver-secret-key"), ("Secret", "test-basic-postgresql"), ("Secret", "test-basic-redis-password"), @@ -422,6 +424,7 @@ def test_labels_are_valid(self, airflow_version): (f"{release_name}-cleanup-rolebinding", "RoleBinding", None), (f"{release_name}-create-user", "Job", "create-user-job"), (f"{release_name}-fernet-key", "Secret", None), + (f"{release_name}-jwt-secret", "Secret", None), (f"{release_name}-flower", "Deployment", "flower"), (f"{release_name}-flower", "Service", "flower"), (f"{release_name}-flower-policy", "NetworkPolicy", "airflow-flower-policy"), From d6c70afff54eeb7a843526f5c560ac60e18ef006 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Tue, 29 Apr 2025 08:16:46 +0300 Subject: [PATCH 05/10] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py b/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py index 78b43d02f73eb..e6b247afb1cd0 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py @@ -323,6 +323,7 @@ def test_should_disable_some_variables(self): "enableBuiltInSecretEnvVars": { "AIRFLOW__CORE__SQL_ALCHEMY_CONN": False, "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN": False, + "AIRFLOW__API_AUTH__JWT_SECRET": False, "AIRFLOW__WEBSERVER__SECRET_KEY": False, # the following vars only appear if remote logging is set, so disabling them in this test is kind of a no-op "AIRFLOW__ELASTICSEARCH__HOST": False, @@ -370,6 +371,7 @@ def test_have_all_variables(self): "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN", "AIRFLOW_CONN_AIRFLOW_DB", "AIRFLOW__WEBSERVER__SECRET_KEY", + "AIRFLOW__API_AUTH__JWT_SECRET", "AIRFLOW__CELERY__BROKER_URL", ] expected_vars_in_worker = ["DUMB_INIT_SETSID"] + expected_vars From 2b59bb626af30f65a33e9178328a19b8a9995a23 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Wed, 30 Apr 2025 13:08:58 +0300 Subject: [PATCH 06/10] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py | 1 - 1 file changed, 1 deletion(-) diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py index 0817655ac1662..85fab23860b52 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -424,7 +424,6 @@ def test_labels_are_valid(self, airflow_version): (f"{release_name}-cleanup-rolebinding", "RoleBinding", None), (f"{release_name}-create-user", "Job", "create-user-job"), (f"{release_name}-fernet-key", "Secret", None), - (f"{release_name}-jwt-secret", "Secret", None), (f"{release_name}-flower", "Deployment", "flower"), (f"{release_name}-flower", "Service", "flower"), (f"{release_name}-flower-policy", "NetworkPolicy", "airflow-flower-policy"), From 51e8f43de353b05512ea85ec3af9f907fff0146f Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Wed, 30 Apr 2025 13:10:24 +0300 Subject: [PATCH 07/10] =?UTF-8?q?=F0=9F=9A=A7=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/helm_tests/airflow_aux/test_basic_helm_chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py index 85fab23860b52..b7cb5d61187ad 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -38,7 +38,7 @@ ("Secret", "test-basic-airflow-metadata"), ("Secret", "test-basic-broker-url"), ("Secret", "test-basic-fernet-key"), - ("Secret", "test-basic-jwt-secret"), + ("Secret", "test-basic-airflow-jwt-secret"), ("Secret", "test-basic-airflow-webserver-secret-key"), ("Secret", "test-basic-redis-password"), ("Secret", "test-basic-postgresql"), From 7722a816c74724b6d6f033690fad83ee8a2238eb Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Tue, 13 May 2025 15:55:18 +0300 Subject: [PATCH 08/10] =?UTF-8?q?=F0=9F=8F=B7=20Use=20`api-server`=20as=20?= =?UTF-8?q?component=20label?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- chart/templates/secrets/jwt-secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/templates/secrets/jwt-secret.yaml b/chart/templates/secrets/jwt-secret.yaml index aee391f33b4cb..e9afdcd3c73fd 100644 --- a/chart/templates/secrets/jwt-secret.yaml +++ b/chart/templates/secrets/jwt-secret.yaml @@ -28,7 +28,7 @@ metadata: name: {{ include "airflow.fullname" . }}-jwt-secret labels: tier: airflow - component: webserver + component: api-server release: {{ .Release.Name }} chart: {{ .Chart.Name }} heritage: {{ .Release.Service }} From 723e4192c386e65938ea3793b32c4a72983244e8 Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Wed, 21 May 2025 17:12:54 +0300 Subject: [PATCH 09/10] Moved secret creation to be a hook --- chart/templates/secrets/jwt-secret.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chart/templates/secrets/jwt-secret.yaml b/chart/templates/secrets/jwt-secret.yaml index e9afdcd3c73fd..a097a55d5565c 100644 --- a/chart/templates/secrets/jwt-secret.yaml +++ b/chart/templates/secrets/jwt-secret.yaml @@ -35,6 +35,10 @@ metadata: {{- with .Values.labels }} {{- toYaml . | nindent 4 }} {{- end }} + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "0" type: Opaque data: jwt-secret: {{ (default $generated_secret_key .Values.jwtSecret) | b64enc | quote }} From 15fcfbb2b64363f63918421d721f946837025efb Mon Sep 17 00:00:00 2001 From: Dov Benyomin Sohacheski Date: Mon, 9 Jun 2025 16:26:20 +0300 Subject: [PATCH 10/10] Update values.schema.json --- chart/values.schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/chart/values.schema.json b/chart/values.schema.json index 3931c636166b3..79583aada47fd 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1481,6 +1481,7 @@ ], "x-docsSection": "Airflow", "default": null + }, "webserverSecretKey": { "description": "The Flask secret key for Airflow Webserver to encrypt browser session.", "type": [