diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 4415372dde7fe..f4f0efd27b63a 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -24,10 +24,15 @@ metadata: {{- toYaml .Values.airflowPodAnnotations | nindent 4 }} {{- end }} spec: -{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} + {{- if or (and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled)) .Values.workers.extraInitContainers }} initContainers: -{{- include "git_sync_container" (dict "Values" .Values "is_init" "true") | indent 4 }} -{{- end }} + {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} + {{- include "git_sync_container" (dict "Values" .Values "is_init" "true") | nindent 4 }} + {{- end }} + {{- if .Values.workers.extraInitContainers }} + {{- toYaml .Values.workers.extraInitContainers | nindent 4 }} + {{- end }} + {{- end }} containers: - args: [] command: [] diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 8137c846ccbbf..0065eca8cb259 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -113,6 +113,9 @@ spec: env: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} + {{- if .Values.scheduler.extraInitContainers }} + {{- toYaml .Values.scheduler.extraInitContainers | nindent 8 }} + {{- end }} containers: # Always run the main scheduler container. - name: scheduler diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml index f17679fcb2201..385270cf18e3e 100644 --- a/chart/templates/webserver/webserver-deployment.yaml +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -107,6 +107,9 @@ spec: env: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} + {{- if .Values.webserver.extraInitContainers }} + {{- toYaml .Values.webserver.extraInitContainers | nindent 8 }} + {{- end }} containers: - name: webserver image: {{ template "airflow_image" . }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index b4897fd0c1cdf..cab0de3d9deaa 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -123,6 +123,9 @@ spec: env: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} + {{- if .Values.workers.extraInitContainers }} + {{- toYaml .Values.workers.extraInitContainers | nindent 8 }} + {{- end }} containers: - name: worker image: {{ template "airflow_image" . }} diff --git a/chart/tests/test_pod_template_file.py b/chart/tests/test_pod_template_file.py index c6d57bff8df0c..0b46f66e05457 100644 --- a/chart/tests/test_pod_template_file.py +++ b/chart/tests/test_pod_template_file.py @@ -420,3 +420,21 @@ def test_airflow_pod_annotations(self): annotations = jmespath.search("metadata.annotations", docs[0]) assert "my_annotation" in annotations assert "annotated!" in annotations["my_annotation"] + + def test_should_add_extra_init_containers(self): + docs = render_chart( + values={ + "workers": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ], + }, + }, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } == jmespath.search("spec.initContainers[-1]", docs[0]) diff --git a/chart/tests/test_scheduler.py b/chart/tests/test_scheduler.py index fd3e198518cbe..446cbf484062f 100644 --- a/chart/tests/test_scheduler.py +++ b/chart/tests/test_scheduler.py @@ -55,18 +55,34 @@ def test_should_add_extra_containers(self): "executor": "CeleryExecutor", "scheduler": { "extraContainers": [ - { - "name": "test-container", - "image": "test-registry/test-repo:test-tag", - "imagePullPolicy": "Always", - } + {"name": "test-container", "image": "test-registry/test-repo:test-tag"} ], }, }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) - assert "test-container" == jmespath.search("spec.template.spec.containers[-1].name", docs[0]) + assert { + "name": "test-container", + "image": "test-registry/test-repo:test-tag", + } == jmespath.search("spec.template.spec.containers[-1]", docs[0]) + + def test_should_add_extra_init_containers(self): + docs = render_chart( + values={ + "scheduler": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ], + }, + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + ) + + assert { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0]) def test_should_add_extra_volume_and_extra_volume_mount(self): docs = render_chart( diff --git a/chart/tests/test_webserver.py b/chart/tests/test_webserver.py index b40e75c99a2e6..d173ca556a23b 100644 --- a/chart/tests/test_webserver.py +++ b/chart/tests/test_webserver.py @@ -132,18 +132,34 @@ def test_should_add_extra_containers(self): "executor": "CeleryExecutor", "webserver": { "extraContainers": [ - { - "name": "test-container", - "image": "test-registry/test-repo:test-tag", - "imagePullPolicy": "Always", - } + {"name": "test-container", "image": "test-registry/test-repo:test-tag"} ], }, }, show_only=["templates/webserver/webserver-deployment.yaml"], ) - assert "test-container" == jmespath.search("spec.template.spec.containers[-1].name", docs[0]) + assert { + "name": "test-container", + "image": "test-registry/test-repo:test-tag", + } == jmespath.search("spec.template.spec.containers[-1]", docs[0]) + + def test_should_add_extra_init_containers(self): + docs = render_chart( + values={ + "webserver": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ], + }, + }, + show_only=["templates/webserver/webserver-deployment.yaml"], + ) + + assert { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0]) def test_should_create_valid_affinity_tolerations_and_node_selector(self): docs = render_chart( diff --git a/chart/tests/test_worker.py b/chart/tests/test_worker.py index eaa772c22c4a9..f2b825fd29c42 100644 --- a/chart/tests/test_worker.py +++ b/chart/tests/test_worker.py @@ -52,18 +52,34 @@ def test_should_add_extra_containers(self): "executor": "CeleryExecutor", "workers": { "extraContainers": [ - { - "name": "test-container", - "image": "test-registry/test-repo:test-tag", - "imagePullPolicy": "Always", - } + {"name": "test-container", "image": "test-registry/test-repo:test-tag"} ], }, }, show_only=["templates/workers/worker-deployment.yaml"], ) - assert "test-container" == jmespath.search("spec.template.spec.containers[-1].name", docs[0]) + assert { + "name": "test-container", + "image": "test-registry/test-repo:test-tag", + } == jmespath.search("spec.template.spec.containers[-1]", docs[0]) + + def test_should_add_extra_init_containers(self): + docs = render_chart( + values={ + "workers": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ], + }, + }, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + assert { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } == jmespath.search("spec.template.spec.initContainers[-1]", docs[0]) def test_should_add_extra_volume_and_extra_volume_mount(self): docs = render_chart( diff --git a/chart/values.schema.json b/chart/values.schema.json index 9f94ae0e35f49..73e05dd63303a 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1001,6 +1001,11 @@ "type": "array", "default": [] }, + "extraInitContainers": { + "description": "Add additional init containers into workers.", + "type": "array", + "default": [] + }, "extraVolumes": { "description": "Mount additional volumes into workers.", "type": "array", @@ -1179,6 +1184,11 @@ "type": "array", "default": [] }, + "extraInitContainers": { + "description": "Add additional init containers into scheduler.", + "type": "array", + "default": [] + }, "extraVolumes": { "description": "Mount additional volumes into scheduler.", "type": "array", @@ -1455,6 +1465,11 @@ "type": "array", "default": [] }, + "extraInitContainers": { + "description": "Add additional init containers into webserver.", + "type": "array", + "default": [] + }, "extraVolumes": { "description": "Mount additional volumes into webserver.", "type": "array", diff --git a/chart/values.yaml b/chart/values.yaml index d498b5d06c3a0..cda9e1e1089e0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -397,6 +397,8 @@ workers: # Launch additional containers into worker. extraContainers: [] + # Add additional init containers into workers. + extraInitContainers: [] # Mount additional volumes into worker. extraVolumes: [] @@ -469,6 +471,8 @@ scheduler: # Launch additional containers into scheduler. extraContainers: [] + # Add additional init containers into scheduler. + extraInitContainers: [] # Mount additional volumes into scheduler. extraVolumes: [] @@ -566,6 +570,8 @@ webserver: # Launch additional containers into webserver. extraContainers: [] + # Add additional init containers into webserver. + extraInitContainers: [] # Mount additional volumes into webserver. extraVolumes: [] diff --git a/docs/helm-chart/using-additional-containers.rst b/docs/helm-chart/using-additional-containers.rst index a3c44fdf7035b..d75214e38d37c 100644 --- a/docs/helm-chart/using-additional-containers.rst +++ b/docs/helm-chart/using-additional-containers.rst @@ -16,20 +16,48 @@ under the License. Using additional containers ----------------------------- +=========================== -If you are using your own sidecar container, you can add it through the ``extraContainers`` value. -You can define different containers for scheduler, webserver and worker pods. +Sidecar Containers +------------------ -For example, a sidecar that syncs DAGs from object storage. +If you want to deploy your own sidecar container, you can add it through the ``extraContainers`` parameter. +You can define different containers for the scheduler, webserver and worker pods. + +For example, sidecars that sync DAGs from object storage. .. note:: - ``extraContainers`` value supports CeleryExecutor only. + ``workers.extraContainers`` is only functional with ``CeleryExecutor``. .. code-block:: yaml + scheduler: + extraContainers: + - name: s3-sync + image: my-company/s3-sync:latest + imagePullPolicy: Always + workers: extraContainers: - name: s3-sync image: my-company/s3-sync:latest imagePullPolicy: Always + + +Init Containers +--------------- + +You can also deploy extra init containers through the ``extraInitContainers`` parameter. +You can define different containers for the scheduler, webserver and worker pods. + +For example, an init container that just says hello: + +.. code-block:: yaml + + scheduler: + extraInitContainers: + - name: hello + image: debian + args: + - echo + - hello