[AIRFLOW-4846] Allow specification of an existing secret containing git credentials for init containers

airflow/config_templates/default_airflow.cfg

@@ -548,8 +548,8 @@ basedn = dc=example,dc=com
 cacert = /etc/ca/ldap_ca.crt
 search_scope = LEVEL
 
-# This setting allows the use of LDAP servers that either return a 
-# broken schema, or do not return a schema. 
+# This setting allows the use of LDAP servers that either return a
+# broken schema, or do not return a schema.
 ignore_malformed_schema = False
 
 [kerberos]
@@ -678,6 +678,20 @@ git_dags_folder_mount_point =
 git_ssh_key_secret_name =
 git_ssh_known_hosts_configmap_name =
 
+# To give the git_sync init container credentials via a secret, create a secret
+# with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and
+# add `git_sync_credentials_secret = <secret_name>` to your airflow config under the kubernetes section
+#
+# Secret Example:
+# apiVersion: v1
+# kind: Secret
+# metadata:
+#   name: git-credentials
+# data:
+#   GIT_SYNC_USERNAME: <base64_encoded_git_username>
+#   GIT_SYNC_PASSWORD: <base64_encoded_git_password>
+git_sync_credentials_secret =
+
 # For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync
 git_sync_container_repository = k8s.gcr.io/git-sync
 git_sync_container_tag = v3.1.1

airflow/executors/kubernetes_executor.py

@@ -186,6 +186,8 @@ def __init__(self):
         self.git_ssh_key_secret_name = conf.get(self.kubernetes_section, 'git_ssh_key_secret_name')
         self.git_ssh_known_hosts_configmap_name = conf.get(self.kubernetes_section,
                                                            'git_ssh_known_hosts_configmap_name')
+        self.git_sync_credentials_secret = conf.get(self.kubernetes_section,
+                                                    'git_sync_credentials_secret')
 
         # NOTE: The user may optionally use a volume claim to mount a PV containing
         # DAGs directly

airflow/kubernetes/worker_configuration.py

@@ -80,6 +80,28 @@ def _get_init_containers(self):
                 'value': self.kube_config.git_password
             })
 
+        if self.kube_config.git_sync_credentials_secret:
+            init_environment.extend([
+                {
+                    'name': 'GIT_SYNC_USERNAME',
+                    'valueFrom': {
+                        'secretKeyRef': {
+                            'name': self.kube_config.git_sync_credentials_secret,
+                            'key': 'GIT_SYNC_USERNAME'
+                        }
+                    }
+                },
+                {
+                    'name': 'GIT_SYNC_PASSWORD',
+                    'valueFrom': {
+                        'secretKeyRef': {
+                            'name': self.kube_config.git_sync_credentials_secret,
+                            'key': 'GIT_SYNC_PASSWORD'
+                        }
+                    }
+                }
+            ])
+
         volume_mounts = [{
             'mountPath': self.kube_config.git_sync_root,
             'name': self.dags_volume_name,

tests/executors/test_kubernetes_executor.py

@@ -376,6 +376,48 @@ def test_init_environment_using_git_sync_user_with_known_hosts(self):
                         'value': '/etc/git-secret/known_hosts'} in env)
         self.assertFalse({'name': 'GIT_SYNC_SSH', 'value': 'true'} in env)
 
+    def test_make_pod_git_sync_credentials_secret(self):
+        # Tests the pod created with git_sync_credentials_secret will get into the init container
+        self.kube_config.git_sync_credentials_secret = 'airflow-git-creds-secret'
+        self.kube_config.dags_volume_claim = None
+        self.kube_config.dags_volume_host = None
+        self.kube_config.dags_in_image = None
+        self.kube_config.worker_fs_group = None
+
+        worker_config = WorkerConfiguration(self.kube_config)
+        kube_executor_config = KubernetesExecutorConfig(annotations=[],
+                                                        volumes=[],
+                                                        volume_mounts=[])
+
+        pod = worker_config.make_pod("default", str(uuid.uuid4()), "test_pod_id", "test_dag_id",
+                                     "test_task_id", str(datetime.utcnow()), 1, "bash -c 'ls /'",
+                                     kube_executor_config)
+
+        username_env = {
+            'name': 'GIT_SYNC_USERNAME',
+            'valueFrom': {
+                'secretKeyRef': {
+                    'name': self.kube_config.git_sync_credentials_secret,
+                    'key': 'GIT_SYNC_USERNAME'
+                }
+            }
+        }
+        password_env = {
+            'name': 'GIT_SYNC_PASSWORD',
+            'valueFrom': {
+                'secretKeyRef': {
+                    'name': self.kube_config.git_sync_credentials_secret,
+                    'key': 'GIT_SYNC_PASSWORD'
+                }
+            }
+        }
+
+        self.assertIn(username_env, pod.init_containers[0]["env"],
+                      'The username env for git credentials did not get into the init container')
+
+        self.assertIn(password_env, pod.init_containers[0]["env"],
+                      'The password env for git credentials did not get into the init container')
+
     def test_init_environment_using_git_sync_run_as_user_empty(self):
         # Tests if git_syn_run_as_user is none, then no securityContext created in init container