Support git-sync v4 in Helm Chart

chart/templates/_helpers.yaml

@@ -202,50 +202,90 @@ If release name contains chart name it will be used as a full name.
     {{- if .Values.dags.gitSync.sshKeySecret }}
     - name: GIT_SSH_KEY_FILE
       value: "/etc/git-secret/ssh"
+    - name: GITSYNC_SSH_KEY_FILE
+      value: "/etc/git-secret/ssh"
     - name: GIT_SYNC_SSH
       value: "true"
+    - name: GITSYNC_SSH
+      value: "true"
     {{- if .Values.dags.gitSync.knownHosts }}
     - name: GIT_KNOWN_HOSTS
       value: "true"
+    - name: GITSYNC_SSH_KNOWN_HOSTS
+      value: "true"
     - name: GIT_SSH_KNOWN_HOSTS_FILE
       value: "/etc/git-secret/known_hosts"
+    - name: GITSYNC_SSH_KNOWN_HOSTS_FILE
+      value: "/etc/git-secret/known_hosts"
     {{- else }}
     - name: GIT_KNOWN_HOSTS
       value: "false"
+    - name: GITSYNC_SSH_KNOWN_HOSTS
+      value: "false"
     {{- end }}
     {{ else if .Values.dags.gitSync.credentialsSecret }}
     - name: GIT_SYNC_USERNAME
       valueFrom:
         secretKeyRef:
           name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
           key: GIT_SYNC_USERNAME
+    - name: GITSYNC_USERNAME
+      valueFrom:
+        secretKeyRef:
+          name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
+          key: GITSYNC_USERNAME
     - name: GIT_SYNC_PASSWORD
       valueFrom:
         secretKeyRef:
           name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
           key: GIT_SYNC_PASSWORD
+    - name: GITSYNC_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
+          key: GITSYNC_PASSWORD
     {{- end }}
     - name: GIT_SYNC_REV
       value: {{ .Values.dags.gitSync.rev | quote }}
+    - name: GITSYNC_REF
+      value: {{ .Values.dags.gitSync.ref | quote }}
     - name: GIT_SYNC_BRANCH
       value: {{ .Values.dags.gitSync.branch | quote }}
     - name: GIT_SYNC_REPO
       value: {{ .Values.dags.gitSync.repo | quote }}
+    - name: GITSYNC_REPO
+      value: {{ .Values.dags.gitSync.repo | quote }}
     - name: GIT_SYNC_DEPTH
       value: {{ .Values.dags.gitSync.depth | quote }}
+    - name: GITSYNC_DEPTH
+      value: {{ .Values.dags.gitSync.depth | quote }}
     - name: GIT_SYNC_ROOT
       value: "/git"
+    - name: GITSYNC_ROOT
+      value: "/git"
     - name: GIT_SYNC_DEST
       value: "repo"
+    - name: GITSYNC_LINK
+      value: "repo"
     - name: GIT_SYNC_ADD_USER
       value: "true"
+    - name: GITSYNC_ADD_USER
+      value: "true"
+    {{- if .Values.dags.gitSync.wait }}
     - name: GIT_SYNC_WAIT
       value: {{ .Values.dags.gitSync.wait | quote }}
+    {{- end }}
+    - name: GITSYNC_PERIOD
+      value: {{ .Values.dags.gitSync.period | quote }}
     - name: GIT_SYNC_MAX_SYNC_FAILURES
       value: {{ .Values.dags.gitSync.maxFailures | quote }}
+    - name: GITSYNC_MAX_FAILURES
+      value: {{ .Values.dags.gitSync.maxFailures | quote }}
     {{- if .is_init }}
     - name: GIT_SYNC_ONE_TIME
       value: "true"
+    - name: GITSYNC_ONE_TIME
+      value: "true"
     {{- end }}
     {{- with .Values.dags.gitSync.env }}
     {{- toYaml . | nindent 4 }}

chart/values.schema.json

@@ -776,7 +776,7 @@
                         "tag": {
                             "description": "The gitSync image tag.",
                             "type": "string",
-                            "default": "v3.6.9"
+                            "default": "v4.1.0"
                         },
                         "pullPolicy": {
                             "description": "The gitSync image pull policy.",
@@ -7148,6 +7148,11 @@
                             "type": "string",
                             "default": "HEAD"
                         },
+                        "ref": {
+                            "description": "Git revision branch, tag, or hash.",
+                            "type": "string",
+                            "default": "v2-2-stable"
+                        },
                         "depth": {
                             "description": "Repository depth.",
                             "type": "integer",
@@ -7165,8 +7170,16 @@
                         },
                         "wait": {
                             "description": "Interval between git sync attempts in seconds. High values are more likely to cause DAGs to become out of sync between different components. Low values cause more traffic to the remote git repository.",
-                            "type": "integer",
-                            "default": 5
+                            "type": [
+                                "integer",
+                                "null"
+                            ],
+                            "default": null
+                        },
+                        "period": {
+                            "description": "Interval between git sync attempts in Go-style duration string. High values are more likely to cause DAGs to become out of sync between different components. Low values cause more traffic to the remote git repository.",
+                            "type": "string",
+                            "default": "5s"
                         },
                         "containerName": {
                             "description": "Git sync container name.",

chart/values.yaml

@@ -121,7 +121,7 @@ images:
     pullPolicy: IfNotPresent
   gitSync:
     repository: registry.k8s.io/git-sync/git-sync
-    tag: v3.6.9
+    tag: v4.1.0
     pullPolicy: IfNotPresent
 
 # Select certain nodes for airflow pods.
@@ -2422,6 +2422,8 @@ dags:
     repo: https://github.com/apache/airflow.git
     branch: v2-2-stable
     rev: HEAD
+    # The git revision (branch, tag, or hash) to check out, v4 only
+    ref: v2-2-stable
     depth: 1
     # the number of consecutive failures allowed before aborting
     maxFailures: 0
@@ -2436,8 +2438,12 @@ dags:
     #   metadata:
     #     name: git-credentials
     #   data:
+    #     # For git-sync v3
     #     GIT_SYNC_USERNAME: <base64_encoded_git_username>
     #     GIT_SYNC_PASSWORD: <base64_encoded_git_password>
+    #     # For git-sync v4
+    #     GITSYNC_USERNAME: <base64_encoded_git_username>
+    #     GITSYNC_PASSWORD: <base64_encoded_git_password>
     # and specify the name of the secret below
     #
     # credentialsSecret: git-credentials
@@ -2466,7 +2472,11 @@ dags:
     # interval between git sync attempts in seconds
     # high values are more likely to cause DAGs to become out of sync between different components
     # low values cause more traffic to the remote git repository
-    wait: 5
+    # Go-style duration string (e.g. "100ms" or "0.1s" = 100ms).
+    # For backwards compatibility, wait will be used if it is specified.
+    period: 5s
+    wait: ~
+
     containerName: git-sync
     uid: 65533
 

helm_tests/airflow_aux/test_pod_template_file.py

@@ -70,10 +70,12 @@ def test_should_add_an_init_container_if_git_sync_is_true(self):
                     "gitSync": {
                         "enabled": True,
                         "containerName": "git-sync-test",
-                        "wait": 66,
+                        "wait": None,
+                        "period": "66s",
                         "maxFailures": 70,
                         "subPath": "path1/path2",
                         "rev": "HEAD",
+                        "ref": "test-branch",
                         "depth": 1,
                         "repo": "https://github.com/apache/airflow.git",
                         "branch": "test-branch",
@@ -95,15 +97,23 @@ def test_should_add_an_init_container_if_git_sync_is_true(self):
             "imagePullPolicy": "Always",
             "env": [
                 {"name": "GIT_SYNC_REV", "value": "HEAD"},
+                {"name": "GITSYNC_REF", "value": "test-branch"},
                 {"name": "GIT_SYNC_BRANCH", "value": "test-branch"},
                 {"name": "GIT_SYNC_REPO", "value": "https://github.com/apache/airflow.git"},
+                {"name": "GITSYNC_REPO", "value": "https://github.com/apache/airflow.git"},
                 {"name": "GIT_SYNC_DEPTH", "value": "1"},
+                {"name": "GITSYNC_DEPTH", "value": "1"},
                 {"name": "GIT_SYNC_ROOT", "value": "/git"},
+                {"name": "GITSYNC_ROOT", "value": "/git"},
                 {"name": "GIT_SYNC_DEST", "value": "repo"},
+                {"name": "GITSYNC_LINK", "value": "repo"},
                 {"name": "GIT_SYNC_ADD_USER", "value": "true"},
-                {"name": "GIT_SYNC_WAIT", "value": "66"},
+                {"name": "GITSYNC_ADD_USER", "value": "true"},
+                {"name": "GITSYNC_PERIOD", "value": "66s"},
                 {"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"},
+                {"name": "GITSYNC_MAX_FAILURES", "value": "70"},
                 {"name": "GIT_SYNC_ONE_TIME", "value": "true"},
+                {"name": "GITSYNC_ONE_TIME", "value": "true"},
             ],
             "volumeMounts": [{"mountPath": "/git", "name": "dags"}],
             "resources": {},
@@ -185,12 +195,21 @@ def test_validate_if_ssh_params_are_added(self):
         assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search(
             "spec.initContainers[0].env", docs[0]
         )
+        assert {"name": "GITSYNC_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search(
+            "spec.initContainers[0].env", docs[0]
+        )
         assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search(
             "spec.initContainers[0].env", docs[0]
         )
+        assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search(
+            "spec.initContainers[0].env", docs[0]
+        )
         assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in jmespath.search(
             "spec.initContainers[0].env", docs[0]
         )
+        assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in jmespath.search(
+            "spec.initContainers[0].env", docs[0]
+        )
         assert {
             "name": "git-sync-ssh-key",
             "mountPath": "/etc/git-secret/ssh",
@@ -256,6 +275,16 @@ def test_should_set_username_and_pass_env_variables(self):
             "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_PASSWORD"}},
         } in jmespath.search("spec.initContainers[0].env", docs[0])
 
+        # Testing git-sync v4
+        assert {
+            "name": "GITSYNC_USERNAME",
+            "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_USERNAME"}},
+        } in jmespath.search("spec.initContainers[0].env", docs[0])
+        assert {
+            "name": "GITSYNC_PASSWORD",
+            "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_PASSWORD"}},
+        } in jmespath.search("spec.initContainers[0].env", docs[0])
+
     def test_should_set_the_dags_volume_claim_correctly_when_using_an_existing_claim(self):
         docs = render_chart(
             values={"dags": {"persistence": {"enabled": True, "existingClaim": "test-claim"}}},

helm_tests/other/test_git_sync_scheduler.py

@@ -52,6 +52,66 @@ def test_should_add_dags_volume(self):
         )
 
     def test_validate_the_git_sync_container_spec(self):
+        docs = render_chart(
+            values={
+                "images": {
+                    "gitSync": {
+                        "repository": "test-registry/test-repo",
+                        "tag": "test-tag",
+                        "pullPolicy": "Always",
+                    }
+                },
+                "dags": {
+                    "gitSync": {
+                        "enabled": True,
+                        "containerName": "git-sync-test",
+                        "wait": None,
+                        "period": "66s",
+                        "maxFailures": 70,
+                        "subPath": "path1/path2",
+                        "rev": "HEAD",
+                        "ref": "test-branch",
+                        "depth": 1,
+                        "repo": "https://github.com/apache/airflow.git",
+                        "branch": "test-branch",
+                        "sshKeySecret": None,
+                        "credentialsSecret": None,
+                        "knownHosts": None,
+                    },
+                    "persistence": {"enabled": True},
+                },
+            },
+            show_only=["templates/scheduler/scheduler-deployment.yaml"],
+        )
+
+        assert {
+            "name": "git-sync-test",
+            "securityContext": {"runAsUser": 65533},
+            "image": "test-registry/test-repo:test-tag",
+            "imagePullPolicy": "Always",
+            "env": [
+                {"name": "GIT_SYNC_REV", "value": "HEAD"},
+                {"name": "GITSYNC_REF", "value": "test-branch"},
+                {"name": "GIT_SYNC_BRANCH", "value": "test-branch"},
+                {"name": "GIT_SYNC_REPO", "value": "https://github.com/apache/airflow.git"},
+                {"name": "GITSYNC_REPO", "value": "https://github.com/apache/airflow.git"},
+                {"name": "GIT_SYNC_DEPTH", "value": "1"},
+                {"name": "GITSYNC_DEPTH", "value": "1"},
+                {"name": "GIT_SYNC_ROOT", "value": "/git"},
+                {"name": "GITSYNC_ROOT", "value": "/git"},
+                {"name": "GIT_SYNC_DEST", "value": "repo"},
+                {"name": "GITSYNC_LINK", "value": "repo"},
+                {"name": "GIT_SYNC_ADD_USER", "value": "true"},
+                {"name": "GITSYNC_ADD_USER", "value": "true"},
+                {"name": "GITSYNC_PERIOD", "value": "66s"},
+                {"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"},
+                {"name": "GITSYNC_MAX_FAILURES", "value": "70"},
+            ],
+            "volumeMounts": [{"mountPath": "/git", "name": "dags"}],
+            "resources": {},
+        } == jmespath.search("spec.template.spec.containers[1]", docs[0])
+
+    def test_validate_the_git_sync_container_spec_if_wait_specified(self):
         docs = render_chart(
             values={
                 "images": {
@@ -66,9 +126,11 @@ def test_validate_the_git_sync_container_spec(self):
                         "enabled": True,
                         "containerName": "git-sync-test",
                         "wait": 66,
+                        "period": "66s",
                         "maxFailures": 70,
                         "subPath": "path1/path2",
                         "rev": "HEAD",
+                        "ref": "test-branch",
                         "depth": 1,
                         "repo": "https://github.com/apache/airflow.git",
                         "branch": "test-branch",
@@ -89,14 +151,22 @@ def test_validate_the_git_sync_container_spec(self):
             "imagePullPolicy": "Always",
             "env": [
                 {"name": "GIT_SYNC_REV", "value": "HEAD"},
+                {"name": "GITSYNC_REF", "value": "test-branch"},
                 {"name": "GIT_SYNC_BRANCH", "value": "test-branch"},
                 {"name": "GIT_SYNC_REPO", "value": "https://github.com/apache/airflow.git"},
+                {"name": "GITSYNC_REPO", "value": "https://github.com/apache/airflow.git"},
                 {"name": "GIT_SYNC_DEPTH", "value": "1"},
+                {"name": "GITSYNC_DEPTH", "value": "1"},
                 {"name": "GIT_SYNC_ROOT", "value": "/git"},
+                {"name": "GITSYNC_ROOT", "value": "/git"},
                 {"name": "GIT_SYNC_DEST", "value": "repo"},
+                {"name": "GITSYNC_LINK", "value": "repo"},
                 {"name": "GIT_SYNC_ADD_USER", "value": "true"},
+                {"name": "GITSYNC_ADD_USER", "value": "true"},
                 {"name": "GIT_SYNC_WAIT", "value": "66"},
+                {"name": "GITSYNC_PERIOD", "value": "66s"},
                 {"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"},
+                {"name": "GITSYNC_MAX_FAILURES", "value": "70"},
             ],
             "volumeMounts": [{"mountPath": "/git", "name": "dags"}],
             "resources": {},
@@ -121,12 +191,21 @@ def test_validate_if_ssh_params_are_added(self):
         assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search(
             "spec.template.spec.containers[1].env", docs[0]
         )
+        assert {"name": "GITSYNC_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search(
+            "spec.template.spec.containers[1].env", docs[0]
+        )
         assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search(
             "spec.template.spec.containers[1].env", docs[0]
         )
+        assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search(
+            "spec.template.spec.containers[1].env", docs[0]
+        )
         assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in jmespath.search(
             "spec.template.spec.containers[1].env", docs[0]
         )
+        assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in jmespath.search(
+            "spec.template.spec.containers[1].env", docs[0]
+        )
         assert {
             "name": "git-sync-ssh-key",
             "secret": {"secretName": "ssh-secret", "defaultMode": 288},
@@ -173,6 +252,16 @@ def test_should_set_username_and_pass_env_variables(self):
             "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_PASSWORD"}},
         } in jmespath.search("spec.template.spec.containers[1].env", docs[0])
 
+        # Testing git-sync v4
+        assert {
+            "name": "GITSYNC_USERNAME",
+            "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_USERNAME"}},
+        } in jmespath.search("spec.template.spec.containers[1].env", docs[0])
+        assert {
+            "name": "GITSYNC_PASSWORD",
+            "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_PASSWORD"}},
+        } in jmespath.search("spec.template.spec.containers[1].env", docs[0])
+
     def test_should_set_the_volume_claim_correctly_when_using_an_existing_claim(self):
         docs = render_chart(
             values={"dags": {"persistence": {"enabled": True, "existingClaim": "test-claim"}}},