diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 719b6647b585f..69e7dae634643 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -202,16 +202,26 @@ If release name contains chart name it will be used as a full name. {{- if .Values.dags.gitSync.sshKeySecret }} - name: GIT_SSH_KEY_FILE value: "/etc/git-secret/ssh" + - name: GITSYNC_SSH_KEY_FILE + value: "/etc/git-secret/ssh" - name: GIT_SYNC_SSH value: "true" + - name: GITSYNC_SSH + value: "true" {{- if .Values.dags.gitSync.knownHosts }} - name: GIT_KNOWN_HOSTS value: "true" + - name: GITSYNC_SSH_KNOWN_HOSTS + value: "true" - name: GIT_SSH_KNOWN_HOSTS_FILE value: "/etc/git-secret/known_hosts" + - name: GITSYNC_SSH_KNOWN_HOSTS_FILE + value: "/etc/git-secret/known_hosts" {{- else }} - name: GIT_KNOWN_HOSTS value: "false" + - name: GITSYNC_SSH_KNOWN_HOSTS + value: "false" {{- end }} {{ else if .Values.dags.gitSync.credentialsSecret }} - name: GIT_SYNC_USERNAME @@ -219,33 +229,63 @@ If release name contains chart name it will be used as a full name. secretKeyRef: name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GIT_SYNC_USERNAME + - name: GITSYNC_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} + key: GITSYNC_USERNAME - name: GIT_SYNC_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GIT_SYNC_PASSWORD + - name: GITSYNC_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.credentialsSecret | quote }} + key: GITSYNC_PASSWORD {{- end }} - name: GIT_SYNC_REV value: {{ .Values.dags.gitSync.rev | quote }} + - name: GITSYNC_REF + value: {{ .Values.dags.gitSync.ref | quote }} - name: GIT_SYNC_BRANCH value: {{ .Values.dags.gitSync.branch | quote }} - name: GIT_SYNC_REPO value: {{ .Values.dags.gitSync.repo | quote }} + - name: GITSYNC_REPO + value: {{ .Values.dags.gitSync.repo | quote }} - name: GIT_SYNC_DEPTH value: {{ .Values.dags.gitSync.depth | quote }} + - name: GITSYNC_DEPTH + value: {{ .Values.dags.gitSync.depth | quote }} - name: GIT_SYNC_ROOT value: "/git" + - name: GITSYNC_ROOT + value: "/git" - name: GIT_SYNC_DEST value: "repo" + - name: GITSYNC_LINK + value: "repo" - name: GIT_SYNC_ADD_USER value: "true" + - name: GITSYNC_ADD_USER + value: "true" + {{- if .Values.dags.gitSync.wait }} - name: GIT_SYNC_WAIT value: {{ .Values.dags.gitSync.wait | quote }} + {{- end }} + - name: GITSYNC_PERIOD + value: {{ .Values.dags.gitSync.period | quote }} - name: GIT_SYNC_MAX_SYNC_FAILURES value: {{ .Values.dags.gitSync.maxFailures | quote }} + - name: GITSYNC_MAX_FAILURES + value: {{ .Values.dags.gitSync.maxFailures | quote }} {{- if .is_init }} - name: GIT_SYNC_ONE_TIME value: "true" + - name: GITSYNC_ONE_TIME + value: "true" {{- end }} {{- with .Values.dags.gitSync.env }} {{- toYaml . | nindent 4 }} diff --git a/chart/values.schema.json b/chart/values.schema.json index e031376b8145d..8f6b87d0d361d 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -776,7 +776,7 @@ "tag": { "description": "The gitSync image tag.", "type": "string", - "default": "v3.6.9" + "default": "v4.1.0" }, "pullPolicy": { "description": "The gitSync image pull policy.", @@ -7148,6 +7148,11 @@ "type": "string", "default": "HEAD" }, + "ref": { + "description": "Git revision branch, tag, or hash.", + "type": "string", + "default": "v2-2-stable" + }, "depth": { "description": "Repository depth.", "type": "integer", @@ -7165,8 +7170,16 @@ }, "wait": { "description": "Interval between git sync attempts in seconds. High values are more likely to cause DAGs to become out of sync between different components. Low values cause more traffic to the remote git repository.", - "type": "integer", - "default": 5 + "type": [ + "integer", + "null" + ], + "default": null + }, + "period": { + "description": "Interval between git sync attempts in Go-style duration string. High values are more likely to cause DAGs to become out of sync between different components. Low values cause more traffic to the remote git repository.", + "type": "string", + "default": "5s" }, "containerName": { "description": "Git sync container name.", diff --git a/chart/values.yaml b/chart/values.yaml index 763d8248379fd..24840031fd893 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -121,7 +121,7 @@ images: pullPolicy: IfNotPresent gitSync: repository: registry.k8s.io/git-sync/git-sync - tag: v3.6.9 + tag: v4.1.0 pullPolicy: IfNotPresent # Select certain nodes for airflow pods. @@ -2422,6 +2422,8 @@ dags: repo: https://github.com/apache/airflow.git branch: v2-2-stable rev: HEAD + # The git revision (branch, tag, or hash) to check out, v4 only + ref: v2-2-stable depth: 1 # the number of consecutive failures allowed before aborting maxFailures: 0 @@ -2436,8 +2438,12 @@ dags: # metadata: # name: git-credentials # data: + # # For git-sync v3 # GIT_SYNC_USERNAME: # GIT_SYNC_PASSWORD: + # # For git-sync v4 + # GITSYNC_USERNAME: + # GITSYNC_PASSWORD: # and specify the name of the secret below # # credentialsSecret: git-credentials @@ -2466,7 +2472,11 @@ dags: # interval between git sync attempts in seconds # high values are more likely to cause DAGs to become out of sync between different components # low values cause more traffic to the remote git repository - wait: 5 + # Go-style duration string (e.g. "100ms" or "0.1s" = 100ms). + # For backwards compatibility, wait will be used if it is specified. + period: 5s + wait: ~ + containerName: git-sync uid: 65533 diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index c1a831e6b1777..4002dfa3a857c 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -70,10 +70,12 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "gitSync": { "enabled": True, "containerName": "git-sync-test", - "wait": 66, + "wait": None, + "period": "66s", "maxFailures": 70, "subPath": "path1/path2", "rev": "HEAD", + "ref": "test-branch", "depth": 1, "repo": "https://github.com/apache/airflow.git", "branch": "test-branch", @@ -95,15 +97,23 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "imagePullPolicy": "Always", "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, + {"name": "GITSYNC_REF", "value": "test-branch"}, {"name": "GIT_SYNC_BRANCH", "value": "test-branch"}, {"name": "GIT_SYNC_REPO", "value": "https://github.com/apache/airflow.git"}, + {"name": "GITSYNC_REPO", "value": "https://github.com/apache/airflow.git"}, {"name": "GIT_SYNC_DEPTH", "value": "1"}, + {"name": "GITSYNC_DEPTH", "value": "1"}, {"name": "GIT_SYNC_ROOT", "value": "/git"}, + {"name": "GITSYNC_ROOT", "value": "/git"}, {"name": "GIT_SYNC_DEST", "value": "repo"}, + {"name": "GITSYNC_LINK", "value": "repo"}, {"name": "GIT_SYNC_ADD_USER", "value": "true"}, - {"name": "GIT_SYNC_WAIT", "value": "66"}, + {"name": "GITSYNC_ADD_USER", "value": "true"}, + {"name": "GITSYNC_PERIOD", "value": "66s"}, {"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"}, + {"name": "GITSYNC_MAX_FAILURES", "value": "70"}, {"name": "GIT_SYNC_ONE_TIME", "value": "true"}, + {"name": "GITSYNC_ONE_TIME", "value": "true"}, ], "volumeMounts": [{"mountPath": "/git", "name": "dags"}], "resources": {}, @@ -185,12 +195,21 @@ def test_validate_if_ssh_params_are_added(self): assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( "spec.initContainers[0].env", docs[0] ) + assert {"name": "GITSYNC_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( + "spec.initContainers[0].env", docs[0] + ) assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search( "spec.initContainers[0].env", docs[0] ) + assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search( + "spec.initContainers[0].env", docs[0] + ) assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in jmespath.search( "spec.initContainers[0].env", docs[0] ) + assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in jmespath.search( + "spec.initContainers[0].env", docs[0] + ) assert { "name": "git-sync-ssh-key", "mountPath": "/etc/git-secret/ssh", @@ -256,6 +275,16 @@ def test_should_set_username_and_pass_env_variables(self): "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_PASSWORD"}}, } in jmespath.search("spec.initContainers[0].env", docs[0]) + # Testing git-sync v4 + assert { + "name": "GITSYNC_USERNAME", + "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_USERNAME"}}, + } in jmespath.search("spec.initContainers[0].env", docs[0]) + assert { + "name": "GITSYNC_PASSWORD", + "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_PASSWORD"}}, + } in jmespath.search("spec.initContainers[0].env", docs[0]) + def test_should_set_the_dags_volume_claim_correctly_when_using_an_existing_claim(self): docs = render_chart( values={"dags": {"persistence": {"enabled": True, "existingClaim": "test-claim"}}}, diff --git a/helm_tests/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py index 91039954f8d6e..7bf7b83ae5572 100644 --- a/helm_tests/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -52,6 +52,66 @@ def test_should_add_dags_volume(self): ) def test_validate_the_git_sync_container_spec(self): + docs = render_chart( + values={ + "images": { + "gitSync": { + "repository": "test-registry/test-repo", + "tag": "test-tag", + "pullPolicy": "Always", + } + }, + "dags": { + "gitSync": { + "enabled": True, + "containerName": "git-sync-test", + "wait": None, + "period": "66s", + "maxFailures": 70, + "subPath": "path1/path2", + "rev": "HEAD", + "ref": "test-branch", + "depth": 1, + "repo": "https://github.com/apache/airflow.git", + "branch": "test-branch", + "sshKeySecret": None, + "credentialsSecret": None, + "knownHosts": None, + }, + "persistence": {"enabled": True}, + }, + }, + show_only=["templates/scheduler/scheduler-deployment.yaml"], + ) + + assert { + "name": "git-sync-test", + "securityContext": {"runAsUser": 65533}, + "image": "test-registry/test-repo:test-tag", + "imagePullPolicy": "Always", + "env": [ + {"name": "GIT_SYNC_REV", "value": "HEAD"}, + {"name": "GITSYNC_REF", "value": "test-branch"}, + {"name": "GIT_SYNC_BRANCH", "value": "test-branch"}, + {"name": "GIT_SYNC_REPO", "value": "https://github.com/apache/airflow.git"}, + {"name": "GITSYNC_REPO", "value": "https://github.com/apache/airflow.git"}, + {"name": "GIT_SYNC_DEPTH", "value": "1"}, + {"name": "GITSYNC_DEPTH", "value": "1"}, + {"name": "GIT_SYNC_ROOT", "value": "/git"}, + {"name": "GITSYNC_ROOT", "value": "/git"}, + {"name": "GIT_SYNC_DEST", "value": "repo"}, + {"name": "GITSYNC_LINK", "value": "repo"}, + {"name": "GIT_SYNC_ADD_USER", "value": "true"}, + {"name": "GITSYNC_ADD_USER", "value": "true"}, + {"name": "GITSYNC_PERIOD", "value": "66s"}, + {"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"}, + {"name": "GITSYNC_MAX_FAILURES", "value": "70"}, + ], + "volumeMounts": [{"mountPath": "/git", "name": "dags"}], + "resources": {}, + } == jmespath.search("spec.template.spec.containers[1]", docs[0]) + + def test_validate_the_git_sync_container_spec_if_wait_specified(self): docs = render_chart( values={ "images": { @@ -66,9 +126,11 @@ def test_validate_the_git_sync_container_spec(self): "enabled": True, "containerName": "git-sync-test", "wait": 66, + "period": "66s", "maxFailures": 70, "subPath": "path1/path2", "rev": "HEAD", + "ref": "test-branch", "depth": 1, "repo": "https://github.com/apache/airflow.git", "branch": "test-branch", @@ -89,14 +151,22 @@ def test_validate_the_git_sync_container_spec(self): "imagePullPolicy": "Always", "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, + {"name": "GITSYNC_REF", "value": "test-branch"}, {"name": "GIT_SYNC_BRANCH", "value": "test-branch"}, {"name": "GIT_SYNC_REPO", "value": "https://github.com/apache/airflow.git"}, + {"name": "GITSYNC_REPO", "value": "https://github.com/apache/airflow.git"}, {"name": "GIT_SYNC_DEPTH", "value": "1"}, + {"name": "GITSYNC_DEPTH", "value": "1"}, {"name": "GIT_SYNC_ROOT", "value": "/git"}, + {"name": "GITSYNC_ROOT", "value": "/git"}, {"name": "GIT_SYNC_DEST", "value": "repo"}, + {"name": "GITSYNC_LINK", "value": "repo"}, {"name": "GIT_SYNC_ADD_USER", "value": "true"}, + {"name": "GITSYNC_ADD_USER", "value": "true"}, {"name": "GIT_SYNC_WAIT", "value": "66"}, + {"name": "GITSYNC_PERIOD", "value": "66s"}, {"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"}, + {"name": "GITSYNC_MAX_FAILURES", "value": "70"}, ], "volumeMounts": [{"mountPath": "/git", "name": "dags"}], "resources": {}, @@ -121,12 +191,21 @@ def test_validate_if_ssh_params_are_added(self): assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( "spec.template.spec.containers[1].env", docs[0] ) + assert {"name": "GITSYNC_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search( "spec.template.spec.containers[1].env", docs[0] ) + assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in jmespath.search( "spec.template.spec.containers[1].env", docs[0] ) + assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in jmespath.search( + "spec.template.spec.containers[1].env", docs[0] + ) assert { "name": "git-sync-ssh-key", "secret": {"secretName": "ssh-secret", "defaultMode": 288}, @@ -173,6 +252,16 @@ def test_should_set_username_and_pass_env_variables(self): "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GIT_SYNC_PASSWORD"}}, } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + # Testing git-sync v4 + assert { + "name": "GITSYNC_USERNAME", + "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_USERNAME"}}, + } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + assert { + "name": "GITSYNC_PASSWORD", + "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key": "GITSYNC_PASSWORD"}}, + } in jmespath.search("spec.template.spec.containers[1].env", docs[0]) + def test_should_set_the_volume_claim_correctly_when_using_an_existing_claim(self): docs = render_chart( values={"dags": {"persistence": {"enabled": True, "existingClaim": "test-claim"}}},