Allow state storage to be cross-cloud configurable in helm

Currently this is hardcoded to work w/ minio only which breaks optimal gcp and aws setups. I believe this is enough to get everything working end-to-end.
airbytehq · Feb 2, 2023 · 8610fd2 · 8610fd2
1 parent bb7af73
commit 8610fd2
Show file tree

Hide file tree

Showing 3 changed files with 81 additions and 4 deletions.
diff --git a/charts/airbyte-server/templates/deployment.yaml b/charts/airbyte-server/templates/deployment.yaml
@@ -175,6 +175,11 @@ spec:
             configMapKeyRef:
               name: {{ .Release.Name }}-airbyte-env
               key: S3_MINIO_ENDPOINT
+        - name: WORKER_STATE_STORAGE_TYPE
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: WORKER_STATE_STORAGE_TYPE
         - name: STATE_STORAGE_MINIO_BUCKET_NAME
           valueFrom:
             configMapKeyRef:
@@ -190,11 +195,47 @@ spec:
             secretKeyRef:
               name: {{ .Release.Name }}-airbyte-secrets
               key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
+        {{- if eq .Values.global.state.storage.type "MINIO" }}
         - name: STATE_STORAGE_MINIO_ENDPOINT
           valueFrom:
             configMapKeyRef:
               name: {{ .Release.Name }}-airbyte-env
               key: STATE_STORAGE_MINIO_ENDPOINT
+        {{- end }}
+        - name: STATE_STORAGE_S3_BUCKET_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: S3_LOG_BUCKET
+        - name: STATE_STORAGE_S3_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-airbyte-secrets
+              key: AWS_ACCESS_KEY_ID
+        - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-airbyte-secrets
+              key: AWS_SECRET_ACCESS_KEY
+        {{- if eq .Values.global.state.storage.type "S3" }}
+        - name: STATE_STORAGE_S3_REGION
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: S3_LOG_BUCKET_REGION
+        {{- end }}
+        - name: STATE_STORAGE_GCS_APPLICATION_CREDENTIALS
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: GOOGLE_APPLICATION_CREDENTIALS
+        {{- if eq .Values.global.state.storage.type "GCS" }}
+        - name: STATE_STORAGE_GCS_BUCKET_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: GCS_LOG_BUCKET
+        {{- end }}
         - name: S3_PATH_STYLE_ACCESS
           valueFrom:
             configMapKeyRef:

diff --git a/charts/airbyte-worker/templates/deployment.yaml b/charts/airbyte-worker/templates/deployment.yaml
@@ -250,11 +250,47 @@ spec:
             secretKeyRef:
               name: {{ .Release.Name }}-airbyte-secrets
               key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
+        {{- if eq .Values.global.state.storage.type "MINIO" }}
         - name: STATE_STORAGE_MINIO_ENDPOINT
           valueFrom:
             configMapKeyRef:
               name: {{ .Release.Name }}-airbyte-env
               key: STATE_STORAGE_MINIO_ENDPOINT
+        {{- end }}
+        - name: STATE_STORAGE_S3_BUCKET_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: S3_LOG_BUCKET
+        - name: STATE_STORAGE_S3_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-airbyte-secrets
+              key: AWS_ACCESS_KEY_ID
+        - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-airbyte-secrets
+              key: AWS_SECRET_ACCESS_KEY
+        {{- if eq .Values.global.state.storage.type "S3" }}
+        - name: STATE_STORAGE_S3_REGION
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: S3_LOG_BUCKET_REGION
+        {{- end }}
+        - name: STATE_STORAGE_GCS_APPLICATION_CREDENTIALS
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: GOOGLE_APPLICATION_CREDENTIALS
+        {{- if eq .Values.global.state.storage.type "GCS" }}
+        - name: STATE_STORAGE_GCS_BUCKET_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: GCS_LOG_BUCKET
+        {{- end }}
         - name: INTERNAL_API_HOST
           valueFrom:
             configMapKeyRef:

diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock
@@ -12,10 +12,10 @@ dependencies:
   repository: https://airbytehq.github.io/helm-charts/
   version: 0.43.22
 - name: server
-  repository: https://airbytehq.github.io/helm-charts/
+  repository: file://../airbyte-server
   version: 0.43.22
 - name: worker
-  repository: https://airbytehq.github.io/helm-charts/
+  repository: file://../airbyte-worker
   version: 0.43.22
 - name: pod-sweeper
   repository: https://airbytehq.github.io/helm-charts/
@@ -29,5 +29,5 @@ dependencies:
 - name: connector-builder-server
   repository: https://airbytehq.github.io/helm-charts/
   version: 0.43.22
-digest: sha256:3f86ccd42c1eb353dd2d45237b21706c6d8a1d564dc8babb26c0fd961c56ed9d
-generated: "2023-01-25T02:03:57.511452543Z"
+digest: sha256:cd0bc2180d518dc292c8d02e62e87f2150e9b6c81d48424cb7c216f0569b8d0c
+generated: "2023-02-01T23:26:15.204239-05:00"