Kustomize: Set up S3 state config variables (#22537)

Kustomize charts are missing this today. OSS users are running into this and complaining. Application logic fixed in #22191. This PR fixes the Kustomize deployments.
airbytehq · Feb 8, 2023 · 4180b57 · 4180b57
1 parent ddcdcbb
commit 4180b57
Show file tree

Hide file tree

Showing 8 changed files with 55 additions and 0 deletions.
diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env
@@ -50,6 +50,9 @@ GCS_LOG_BUCKET=
 STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs
 STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 
+STATE_STORAGE_S3_REGION=
+STATE_STORAGE_S3_BUCKET_NAME=
+
 # Docker Resource Limits
 JOB_MAIN_CONTAINER_CPU_REQUEST=
 JOB_MAIN_CONTAINER_CPU_LIMIT=

diff --git a/kube/overlays/dev/.secrets b/kube/overlays/dev/.secrets
@@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123
 GOOGLE_APPLICATION_CREDENTIALS=
 STATE_STORAGE_MINIO_ACCESS_KEY=minio
 STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123
+STATE_STORAGE_S3_ACCESS_KEY=
+STATE_STORAGE_S3_SECRET_ACCESS_KEY=
diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env
@@ -50,6 +50,9 @@ GCS_LOG_BUCKET=
 STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs
 STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 
+STATE_STORAGE_S3_REGION=
+STATE_STORAGE_S3_BUCKET_NAME=
+
 # Docker Resource Limits
 JOB_MAIN_CONTAINER_CPU_REQUEST=
 JOB_MAIN_CONTAINER_CPU_LIMIT=

diff --git a/kube/overlays/stable-with-resource-limits/.secrets b/kube/overlays/stable-with-resource-limits/.secrets
@@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123
 GOOGLE_APPLICATION_CREDENTIALS=
 STATE_STORAGE_MINIO_ACCESS_KEY=minio
 STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123
+STATE_STORAGE_S3_ACCESS_KEY=
+STATE_STORAGE_S3_SECRET_ACCESS_KEY=
diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env
@@ -50,6 +50,9 @@ GCS_LOG_BUCKET=
 STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs
 STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000
 
+STATE_STORAGE_S3_REGION=
+STATE_STORAGE_S3_BUCKET_NAME=
+
 # Docker Resource Limits
 JOB_MAIN_CONTAINER_CPU_REQUEST=
 JOB_MAIN_CONTAINER_CPU_LIMIT=

diff --git a/kube/overlays/stable/.secrets b/kube/overlays/stable/.secrets
@@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123
 GOOGLE_APPLICATION_CREDENTIALS=
 STATE_STORAGE_MINIO_ACCESS_KEY=minio
 STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123
+STATE_STORAGE_S3_ACCESS_KEY=
+STATE_STORAGE_S3_SECRET_ACCESS_KEY=
diff --git a/kube/resources/server.yaml b/kube/resources/server.yaml
@@ -188,6 +188,26 @@ spec:
                 secretKeyRef:
                   name: airbyte-secrets
                   key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
+            - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: airbyte-secrets
+                  key: STATE_STORAGE_S3_SECRET_ACCESS_KEY
+            - name: STATE_STORAGE_S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: airbyte-secrets
+                  key: STATE_STORAGE_S3_ACCESS_KEY
+            - name: STATE_STORAGE_S3_REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: STATE_STORAGE_S3_REGION
+            - name: STATE_STORAGE_S3_BUCKET_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: STATE_STORAGE_S3_BUCKET_NAME
           ports:
             - containerPort: 8001
           volumeMounts:

diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml
@@ -220,6 +220,26 @@ spec:
                 secretKeyRef:
                   name: airbyte-secrets
                   key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
+            - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: airbyte-secrets
+                  key: STATE_STORAGE_S3_SECRET_ACCESS_KEY
+            - name: STATE_STORAGE_S3_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: airbyte-secrets
+                  key: STATE_STORAGE_S3_ACCESS_KEY
+            - name: STATE_STORAGE_S3_REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: STATE_STORAGE_S3_REGION
+            - name: STATE_STORAGE_S3_BUCKET_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: airbyte-env
+                  key: STATE_STORAGE_S3_BUCKET_NAME
             - name: CONTAINER_ORCHESTRATOR_ENABLED
               valueFrom:
                 configMapKeyRef: