From 4180b577b8ded48b177f2421673e1230c964f6b4 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Tue, 7 Feb 2023 17:54:00 -0800 Subject: [PATCH] Kustomize: Set up S3 state config variables (#22537) Kustomize charts are missing this today. OSS users are running into this and complaining. Application logic fixed in #22191. This PR fixes the Kustomize deployments. --- kube/overlays/dev/.env | 3 +++ kube/overlays/dev/.secrets | 2 ++ .../overlays/stable-with-resource-limits/.env | 3 +++ .../stable-with-resource-limits/.secrets | 2 ++ kube/overlays/stable/.env | 3 +++ kube/overlays/stable/.secrets | 2 ++ kube/resources/server.yaml | 20 +++++++++++++++++++ kube/resources/worker.yaml | 20 +++++++++++++++++++ 8 files changed, 55 insertions(+) diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index 1bc8b49dfe339..e73c127b89ef6 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -50,6 +50,9 @@ GCS_LOG_BUCKET= STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 +STATE_STORAGE_S3_REGION= +STATE_STORAGE_S3_BUCKET_NAME= + # Docker Resource Limits JOB_MAIN_CONTAINER_CPU_REQUEST= JOB_MAIN_CONTAINER_CPU_LIMIT= diff --git a/kube/overlays/dev/.secrets b/kube/overlays/dev/.secrets index 67e8f4aae9770..c03579ec9b59e 100644 --- a/kube/overlays/dev/.secrets +++ b/kube/overlays/dev/.secrets @@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123 GOOGLE_APPLICATION_CREDENTIALS= STATE_STORAGE_MINIO_ACCESS_KEY=minio STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123 +STATE_STORAGE_S3_ACCESS_KEY= +STATE_STORAGE_S3_SECRET_ACCESS_KEY= diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 70a935242db69..fb9d8d4c07adf 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -50,6 +50,9 @@ GCS_LOG_BUCKET= STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 +STATE_STORAGE_S3_REGION= +STATE_STORAGE_S3_BUCKET_NAME= + # Docker Resource Limits JOB_MAIN_CONTAINER_CPU_REQUEST= JOB_MAIN_CONTAINER_CPU_LIMIT= diff --git a/kube/overlays/stable-with-resource-limits/.secrets b/kube/overlays/stable-with-resource-limits/.secrets index 67e8f4aae9770..c03579ec9b59e 100644 --- a/kube/overlays/stable-with-resource-limits/.secrets +++ b/kube/overlays/stable-with-resource-limits/.secrets @@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123 GOOGLE_APPLICATION_CREDENTIALS= STATE_STORAGE_MINIO_ACCESS_KEY=minio STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123 +STATE_STORAGE_S3_ACCESS_KEY= +STATE_STORAGE_S3_SECRET_ACCESS_KEY= diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 4ad0a31b51f77..bd3fe335fce3c 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -50,6 +50,9 @@ GCS_LOG_BUCKET= STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 +STATE_STORAGE_S3_REGION= +STATE_STORAGE_S3_BUCKET_NAME= + # Docker Resource Limits JOB_MAIN_CONTAINER_CPU_REQUEST= JOB_MAIN_CONTAINER_CPU_LIMIT= diff --git a/kube/overlays/stable/.secrets b/kube/overlays/stable/.secrets index 67e8f4aae9770..c03579ec9b59e 100644 --- a/kube/overlays/stable/.secrets +++ b/kube/overlays/stable/.secrets @@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123 GOOGLE_APPLICATION_CREDENTIALS= STATE_STORAGE_MINIO_ACCESS_KEY=minio STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123 +STATE_STORAGE_S3_ACCESS_KEY= +STATE_STORAGE_S3_SECRET_ACCESS_KEY= diff --git a/kube/resources/server.yaml b/kube/resources/server.yaml index 85b956e1ae9c2..057eaab9fe26f 100644 --- a/kube/resources/server.yaml +++ b/kube/resources/server.yaml @@ -188,6 +188,26 @@ spec: secretKeyRef: name: airbyte-secrets key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY + - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: STATE_STORAGE_S3_SECRET_ACCESS_KEY + - name: STATE_STORAGE_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: STATE_STORAGE_S3_ACCESS_KEY + - name: STATE_STORAGE_S3_REGION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: STATE_STORAGE_S3_REGION + - name: STATE_STORAGE_S3_BUCKET_NAME + valueFrom: + configMapKeyRef: + name: airbyte-env + key: STATE_STORAGE_S3_BUCKET_NAME ports: - containerPort: 8001 volumeMounts: diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index 38cf19fe268df..4fb2605845f6f 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -220,6 +220,26 @@ spec: secretKeyRef: name: airbyte-secrets key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY + - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: STATE_STORAGE_S3_SECRET_ACCESS_KEY + - name: STATE_STORAGE_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: STATE_STORAGE_S3_ACCESS_KEY + - name: STATE_STORAGE_S3_REGION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: STATE_STORAGE_S3_REGION + - name: STATE_STORAGE_S3_BUCKET_NAME + valueFrom: + configMapKeyRef: + name: airbyte-env + key: STATE_STORAGE_S3_BUCKET_NAME - name: CONTAINER_ORCHESTRATOR_ENABLED valueFrom: configMapKeyRef: