From cb820cc96d9a0d2ab4587af9ce3abccc7fe9c43b Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Tue, 7 Feb 2023 15:53:24 -0800 Subject: [PATCH 1/3] Set up S3 state config variables. --- kube/overlays/dev/.env | 3 +++ kube/overlays/dev/.secrets | 2 ++ kube/resources/server.yaml | 20 ++++++++++++++++++++ kube/resources/worker.yaml | 20 ++++++++++++++++++++ 4 files changed, 45 insertions(+) diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index 1bc8b49dfe33..e73c127b89ef 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -50,6 +50,9 @@ GCS_LOG_BUCKET= STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 +STATE_STORAGE_S3_REGION= +STATE_STORAGE_S3_BUCKET_NAME= + # Docker Resource Limits JOB_MAIN_CONTAINER_CPU_REQUEST= JOB_MAIN_CONTAINER_CPU_LIMIT= diff --git a/kube/overlays/dev/.secrets b/kube/overlays/dev/.secrets index 67e8f4aae977..c03579ec9b59 100644 --- a/kube/overlays/dev/.secrets +++ b/kube/overlays/dev/.secrets @@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123 GOOGLE_APPLICATION_CREDENTIALS= STATE_STORAGE_MINIO_ACCESS_KEY=minio STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123 +STATE_STORAGE_S3_ACCESS_KEY= +STATE_STORAGE_S3_SECRET_ACCESS_KEY= diff --git a/kube/resources/server.yaml b/kube/resources/server.yaml index 85b956e1ae9c..057eaab9fe26 100644 --- a/kube/resources/server.yaml +++ b/kube/resources/server.yaml @@ -188,6 +188,26 @@ spec: secretKeyRef: name: airbyte-secrets key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY + - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: STATE_STORAGE_S3_SECRET_ACCESS_KEY + - name: STATE_STORAGE_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: STATE_STORAGE_S3_ACCESS_KEY + - name: STATE_STORAGE_S3_REGION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: STATE_STORAGE_S3_REGION + - name: STATE_STORAGE_S3_BUCKET_NAME + valueFrom: + configMapKeyRef: + name: airbyte-env + key: STATE_STORAGE_S3_BUCKET_NAME ports: - containerPort: 8001 volumeMounts: diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index d8f7be704011..3c426ded1708 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -220,6 +220,26 @@ spec: secretKeyRef: name: airbyte-secrets key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY + - name: STATE_STORAGE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: STATE_STORAGE_S3_SECRET_ACCESS_KEY + - name: STATE_STORAGE_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: airbyte-secrets + key: STATE_STORAGE_S3_ACCESS_KEY + - name: STATE_STORAGE_S3_REGION + valueFrom: + configMapKeyRef: + name: airbyte-env + key: STATE_STORAGE_S3_REGION + - name: STATE_STORAGE_S3_BUCKET_NAME + valueFrom: + configMapKeyRef: + name: airbyte-env + key: STATE_STORAGE_S3_BUCKET_NAME - name: CONTAINER_ORCHESTRATOR_ENABLED valueFrom: configMapKeyRef: From 383342b15c5344fba79ad11b17bc70ad82452d0b Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Tue, 7 Feb 2023 15:58:26 -0800 Subject: [PATCH 2/3] Add this for stable and stable with resource limits kustomise. --- kube/overlays/stable-with-resource-limits/.env | 3 +++ kube/overlays/stable-with-resource-limits/.secrets | 2 ++ kube/overlays/stable/.env | 3 +++ kube/overlays/stable/.secrets | 2 ++ 4 files changed, 10 insertions(+) diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 70a935242db6..fb9d8d4c07ad 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -50,6 +50,9 @@ GCS_LOG_BUCKET= STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 +STATE_STORAGE_S3_REGION= +STATE_STORAGE_S3_BUCKET_NAME= + # Docker Resource Limits JOB_MAIN_CONTAINER_CPU_REQUEST= JOB_MAIN_CONTAINER_CPU_LIMIT= diff --git a/kube/overlays/stable-with-resource-limits/.secrets b/kube/overlays/stable-with-resource-limits/.secrets index 67e8f4aae977..c03579ec9b59 100644 --- a/kube/overlays/stable-with-resource-limits/.secrets +++ b/kube/overlays/stable-with-resource-limits/.secrets @@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123 GOOGLE_APPLICATION_CREDENTIALS= STATE_STORAGE_MINIO_ACCESS_KEY=minio STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123 +STATE_STORAGE_S3_ACCESS_KEY= +STATE_STORAGE_S3_SECRET_ACCESS_KEY= diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 4ad0a31b51f7..bd3fe335fce3 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -50,6 +50,9 @@ GCS_LOG_BUCKET= STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 +STATE_STORAGE_S3_REGION= +STATE_STORAGE_S3_BUCKET_NAME= + # Docker Resource Limits JOB_MAIN_CONTAINER_CPU_REQUEST= JOB_MAIN_CONTAINER_CPU_LIMIT= diff --git a/kube/overlays/stable/.secrets b/kube/overlays/stable/.secrets index 67e8f4aae977..c03579ec9b59 100644 --- a/kube/overlays/stable/.secrets +++ b/kube/overlays/stable/.secrets @@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123 GOOGLE_APPLICATION_CREDENTIALS= STATE_STORAGE_MINIO_ACCESS_KEY=minio STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123 +STATE_STORAGE_S3_ACCESS_KEY= +STATE_STORAGE_S3_SECRET_ACCESS_KEY= From 51e0b04f44930888d46e644676d5a005fcdf25f6 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Tue, 7 Feb 2023 16:42:04 -0800 Subject: [PATCH 3/3] Also add this to the dev integration test scripts. --- kube/overlays/dev-integration-test/.env | 3 +++ kube/overlays/dev-integration-test/.secrets | 2 ++ 2 files changed, 5 insertions(+) diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env index aad4911e7fda..5d5274a3faa2 100644 --- a/kube/overlays/dev-integration-test/.env +++ b/kube/overlays/dev-integration-test/.env @@ -48,6 +48,9 @@ GCS_LOG_BUCKET= STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-dev-logs STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 +STATE_STORAGE_S3_REGION= +STATE_STORAGE_S3_BUCKET_NAME= + # Docker Resource Limits JOB_MAIN_CONTAINER_CPU_REQUEST= JOB_MAIN_CONTAINER_CPU_LIMIT= diff --git a/kube/overlays/dev-integration-test/.secrets b/kube/overlays/dev-integration-test/.secrets index 67e8f4aae977..c03579ec9b59 100644 --- a/kube/overlays/dev-integration-test/.secrets +++ b/kube/overlays/dev-integration-test/.secrets @@ -5,3 +5,5 @@ AWS_SECRET_ACCESS_KEY=minio123 GOOGLE_APPLICATION_CREDENTIALS= STATE_STORAGE_MINIO_ACCESS_KEY=minio STATE_STORAGE_MINIO_SECRET_ACCESS_KEY=minio123 +STATE_STORAGE_S3_ACCESS_KEY= +STATE_STORAGE_S3_SECRET_ACCESS_KEY=