Skip to content

Commit

Permalink
Fixes for state storage env vars (#22191)
Browse files Browse the repository at this point in the history
The current env var checks for state storage do not properly exclude empty env vars, as is currently implemented for log storage. This means that it is impossible to configure S3 state storage using both kustomize and helm, as those define env vars for the minio state storage (even when they are empty)
S3 state storage had two different env vars used for region

* check for empty env vars
* use consistent env var for STATE_STORAGE_S3_REGION

---------

Co-authored-by: Davin Chia <davinchia@gmail.com>
  • Loading branch information
adam-bloom and davinchia authored Feb 7, 2023
1 parent 48a5554 commit 34ecd83
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -281,17 +281,17 @@ private Optional<CloudStorageConfigs> getLogConfiguration() {
}

private Optional<CloudStorageConfigs> getStateStorageConfiguration() {
if (getEnv(STATE_STORAGE_GCS_BUCKET_NAME) != null) {
if (getEnv(STATE_STORAGE_GCS_BUCKET_NAME) != null && !getEnv(STATE_STORAGE_GCS_BUCKET_NAME).isBlank()) {
return Optional.of(CloudStorageConfigs.gcs(new GcsConfig(
getEnvOrDefault(STATE_STORAGE_GCS_BUCKET_NAME, ""),
getEnvOrDefault(STATE_STORAGE_GCS_APPLICATION_CREDENTIALS, ""))));
} else if (getEnv(STATE_STORAGE_MINIO_ENDPOINT) != null) {
} else if (getEnv(STATE_STORAGE_MINIO_ENDPOINT) != null && !getEnv(STATE_STORAGE_MINIO_ENDPOINT).isBlank()) {
return Optional.of(CloudStorageConfigs.minio(new MinioConfig(
getEnvOrDefault(STATE_STORAGE_MINIO_BUCKET_NAME, ""),
getEnvOrDefault(STATE_STORAGE_MINIO_ACCESS_KEY, ""),
getEnvOrDefault(STATE_STORAGE_MINIO_SECRET_ACCESS_KEY, ""),
getEnvOrDefault(STATE_STORAGE_MINIO_ENDPOINT, ""))));
} else if (getEnv(STATE_STORAGE_S3_REGION) != null) {
} else if (getEnv(STATE_STORAGE_S3_REGION) != null && !getEnv(STATE_STORAGE_S3_REGION).isBlank()) {
return Optional.of(CloudStorageConfigs.s3(new S3Config(
getEnvOrDefault(STATE_STORAGE_S3_BUCKET_NAME, ""),
getEnvOrDefault(STATE_STORAGE_S3_ACCESS_KEY, ""),
Expand Down
8 changes: 4 additions & 4 deletions airbyte-server/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ airbyte:
s3:
access-key: ${STATE_STORAGE_S3_ACCESS_KEY:}
bucket: ${STATE_STORAGE_S3_BUCKET_NAME:}
region: ${STATE_STORAGE_S3_BUCKET_REGION:}
region: ${STATE_STORAGE_S3_REGION:}
secret-access-key: ${STATE_STORAGE_S3_SECRET_ACCESS_KEY:}
deployment-mode: ${DEPLOYMENT_MODE:OSS}
flyway:
Expand Down Expand Up @@ -101,7 +101,7 @@ datasources:
maximum-pool-size: 10
minimum-idle: 0
idle-timeout: 600000
initialization-fail-timeout: -1 # Disable fail fast checking to avoid issues due to other pods not being started in time
initialization-fail-timeout: -1 # Disable fail fast checking to avoid issues due to other pods not being started in time
url: ${DATABASE_URL}
driverClassName: org.postgresql.Driver
username: ${DATABASE_USER}
Expand Down Expand Up @@ -139,11 +139,11 @@ flyway:
config:
enabled: false
locations:
- 'classpath:io/airbyte/db/instance/configs/migrations'
- "classpath:io/airbyte/db/instance/configs/migrations"
jobs:
enabled: false
locations:
- 'classpath:io/airbyte/db/instance/jobs/migrations'
- "classpath:io/airbyte/db/instance/jobs/migrations"

jooq:
datasources:
Expand Down
2 changes: 1 addition & 1 deletion airbyte-workers/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ airbyte:
s3:
access-key: ${STATE_STORAGE_S3_ACCESS_KEY:}
bucket: ${STATE_STORAGE_S3_BUCKET_NAME:}
region: ${STATE_STORAGE_S3_BUCKET_REGION:}
region: ${STATE_STORAGE_S3_REGION:}
secret-access-key: ${STATE_STORAGE_S3_SECRET_ACCESS_KEY:}
connector:
specific-resource-defaults-enabled: ${CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED:false}
Expand Down

0 comments on commit 34ecd83

Please sign in to comment.