From 38fdf37ef5c5ddfdb34ecadb229a217f97f7f14d Mon Sep 17 00:00:00 2001 From: ryankfu Date: Wed, 25 Jan 2023 14:14:14 -0800 Subject: [PATCH 1/4] Fixed mismatch spec variable and updates default value --- .../destination/redshift/RedshiftStagingS3Destination.java | 2 +- .../destination-redshift/src/main/resources/spec.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index f185235e4bf8f..6fc5d60710e80 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -170,7 +170,7 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, */ @VisibleForTesting public int getNumberOfFileBuffers(final JsonNode config) { - int numOfFileBuffers = 1; + int numOfFileBuffers = FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER; if (config.has(FileBuffer.FILE_BUFFER_COUNT_KEY)) { numOfFileBuffers = Math.min(config.get(FileBuffer.FILE_BUFFER_COUNT_KEY).asInt(), FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER); } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json index 0e62601e81518..4a3180e33c5b8 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json @@ -207,7 +207,7 @@ } ] }, - "file_buffer_size": { + "file_buffer_count": { "title": "File Buffer Count", "type": "integer", "minimum": 15, From 28b2c69550328728d7f4ab08b7e6498672b5487a Mon Sep 17 00:00:00 2001 From: ryankfu Date: Wed, 25 Jan 2023 14:16:32 -0800 Subject: [PATCH 2/4] Updates spec default values --- .../destination-redshift/src/main/resources/spec.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json index 4a3180e33c5b8..53c686b04f914 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json @@ -210,11 +210,11 @@ "file_buffer_count": { "title": "File Buffer Count", "type": "integer", - "minimum": 15, + "minimum": 10, "maximum": 50, - "default": 15, + "default": 10, "description": "Number of file buffers allocated for writing data. Increasing this number is beneficial for connections using Change Data Capture (CDC) and up to the number of streams within a connection. Increasing the number of file buffers past the maximum number of streams has deteriorating effects", - "examples": ["15"] + "examples": ["10"] } } } From c9fa76598df1bb831ad46344ca63b4fef267fc8f Mon Sep 17 00:00:00 2001 From: ryankfu Date: Thu, 26 Jan 2023 15:09:19 -0800 Subject: [PATCH 3/4] Bumps connector version and documentation --- .../connectors/destination-redshift/Dockerfile | 2 +- docs/integrations/destinations/redshift.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index 5ca7be2d9b983..6a56861b7fa51 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.55 +LABEL io.airbyte.version=0.3.56 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 0dbaf74cf3c82..aa20acc9d2760 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -141,10 +141,11 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.56 | 2023-01-26 | [\#21890](https://github.com/airbytehq/airbyte/pull/21890) | Fixed configurable parameter for number of file buffers | | 0.3.55 | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging | | 0.3.54 | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | | 0.3.53 | 2023-01-03 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Flatten JSON arrays to fix maximum size check for SUPER field | -| 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers | +| 0.3.52 | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead) | | 0.3.51 | 2022-10-26 | [\#18434](https://github.com/airbytehq/airbyte/pull/18434) | Fix empty S3 bucket path handling | | 0.3.50 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | | 0.3.49 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | From bc4c907bf0b6ff9e6b949653f95af28c20300689 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 27 Jan 2023 19:20:44 +0000 Subject: [PATCH 4/4] auto-bump connector version --- .../main/resources/seed/destination_definitions.yaml | 2 +- .../src/main/resources/seed/destination_specs.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 98f3542a3a1fe..01bc2e8eb9dc7 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -290,7 +290,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.55 + dockerImageTag: 0.3.56 documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift icon: redshift.svg normalizationConfig: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 1ceff75d32cac..a383f60365bd2 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -5123,7 +5123,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.55" +- dockerImage: "airbyte/destination-redshift:0.3.56" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/redshift" connectionSpecification: @@ -5337,19 +5337,19 @@ \ or 256 bits. Leave blank to have Airbyte generate an ephemeral\ \ key for each sync." airbyte_secret: true - file_buffer_size: + file_buffer_count: title: "File Buffer Count" type: "integer" - minimum: 15 + minimum: 10 maximum: 50 - default: 15 + default: 10 description: "Number of file buffers allocated for writing data. Increasing\ \ this number is beneficial for connections using Change Data Capture\ \ (CDC) and up to the number of streams within a connection. Increasing\ \ the number of file buffers past the maximum number of streams\ \ has deteriorating effects" examples: - - "15" + - "10" supportsIncremental: true supportsNormalization: true supportsDBT: true