From a0b7dd02895a352998e759fe8edf58716a6d13de Mon Sep 17 00:00:00 2001 From: VitaliiMaltsev <39538064+VitaliiMaltsev@users.noreply.github.com> Date: Thu, 7 Apr 2022 13:08:57 +0300 Subject: [PATCH] :bug: Destination Redshift and Postgres handle custom namespace with '-' (#11729) * fix for jdk 17 * add CHANGELOG * JdbcBufferedConsumerFactory resolve namespace with dash * add CHANGELOG * bump versions * bump postgres version * update destination-jdbc version Co-authored-by: vmaltsev --- .../resources/seed/destination_definitions.yaml | 4 ++-- .../src/main/resources/seed/destination_specs.yaml | 4 ++-- .../connectors/destination-jdbc/Dockerfile | 2 +- .../jdbc/JdbcBufferedConsumerFactory.java | 14 +++++++------- .../connectors/destination-postgres/Dockerfile | 2 +- .../connectors/destination-redshift/Dockerfile | 2 +- docs/integrations/destinations/postgres.md | 1 + docs/integrations/destinations/redshift.md | 1 + 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index adf35aae9727..56adc22b07ef 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -167,7 +167,7 @@ - name: Postgres destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503 dockerRepository: airbyte/destination-postgres - dockerImageTag: 0.3.15 + dockerImageTag: 0.3.17 documentationUrl: https://docs.airbyte.io/integrations/destinations/postgres icon: postgresql.svg - name: Pulsar @@ -191,7 +191,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.3.28 + dockerImageTag: 0.3.29 documentationUrl: https://docs.airbyte.io/integrations/destinations/redshift icon: redshift.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index ddb3cdd99a6e..6260442e2de8 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2985,7 +2985,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-postgres:0.3.15" +- dockerImage: "airbyte/destination-postgres:0.3.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres" connectionSpecification: @@ -3400,7 +3400,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.3.28" +- dockerImage: "airbyte/destination-redshift:0.3.29" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/redshift" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-jdbc/Dockerfile b/airbyte-integrations/connectors/destination-jdbc/Dockerfile index e7f3ae1e78a0..c24c1cbc0033 100644 --- a/airbyte-integrations/connectors/destination-jdbc/Dockerfile +++ b/airbyte-integrations/connectors/destination-jdbc/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-jdbc COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.11 +LABEL io.airbyte.version=0.3.12 LABEL io.airbyte.name=airbyte/destination-jdbc diff --git a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java index 937750b9f301..5d8c05704c82 100644 --- a/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java @@ -86,7 +86,7 @@ private static Function toWriteConfig( final String defaultSchemaName = schemaRequired ? namingResolver.getIdentifier(config.get("schema").asText()) : namingResolver.getIdentifier(config.get("database").asText()); - final String outputSchema = getOutputSchema(abStream, defaultSchemaName); + final String outputSchema = getOutputSchema(abStream, defaultSchemaName, namingResolver); final String streamName = abStream.getName(); final String tableName = namingResolver.getRawTableName(streamName); @@ -107,12 +107,12 @@ private static Function toWriteConfig( * The logic here matches the logic in the catalog_process.py for Normalization. Any modifications * need to be reflected there and vice versa. */ - private static String getOutputSchema(final AirbyteStream stream, final String defaultDestSchema) { - final String sourceSchema = stream.getNamespace(); - if (sourceSchema != null) { - return sourceSchema; - } - return defaultDestSchema; + private static String getOutputSchema(final AirbyteStream stream, + final String defaultDestSchema, + final NamingConventionTransformer namingResolver) { + return stream.getNamespace() != null + ? namingResolver.getNamespace(stream.getNamespace()) + : namingResolver.getNamespace(defaultDestSchema); } private static OnStartFunction onStartFunction(final JdbcDatabase database, diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index 4bc95fceba40..e729da5676ad 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.15 +LABEL io.airbyte.version=0.3.17 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index f99699e788fa..d8895ae7c11b 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.28 +LABEL io.airbyte.version=0.3.29 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index 3c10867e65d8..438d7baf612e 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -84,6 +84,7 @@ Therefore, Airbyte Postgres destination will create tables and schemas using the | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.3.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | | 0.3.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.3.14 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.3.13 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 252a36bcea06..77efd0776be1 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -124,6 +124,7 @@ All Redshift connections are encrypted using SSL | Version | Date | Pull Request | Subject | |:--------| :-------- | :----- | :------ | +| 0.3.29 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | | | 0.3.28 | 2022-03-18 | [\#11254](https://github.com/airbytehq/airbyte/pull/11254) | Fixed missing records during S3 staging | | 0.3.27 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.3.25 | 2022-02-14 | [#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. |