From 5176a322755380eab5ab3d5c65fd1a460bf74448 Mon Sep 17 00:00:00 2001 From: Eugene Date: Sat, 18 Mar 2023 10:52:37 +0200 Subject: [PATCH] S3-Parquet: added handler to process null values in arrays (#23788) * [22807] Parquet - added handler to null values in arrays --- .../main/resources/seed/destination_definitions.yaml | 12 ++++++------ .../src/main/resources/seed/destination_specs.yaml | 12 ++++++------ .../s3/parquet/ParquetSerializedBuffer.java | 7 ++++++- .../destination/s3/parquet/S3ParquetWriter.java | 4 ++++ .../s3/parquet/ParquetSerializedBufferTest.java | 12 +++++++++--- .../destination-bigquery-denormalized/Dockerfile | 2 +- .../connectors/destination-bigquery/Dockerfile | 2 +- .../connectors/destination-gcs/Dockerfile | 2 +- .../connectors/destination-redshift/Dockerfile | 2 +- .../connectors/destination-s3/Dockerfile | 2 +- .../connectors/destination-snowflake/Dockerfile | 2 +- connectors.md | 12 ++++++------ .../destinations/bigquery-denormalized.md | 4 +++- docs/integrations/destinations/bigquery.md | 1 + docs/integrations/destinations/gcs.md | 1 + docs/integrations/destinations/redshift.md | 1 + docs/integrations/destinations/s3.md | 1 + docs/integrations/destinations/snowflake.md | 1 + 18 files changed, 51 insertions(+), 29 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index f79bd60a18d3..d2b7c412b91d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -40,7 +40,7 @@ - name: BigQuery destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 dockerRepository: airbyte/destination-bigquery - dockerImageTag: 1.2.16 + dockerImageTag: 1.2.17 documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery icon: bigquery.svg normalizationConfig: @@ -58,7 +58,7 @@ - name: BigQuery (denormalized typed struct) destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-bigquery-denormalized - dockerImageTag: 1.2.16 + dockerImageTag: 1.2.17 documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery icon: bigquery.svg resourceRequirements: @@ -145,7 +145,7 @@ - name: Google Cloud Storage (GCS) destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a dockerRepository: airbyte/destination-gcs - dockerImageTag: 0.2.15 + dockerImageTag: 0.2.16 documentationUrl: https://docs.airbyte.com/integrations/destinations/gcs icon: googlecloudstorage.svg resourceRequirements: @@ -290,7 +290,7 @@ - name: Redshift destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerRepository: airbyte/destination-redshift - dockerImageTag: 0.4.2 + dockerImageTag: 0.4.3 documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift icon: redshift.svg normalizationConfig: @@ -321,7 +321,7 @@ - name: S3 destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.3.21 + dockerImageTag: 0.3.22 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3 icon: s3.svg resourceRequirements: @@ -348,7 +348,7 @@ - name: Snowflake destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba dockerRepository: airbyte/destination-snowflake - dockerImageTag: 0.4.53 + dockerImageTag: 0.4.54 documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake icon: snowflake.svg normalizationConfig: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 986d9e6e9ac9..f7d45ee44be3 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -630,7 +630,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-bigquery:1.2.16" +- dockerImage: "airbyte/destination-bigquery:1.2.17" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery" connectionSpecification: @@ -840,7 +840,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.16" +- dockerImage: "airbyte/destination-bigquery-denormalized:1.2.17" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/bigquery" connectionSpecification: @@ -2399,7 +2399,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-gcs:0.2.15" +- dockerImage: "airbyte/destination-gcs:0.2.16" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/gcs" connectionSpecification: @@ -5055,7 +5055,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-redshift:0.4.2" +- dockerImage: "airbyte/destination-redshift:0.4.3" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/redshift" connectionSpecification: @@ -5525,7 +5525,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-s3:0.3.21" +- dockerImage: "airbyte/destination-s3:0.3.22" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/s3" connectionSpecification: @@ -6151,7 +6151,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-snowflake:0.4.53" +- dockerImage: "airbyte/destination-snowflake:0.4.54" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/snowflake" connectionSpecification: diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java index 29f96e533520..c8c0979f09f7 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.destination.s3.parquet; +import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE; + import io.airbyte.commons.functional.CheckedBiFunction; import io.airbyte.integrations.destination.record_buffer.FileBuffer; import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; @@ -70,8 +72,11 @@ public ParquetSerializedBuffer(final S3DestinationConfig config, Files.deleteIfExists(bufferFile); avroRecordFactory = new AvroRecordFactory(schema, AvroConstants.JSON_CONVERTER); final S3ParquetFormatConfig formatConfig = (S3ParquetFormatConfig) config.getFormatConfig(); + Configuration avroConfig = new Configuration(); + avroConfig.setBoolean(WRITE_OLD_LIST_STRUCTURE, false); parquetWriter = AvroParquetWriter.builder(HadoopOutputFile - .fromPath(new org.apache.hadoop.fs.Path(bufferFile.toUri()), new Configuration())) + .fromPath(new org.apache.hadoop.fs.Path(bufferFile.toUri()), avroConfig)) + .withConf(avroConfig) // yes, this should be here despite the fact we pass this config above in path .withSchema(schema) .withCompressionCodec(formatConfig.getCompressionCodec()) .withRowGroupSize(formatConfig.getBlockSize()) diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java index cb6015dffb87..420a959f73ac 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.destination.s3.parquet; +import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE; + import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.destination.s3.S3DestinationConfig; @@ -69,7 +71,9 @@ public S3ParquetWriter(final S3DestinationConfig config, final Path path = new Path(new URI(fullFilePath)); final S3ParquetFormatConfig formatConfig = (S3ParquetFormatConfig) config.getFormatConfig(); final Configuration hadoopConfig = getHadoopConfig(config); + hadoopConfig.setBoolean(WRITE_OLD_LIST_STRUCTURE, false); this.parquetWriter = AvroParquetWriter.builder(HadoopOutputFile.fromPath(path, hadoopConfig)) + .withConf(hadoopConfig) // yes, this should be here despite the fact we pass this config above in path .withSchema(schema) .withCompressionCodec(formatConfig.getCompressionCodec()) .withRowGroupSize(formatConfig.getBlockSize()) diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java index 3d11b140aeac..2f2911cb51ac 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java +++ b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java @@ -14,6 +14,7 @@ import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -26,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Stream; import org.apache.avro.generic.GenericData.Record; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -40,6 +42,7 @@ public class ParquetSerializedBufferTest { "column2", "string value", "another field", true, "nested_column", Map.of("array_column", List.of(1, 2, 3)), + "string_array_column", Stream.of("test_string", null).toList(), "datetime_with_timezone", "2022-05-12T15:35:44.192950Z")); private static final String STREAM = "stream1"; private static final AirbyteStreamNameNamespacePair streamPair = new AirbyteStreamNameNamespacePair(STREAM, null); @@ -52,6 +55,8 @@ public class ParquetSerializedBufferTest { Field.of("column2", JsonSchemaType.STRING), Field.of("another field", JsonSchemaType.BOOLEAN), Field.of("nested_column", JsonSchemaType.OBJECT), + Field.of("string_array_column", JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.STRING).build()), Field.of("datetime_with_timezone", JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE)); private static final ConfiguredAirbyteCatalog catalog = CatalogHelpers.createConfiguredAirbyteCatalog(STREAM, null, FIELDS); @@ -62,7 +67,7 @@ public void testUncompressedParquetWriter() throws Exception { "format_type", "parquet"), "s3_bucket_name", "test", "s3_bucket_region", "us-east-2"))); - runTest(195L, 215L, config, getExpectedString()); + runTest(225L, 245L, config, getExpectedString()); } @Test @@ -74,7 +79,7 @@ public void testCompressedParquetWriter() throws Exception { "s3_bucket_name", "test", "s3_bucket_region", "us-east-2"))); // TODO: Compressed parquet is the same size as uncompressed?? - runTest(195L, 215L, config, getExpectedString()); + runTest(225L, 245L, config, getExpectedString()); } private static String resolveArchitecture() { @@ -119,7 +124,7 @@ private void runLzoParquetTest() throws Exception { "compression_codec", "LZO"), "s3_bucket_name", "test", "s3_bucket_region", "us-east-2"))); - runTest(195L, 215L, config, getExpectedString()); + runTest(225L, 245L, config, getExpectedString()); } private static String getExpectedString() { @@ -127,6 +132,7 @@ private static String getExpectedString() { + "\"field1\": 10000.0, \"another_field\": true, " + "\"nested_column\": {\"_airbyte_additional_properties\": {\"array_column\": \"[1,2,3]\"}}, " + "\"column2\": \"string value\", " + + "\"string_array_column\": [\"test_string\", null], " + "\"datetime_with_timezone\": 1652369744192000, " + "\"_airbyte_additional_properties\": null}"; } diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 8cbfa9dd448c..4c283b269888 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.16 +LABEL io.airbyte.version=1.2.17 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 6e931e3ed088..903638060908 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.2.16 +LABEL io.airbyte.version=1.2.17 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile index 8e9e7d63ada2..d0481852e594 100644 --- a/airbyte-integrations/connectors/destination-gcs/Dockerfile +++ b/airbyte-integrations/connectors/destination-gcs/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.15 +LABEL io.airbyte.version=0.2.16 LABEL io.airbyte.name=airbyte/destination-gcs diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index 02b7bac2551f..343577ae5e6f 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.2 +LABEL io.airbyte.version=0.4.3 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index 5aee8fb1b55e..ac1b3b290e5a 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -40,5 +40,5 @@ RUN /bin/bash -c 'set -e && \ echo "unknown arch" ;\ fi' -LABEL io.airbyte.version=0.3.21 +LABEL io.airbyte.version=0.3.22 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index f9d092b7adef..d6741bc3d48b 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -20,5 +20,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=0.4.53 +LABEL io.airbyte.version=0.4.54 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/connectors.md b/connectors.md index d0a2e6ffc3ac..47136d648332 100644 --- a/connectors.md +++ b/connectors.md @@ -288,8 +288,8 @@ | **Apache Doris** | Apache Doris icon | Destination | airbyte/destination-doris:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/doris) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-doris) | `05c161bf-ca73-4d48-b524-d392be417002` | | **Apache Iceberg** | x | Destination | airbyte/destination-iceberg:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/iceberg) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-iceberg) | `df65a8f3-9908-451b-aa9b-445462803560` | | **Azure Blob Storage** | Azure Blob Storage icon | Destination | airbyte/destination-azure-blob-storage:0.2.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/azureblobstorage) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-azure-blob-storage) | `b4c5d105-31fd-4817-96b6-cb923bfc04cb` | -| **BigQuery** | BigQuery icon | Destination | airbyte/destination-bigquery:1.2.16 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/bigquery) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-bigquery) | `22f6c74f-5699-40ff-833c-4a879ea40133` | -| **BigQuery (denormalized typed struct)** | BigQuery (denormalized typed struct) icon | Destination | airbyte/destination-bigquery-denormalized:1.2.16 | beta | [link](https://docs.airbyte.com/integrations/destinations/bigquery) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-bigquery-denormalized) | `079d5540-f236-4294-ba7c-ade8fd918496` | +| **BigQuery** | BigQuery icon | Destination | airbyte/destination-bigquery:1.2.17 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/bigquery) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-bigquery) | `22f6c74f-5699-40ff-833c-4a879ea40133` | +| **BigQuery (denormalized typed struct)** | BigQuery (denormalized typed struct) icon | Destination | airbyte/destination-bigquery-denormalized:1.2.17 | beta | [link](https://docs.airbyte.com/integrations/destinations/bigquery) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-bigquery-denormalized) | `079d5540-f236-4294-ba7c-ade8fd918496` | | **Cassandra** | Cassandra icon | Destination | airbyte/destination-cassandra:0.1.4 | alpha | [link](https://docs.airbyte.com/integrations/destinations/cassandra) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-cassandra) | `707456df-6f4f-4ced-b5c6-03f73bcad1c5` | | **Chargify (Keen)** | Chargify (Keen) icon | Destination | airbyte/destination-keen:0.2.4 | alpha | [link](https://docs.airbyte.com/integrations/destinations/keen) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-keen) | `81740ce8-d764-4ea7-94df-16bb41de36ae` | | **Clickhouse** | Clickhouse icon | Destination | airbyte/destination-clickhouse:0.2.2 | alpha | [link](https://docs.airbyte.com/integrations/destinations/clickhouse) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-clickhouse) | `ce0d828e-1dc4-496c-b122-2da42e637e48` | @@ -302,7 +302,7 @@ | **ElasticSearch** | ElasticSearch icon | Destination | airbyte/destination-elasticsearch:0.1.6 | alpha | [link](https://docs.airbyte.com/integrations/destinations/elasticsearch) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-elasticsearch) | `68f351a7-2745-4bef-ad7f-996b8e51bb8c` | | **Exasol** | x | Destination | airbyte/destination-exasol:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/destinations/exasol) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-exasol) | `bb6071d9-6f34-4766-bec2-d1d4ed81a653` | | **Firebolt** | Firebolt icon | Destination | airbyte/destination-firebolt:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/firebolt) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-firebolt) | `18081484-02a5-4662-8dba-b270b582f321` | -| **Google Cloud Storage (GCS)** | Google Cloud Storage (GCS) icon | Destination | airbyte/destination-gcs:0.2.15 | beta | [link](https://docs.airbyte.com/integrations/destinations/gcs) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-gcs) | `ca8f6566-e555-4b40-943a-545bf123117a` | +| **Google Cloud Storage (GCS)** | Google Cloud Storage (GCS) icon | Destination | airbyte/destination-gcs:0.2.16 | beta | [link](https://docs.airbyte.com/integrations/destinations/gcs) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-gcs) | `ca8f6566-e555-4b40-943a-545bf123117a` | | **Google Firestore** | Google Firestore icon | Destination | airbyte/destination-firestore:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/destinations/firestore) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-firestore) | `27dc7500-6d1b-40b1-8b07-e2f2aea3c9f4` | | **Google PubSub** | Google PubSub icon | Destination | airbyte/destination-pubsub:0.2.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/pubsub) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-pubsub) | `356668e2-7e34-47f3-a3b0-67a8a481b692` | | **Google Sheets** | Google Sheets icon | Destination | airbyte/destination-google-sheets:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/destinations/google-sheets) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-google-sheets) | `a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a` | @@ -323,13 +323,13 @@ | **RabbitMQ** | RabbitMQ icon | Destination | airbyte/destination-rabbitmq:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/destinations/rabbitmq) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-rabbitmq) | `e06ad785-ad6f-4647-b2e8-3027a5c59454` | | **Redis** | Redis icon | Destination | airbyte/destination-redis:0.1.4 | alpha | [link](https://docs.airbyte.com/integrations/destinations/redis) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-redis) | `d4d3fef9-e319-45c2-881a-bd02ce44cc9f` | | **Redpanda** | Redpanda icon | Destination | airbyte/destination-redpanda:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/redpanda) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-redpanda) | `825c5ee3-ed9a-4dd1-a2b6-79ed722f7b13` | -| **Redshift** | Redshift icon | Destination | airbyte/destination-redshift:0.4.2 | beta | [link](https://docs.airbyte.com/integrations/destinations/redshift) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-redshift) | `f7a7d195-377f-cf5b-70a5-be6b819019dc` | +| **Redshift** | Redshift icon | Destination | airbyte/destination-redshift:0.4.3 | beta | [link](https://docs.airbyte.com/integrations/destinations/redshift) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-redshift) | `f7a7d195-377f-cf5b-70a5-be6b819019dc` | | **Rockset** | x | Destination | airbyte/destination-rockset:0.1.4 | alpha | [link](https://docs.airbyte.com/integrations/destinations/rockset) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-rockset) | `2c9d93a7-9a17-4789-9de9-f46f0097eb70` | -| **S3** | S3 icon | Destination | airbyte/destination-s3:0.3.21 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/s3) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3) | `4816b78f-1489-44c1-9060-4b19d5fa9362` | +| **S3** | S3 icon | Destination | airbyte/destination-s3:0.3.22 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/s3) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3) | `4816b78f-1489-44c1-9060-4b19d5fa9362` | | **S3 Glue** | S3 Glue icon | Destination | airbyte/destination-s3-glue:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/destinations/s3-glue) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3-glue) | `471e5cab-8ed1-49f3-ba11-79c687784737` | | **SFTP-JSON** | SFTP-JSON icon | Destination | airbyte/destination-sftp-json:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/sftp-json) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-sftp-json) | `e9810f61-4bab-46d2-bb22-edfc902e0644` | | **Scylla** | Scylla icon | Destination | airbyte/destination-scylla:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/destinations/scylla) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-scylla) | `3dc6f384-cd6b-4be3-ad16-a41450899bf0` | -| **Snowflake** | Snowflake icon | Destination | airbyte/destination-snowflake:0.4.53 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/snowflake) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-snowflake) | `424892c4-daac-4491-b35d-c6688ba547ba` | +| **Snowflake** | Snowflake icon | Destination | airbyte/destination-snowflake:0.4.54 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/snowflake) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-snowflake) | `424892c4-daac-4491-b35d-c6688ba547ba` | | **Streamr** | Streamr icon | Destination | ghcr.io/devmate-cloud/streamr-airbyte-connectors:0.0.1 | alpha | [link](https://docs.airbyte.com/integrations/destinations/streamr) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/devmate-cloud) | `eebd85cf-60b2-4af6-9ba0-edeca01437b0` | | **Teradata Vantage** | Teradata Vantage icon | Destination | airbyte/destination-teradata:0.1.1 | alpha | [link](https://docs.airbyte.io/integrations/destinations/teradata) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-teradata) | `58e6f9da-904e-11ed-a1eb-0242ac120002` | | **TiDB** | TiDB icon | Destination | airbyte/destination-tidb:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/tidb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-tidb) | `06ec60c7-7468-45c0-91ac-174f6e1a788b` | diff --git a/docs/integrations/destinations/bigquery-denormalized.md b/docs/integrations/destinations/bigquery-denormalized.md index 75d79ba863b0..283ad82c7fd9 100644 --- a/docs/integrations/destinations/bigquery-denormalized.md +++ b/docs/integrations/destinations/bigquery-denormalized.md @@ -8,7 +8,9 @@ See [destinations/bigquery](https://docs.airbyte.com/integrations/destinations/b | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| -| 1.2.15 | 2023-03-10 | [#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | +| 1.2.17 | 2023-03-17 | [#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | +| 1.2.16 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931)| Added support for periodic buffer flush | +| 1.2.15 | 2023-03-10 | [#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | | 1.2.14 | 2023-02-08 | [#22497](https://github.com/airbytehq/airbyte/pull/22497) | Fixed table already exists error | | 1.2.13 | 2023-01-26 | [#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging | | 1.2.12 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 96a5567f6a39..5bc7c51888f2 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -136,6 +136,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------| +| 1.2.17 | 2023-03-17 | [#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | | 1.2.16 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush | | 1.2.15 | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | | 1.2.14 | 2023-02-08 | [\#22497](https://github.com/airbytehq/airbyte/pull/22497) | Fixed table already exists error | diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md index 1ec0d523281a..e06c902069f0 100644 --- a/docs/integrations/destinations/gcs.md +++ b/docs/integrations/destinations/gcs.md @@ -237,6 +237,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | |:--------| :--- |:------------------------------------------------------------| :--- | +| 0.2.16 | 2023-03-17 | [#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | | 0.2.15 | 2023-03-10 | [#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | | 0.2.14 | 2023-11-23 | [\#21682](https://github.com/airbytehq/airbyte/pull/21682) | Add support for buckets with Customer-Managed Encryption Key | | 0.2.13 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 3b9156445ca1..7bb755e2b897 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -152,6 +152,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.3 | 2023-03-17 | [#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | | 0.4.2 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush | | 0.4.1 | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | | 0.4.0 | 2023-02-28 | [\#23523](https://github.com/airbytehq/airbyte/pull/23523) | Add SSH Bastion Host configuration options | diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index 6b8804aa43b9..614b0548d486 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -339,6 +339,7 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.22 | 2023-03-17 | [#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | | 0.3.21 | 2023-03-10 | [#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | | 0.3.20 | 2023-02-23 | [#21355](https://github.com/airbytehq/airbyte/pull/21355) | Add root level flattening option to JSONL output. | | 0.3.19 | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 310abbf0b0ee..ef89e790187c 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -287,6 +287,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.54 | 2023-03-17 | [#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | | 0.4.53 | 2023-03-15 | [\#24058](https://github.com/airbytehq/airbyte/pull/24058) | added write attempt to internal staging Check method | | 0.4.52 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush | | 0.4.51 | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long |