From 50aa1003684903477467972c09378bc6d4c0ab4a Mon Sep 17 00:00:00 2001 From: Tobias Macey Date: Fri, 10 Mar 2023 16:56:45 -0500 Subject: [PATCH 1/5] destination-s3-glue: Fix struct and array schema generation The `struct` schema type was being generated with invalid syntax due to the inclusion of space characters around the `:` character in the type mapping of the struct field. The `array` data type was not properly handling the optionality of the `items` property. This lead to hard failures whenever dealing with array types that didn't set the `items` property. This adds conditional handling of when the `items` type is present and otherwise assumes that the type is a string. --- .../src/main/resources/seed/destination_definitions.yaml | 2 +- .../connectors/destination-s3-glue/Dockerfile | 2 +- .../integrations/destination/s3_glue/GlueOperations.java | 9 ++++++--- .../destination/s3_glue/S3GlueConsumerFactory.java | 2 +- docs/integrations/destinations/s3-glue.md | 1 + 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 187c16aabaab..73ef031e9297 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -342,7 +342,7 @@ - name: S3 Glue destinationDefinitionId: 471e5cab-8ed1-49f3-ba11-79c687784737 dockerRepository: airbyte/destination-s3-glue - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-glue icon: s3-glue.svg releaseStage: alpha diff --git a/airbyte-integrations/connectors/destination-s3-glue/Dockerfile b/airbyte-integrations/connectors/destination-s3-glue/Dockerfile index c9c573b6fb96..675ad2003a72 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3-glue/Dockerfile @@ -14,5 +14,5 @@ ENV APPLICATION destination-s3-glue COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-s3-glue diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueOperations.java b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueOperations.java index c378ce6dbe13..2e6666ebc48b 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueOperations.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueOperations.java @@ -129,13 +129,15 @@ private String transformSchemaRecursive(JsonNode jsonNode) { if (jsonNode.has("airbyte_type") && jsonNode.get("airbyte_type").asText().equals("integer")) { yield "int"; } - yield "float"; + yield "decimal"; // Default to use decimal as it is a more precise type and allows for large values } case "boolean" -> "boolean"; case "integer" -> "int"; case "array" -> { String arrayType = "array<"; - Set itemTypes = filterTypes(jsonNode.get("items").get("type")); + Set itemTypes; + if (jsonNode.has("items")) { + itemTypes = filterTypes(jsonNode.get("items").get("type")); if (itemTypes.size() > 1) { // TODO(itaseski) use union instead of array when having multiple types (rare occurrence)? arrayType += "string>"; @@ -143,6 +145,7 @@ private String transformSchemaRecursive(JsonNode jsonNode) { String subtype = transformSchemaRecursive(jsonNode.get("items")); arrayType += (subtype + ">"); } + } else arrayType += "string>"; yield arrayType; } case "object" -> { @@ -150,7 +153,7 @@ private String transformSchemaRecursive(JsonNode jsonNode) { String objectType = "struct<"; Map properties = objectMapper.convertValue(jsonNode.get("properties"), new TypeReference<>() {}); String columnTypes = properties.entrySet().stream() - .map(p -> p.getKey() + " : " + transformSchemaRecursive(p.getValue())) + .map(p -> p.getKey() + ":" + transformSchemaRecursive(p.getValue())) .collect(Collectors.joining(",")); objectType += (columnTypes + ">"); yield objectType; diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java index dba3e76e6810..3815730df426 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java @@ -88,7 +88,7 @@ private static Function toWriteConfi final DestinationSyncMode syncMode = stream.getDestinationSyncMode(); final JsonNode jsonSchema = abStream.getJsonSchema(); ((ObjectNode) jsonSchema.get("properties")).putPOJO(JavaBaseConstants.COLUMN_NAME_AB_ID, Map.of("type", "string")); - ((ObjectNode) jsonSchema.get("properties")).putPOJO(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, Map.of("type", "integer")); + ((ObjectNode) jsonSchema.get("properties")).putPOJO(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, Map.of("type", "number")); final String location = "s3://" + s3Config.getBucketName() + "/" + fullOutputPath.substring(0, fullOutputPath.lastIndexOf("/") + 1); final S3GlueWriteConfig writeConfig = diff --git a/docs/integrations/destinations/s3-glue.md b/docs/integrations/destinations/s3-glue.md index 6e8c142a6642..1ba9a36f8b34 100644 --- a/docs/integrations/destinations/s3-glue.md +++ b/docs/integrations/destinations/s3-glue.md @@ -245,6 +245,7 @@ Output files can be compressed. The default option is GZIP compression. If compr | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------| +| 0.1.4 | 2023-03-10 | [23950](https://github.com/airbytehq/airbyte/pull/23950) | Fix schema syntax error for struct fields and handle missing `items` in array fields | | 0.1.3 | 2023-02-10 | [22822](https://github.com/airbytehq/airbyte/pull/22822) | Fix data type for _ab_emitted_at column in table definition | | 0.1.2 | 2023-02-01 | [22220](https://github.com/airbytehq/airbyte/pull/22220) | Fix race condition in test, table metadata, add Airbyte sync fields to table definition | | 0.1.1 | 2022-12-13 | [19907](https://github.com/airbytehq/airbyte/pull/19907) | Fix parsing empty object in schema | From b9220ac31c88561ff399782e9963473d15cb6da8 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 29 Mar 2023 17:19:35 -0300 Subject: [PATCH 2/5] rollback change --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 73ef031e9297..187c16aabaab 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -342,7 +342,7 @@ - name: S3 Glue destinationDefinitionId: 471e5cab-8ed1-49f3-ba11-79c687784737 dockerRepository: airbyte/destination-s3-glue - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-glue icon: s3-glue.svg releaseStage: alpha From 8c17a823df64c08bff64ea8a0baeefa692c7b7c4 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 29 Mar 2023 17:34:39 -0300 Subject: [PATCH 3/5] bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 187c16aabaab..73ef031e9297 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -342,7 +342,7 @@ - name: S3 Glue destinationDefinitionId: 471e5cab-8ed1-49f3-ba11-79c687784737 dockerRepository: airbyte/destination-s3-glue - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-glue icon: s3-glue.svg releaseStage: alpha From 9e3aa72af596667c5e5ad490f3aa2a0472004151 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 29 Mar 2023 18:15:46 -0300 Subject: [PATCH 4/5] return version to 0.1.3 --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 73ef031e9297..187c16aabaab 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -342,7 +342,7 @@ - name: S3 Glue destinationDefinitionId: 471e5cab-8ed1-49f3-ba11-79c687784737 dockerRepository: airbyte/destination-s3-glue - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-glue icon: s3-glue.svg releaseStage: alpha From 140443504e71fae254cd16b71aa967909948d61a Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 29 Mar 2023 21:48:32 +0000 Subject: [PATCH 5/5] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- connectors.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 187c16aabaab..73ef031e9297 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -342,7 +342,7 @@ - name: S3 Glue destinationDefinitionId: 471e5cab-8ed1-49f3-ba11-79c687784737 dockerRepository: airbyte/destination-s3-glue - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-glue icon: s3-glue.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 54066446571d..ead649222723 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -6071,7 +6071,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-s3-glue:0.1.3" +- dockerImage: "airbyte/destination-s3-glue:0.1.4" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/s3-glue" connectionSpecification: diff --git a/connectors.md b/connectors.md index 057b127ef54b..4140b0a8f500 100644 --- a/connectors.md +++ b/connectors.md @@ -330,7 +330,7 @@ | **Redshift** | Redshift icon | Destination | airbyte/destination-redshift:0.4.3 | beta | [link](https://docs.airbyte.com/integrations/destinations/redshift) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-redshift) | `f7a7d195-377f-cf5b-70a5-be6b819019dc` | | **Rockset** | x | Destination | airbyte/destination-rockset:0.1.4 | alpha | [link](https://docs.airbyte.com/integrations/destinations/rockset) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-rockset) | `2c9d93a7-9a17-4789-9de9-f46f0097eb70` | | **S3** | S3 icon | Destination | airbyte/destination-s3:0.3.22 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/s3) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3) | `4816b78f-1489-44c1-9060-4b19d5fa9362` | -| **S3 Glue** | S3 Glue icon | Destination | airbyte/destination-s3-glue:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/destinations/s3-glue) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3-glue) | `471e5cab-8ed1-49f3-ba11-79c687784737` | +| **S3 Glue** | S3 Glue icon | Destination | airbyte/destination-s3-glue:0.1.4 | alpha | [link](https://docs.airbyte.com/integrations/destinations/s3-glue) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3-glue) | `471e5cab-8ed1-49f3-ba11-79c687784737` | | **SFTP-JSON** | SFTP-JSON icon | Destination | airbyte/destination-sftp-json:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/sftp-json) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-sftp-json) | `e9810f61-4bab-46d2-bb22-edfc902e0644` | | **Scylla** | Scylla icon | Destination | airbyte/destination-scylla:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/destinations/scylla) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-scylla) | `3dc6f384-cd6b-4be3-ad16-a41450899bf0` | | **Snowflake** | Snowflake icon | Destination | airbyte/destination-snowflake:0.4.58 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/snowflake) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-snowflake) | `424892c4-daac-4491-b35d-c6688ba547ba` |