Destination S3 Glue: fix emitted at column (#24359)

* destination-s3-glue: Fix data type for emitted at column The `_ab_emitted_at` column is actually a numeric type and not a string. This updates the generated schema to treat it as the proper type. * fix tests * rollback seed dest spec update * auto-bump connector version --------- Co-authored-by: Tobias Macey <tmacey@mit.edu> Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Mar 22, 2023 · 71097dd · 71097dd
1 parent c1323f6
commit 71097dd
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 7 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -334,7 +334,7 @@
 - name: S3 Glue
   destinationDefinitionId: 471e5cab-8ed1-49f3-ba11-79c687784737
   dockerRepository: airbyte/destination-s3-glue
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-glue
   icon: s3-glue.svg
   releaseStage: alpha

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -5921,9 +5921,9 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-s3-glue:0.1.2"
+- dockerImage: "airbyte/destination-s3-glue:0.1.3"
   spec:
-    documentationUrl: "https://docs.airbyte.com/integrations/destinations/s3"
+    documentationUrl: "https://docs.airbyte.com/integrations/destinations/s3-glue"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "S3 Destination Spec"

diff --git a/airbyte-integrations/connectors/destination-s3-glue/Dockerfile b/airbyte-integrations/connectors/destination-s3-glue/Dockerfile
@@ -14,5 +14,5 @@ ENV APPLICATION destination-s3-glue
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/destination-s3-glue
diff --git a/...glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java b/...glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java
@@ -88,7 +88,7 @@ private static Function<ConfiguredAirbyteStream, S3GlueWriteConfig> toWriteConfi
       final DestinationSyncMode syncMode = stream.getDestinationSyncMode();
       final JsonNode jsonSchema = abStream.getJsonSchema();
       ((ObjectNode) jsonSchema.get("properties")).putPOJO(JavaBaseConstants.COLUMN_NAME_AB_ID, Map.of("type", "string"));
-      ((ObjectNode) jsonSchema.get("properties")).putPOJO(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, Map.of("type", "string"));
+      ((ObjectNode) jsonSchema.get("properties")).putPOJO(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, Map.of("type", "integer"));
       final String location = "s3://" + s3Config.getBucketName() + "/" +
           fullOutputPath.substring(0, fullOutputPath.lastIndexOf("/") + 1);
       final S3GlueWriteConfig writeConfig =

diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json
@@ -1,5 +1,5 @@
 {
-  "documentationUrl": "https://docs.airbyte.com/integrations/destinations/s3",
+  "documentationUrl": "https://docs.airbyte.com/integrations/destinations/s3-glue",
   "supportsIncremental": true,
   "supportsNormalization": false,
   "supportsDBT": false,

diff --git a/...ava/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java b/...ava/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java
@@ -5,6 +5,9 @@
 package io.airbyte.integrations.destination.s3_glue;
 
 import io.airbyte.integrations.destination.s3.S3BaseJsonlDestinationAcceptanceTest;
+import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ArgumentsSource;
 
 public class S3GlueJsonlDestinationAcceptanceTest extends S3BaseJsonlDestinationAcceptanceTest {
 
@@ -25,4 +28,18 @@ protected String getImageName() {
     return "airbyte/destination-s3-glue:dev";
   }
 
+  @ParameterizedTest
+  @ArgumentsSource(DataTypeTestArgumentProvider.class)
+  public void testDataTypeTestWithNormalization(final String messagesFilename,
+                                                final String catalogFilename,
+                                                final DataTypeTestArgumentProvider.TestCompatibility testCompatibility)
+      throws Exception {
+
+    if (messagesFilename.contains("array")) {
+      return;
+    }
+
+    super.testDataTypeTestWithNormalization(messagesFilename, catalogFilename, testCompatibility);
+  }
+
 }
diff --git a/connectors.md b/connectors.md
@@ -327,7 +327,7 @@
 | **Redshift** | <img alt="Redshift icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/redshift.svg" height="30" height="30"/> | Destination | airbyte/destination-redshift:0.4.3 | beta | [link](https://docs.airbyte.com/integrations/destinations/redshift) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-redshift) | <small>`f7a7d195-377f-cf5b-70a5-be6b819019dc`</small> |
 | **Rockset** | x | Destination | airbyte/destination-rockset:0.1.4 | alpha | [link](https://docs.airbyte.com/integrations/destinations/rockset) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-rockset) | <small>`2c9d93a7-9a17-4789-9de9-f46f0097eb70`</small> |
 | **S3** | <img alt="S3 icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/s3.svg" height="30" height="30"/> | Destination | airbyte/destination-s3:0.3.22 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/s3) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3) | <small>`4816b78f-1489-44c1-9060-4b19d5fa9362`</small> |
-| **S3 Glue** | <img alt="S3 Glue icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/s3-glue.svg" height="30" height="30"/> | Destination | airbyte/destination-s3-glue:0.1.2 | alpha | [link](https://docs.airbyte.com/integrations/destinations/s3-glue) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3-glue) | <small>`471e5cab-8ed1-49f3-ba11-79c687784737`</small> |
+| **S3 Glue** | <img alt="S3 Glue icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/s3-glue.svg" height="30" height="30"/> | Destination | airbyte/destination-s3-glue:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/destinations/s3-glue) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-s3-glue) | <small>`471e5cab-8ed1-49f3-ba11-79c687784737`</small> |
 | **SFTP-JSON** | <img alt="SFTP-JSON icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/sftp.svg" height="30" height="30"/> | Destination | airbyte/destination-sftp-json:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/destinations/sftp-json) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-sftp-json) | <small>`e9810f61-4bab-46d2-bb22-edfc902e0644`</small> |
 | **Scylla** | <img alt="Scylla icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/scylla.svg" height="30" height="30"/> | Destination | airbyte/destination-scylla:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/destinations/scylla) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-scylla) | <small>`3dc6f384-cd6b-4be3-ad16-a41450899bf0`</small> |
 | **Snowflake** | <img alt="Snowflake icon" src="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-config/init/src/main/resources/icons/snowflake.svg" height="30" height="30"/> | Destination | airbyte/destination-snowflake:0.4.56 | generally_available | [link](https://docs.airbyte.com/integrations/destinations/snowflake) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/destination-snowflake) | <small>`424892c4-daac-4491-b35d-c6688ba547ba`</small> |

diff --git a/docs/integrations/destinations/s3-glue.md b/docs/integrations/destinations/s3-glue.md
@@ -245,6 +245,7 @@ Output files can be compressed. The default option is GZIP compression. If compr
 
 | Version | Date       | Pull Request                                             | Subject                                                                                 |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------|
+| 0.1.3   | 2023-02-10 | [22822](https://github.com/airbytehq/airbyte/pull/22822) | Fix data type for _ab_emitted_at column in table definition                             |
 | 0.1.2   | 2023-02-01 | [22220](https://github.com/airbytehq/airbyte/pull/22220) | Fix race condition in test, table metadata, add Airbyte sync fields to table definition |
 | 0.1.1   | 2022-12-13 | [19907](https://github.com/airbytehq/airbyte/pull/19907) | Fix parsing empty object in schema                                                      |
 | 0.1.0   | 2022-11-17 | [18695](https://github.com/airbytehq/airbyte/pull/18695) | Initial Commit                                                                          |