Merge remote-tracking branch 'origin/master' into ddavydov/#12591-sou…

…rce-zendesk-support-upd-cdk-version
airbytehq · Jun 1, 2022 · a43b2d6 · a43b2d6
2 parents 0275fb9 + 4ddb4f5
commit a43b2d6
Show file tree

Hide file tree

Showing 77 changed files with 1,227 additions and 305 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.39.5-alpha
+current_version = 0.39.7-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

diff --git a/.env b/.env
@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.39.5-alpha
+VERSION=0.39.7-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md
@@ -8,6 +8,7 @@ assignees: ''
 ---
 <!-- 
 Welcome to Airbyte!
+
 We're really appreciate your report and know that this will help us build an 
 amazing tool. If you want to contribute yourself, you can find a good place 
 to start by searching for the good-first-issues label or maybe... by trying 
@@ -28,7 +29,6 @@ You can remove the examples bellow and fill out with your information.
 - **Deployment**: example are Docker or Kubernetes deploy env
 - **Source Connector and version**: (if applicable example Salesforce 0.2.3) <!-- Found in the admin page in the UI in the Source tab. -->
 - **Destination Connector and version**: (if applicable example Postgres 0.3.3) <!-- Found in the admin page in the UI in the Destination tab. -->
-- **Severity**: Very Low / Low / Medium / High / Critical
 - **Step where error happened**: Deploy / Sync job / Setup new connection / Update connector / Upgrade Airbyte
 
 ## Current Behavior
@@ -39,8 +39,11 @@ You can remove the examples bellow and fill out with your information.
 
 ## Logs
 <!--
-If applicable: Please upload the complete log files from the failing operation. 
-We ask for complete log files because these files capture essential metadata about your environment. The team will read the full stack trace of errors and try to find the root cause of the error. For a sync failure, you can download the log in the sync page, and for other errors please download the server and scheduler logs in the Settings Page -> Configuration.
+Please upload the complete log files from the failing operation. 
+
+We ask for complete log files because these files capture essential metadata about your environment.
+The team will read the full stack trace of errors and try to find the root cause of the error. 
+For a sync failure, you can download the log in the sync page, and for other errors please download the server and scheduler logs in the Settings Page -> Configuration.
 You can remove this after uploading the file.
 -->
 

diff --git a/.github/actions/build-and-push-branch/action.yml b/.github/actions/build-and-push-branch/action.yml
@@ -4,6 +4,9 @@ inputs:
   branch_version_tag:
     description: 'Used to tag jars and docker images with a branch-specific version (should use the form "dev-<commit_hash>" to pass AirbyteVersion validation)'
     required: false
+  dockerhub_username:
+    description: "Used to log in to dockerhub for pushing images"
+    required: true
   dockerhub_token:
     description: "Used to log in to dockerhub for pushing images"
     required: true
@@ -19,7 +22,7 @@ runs:
     - name: Login to Docker (on Master)
       uses: docker/login-action@v1
       with:
-        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        username: ${{ inputs.dockerhub_username }}
         password: ${{ inputs.dockerhub_token }}
 
     - name: Push Docker Images

diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile
@@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.1
 ARG JDK_IMAGE=openjdk:${JDK_VERSION}-slim
 FROM ${JDK_IMAGE}
 
-ARG VERSION=0.39.5-alpha
+ARG VERSION=0.39.7-alpha
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
@@ -264,7 +264,7 @@
 - name: Snowflake
   destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
   dockerRepository: airbyte/destination-snowflake
-  dockerImageTag: 0.4.26
+  dockerImageTag: 0.4.28
   documentationUrl: https://docs.airbyte.io/integrations/destinations/snowflake
   icon: snowflake.svg
   resourceRequirements:

diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml
@@ -4238,7 +4238,7 @@
     supported_destination_sync_modes:
     - "overwrite"
     - "append"
-- dockerImage: "airbyte/destination-snowflake:0.4.26"
+- dockerImage: "airbyte/destination-snowflake:0.4.28"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/destinations/snowflake"
     connectionSpecification:

diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -715,7 +715,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 0.4.18
+  dockerImageTag: 0.4.19
   documentationUrl: https://docs.airbyte.io/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database
@@ -786,11 +786,11 @@
 - name: S3
   sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
   dockerRepository: airbyte/source-s3
-  dockerImageTag: 0.1.14
+  dockerImageTag: 0.1.15
   documentationUrl: https://docs.airbyte.io/integrations/sources/s3
   icon: s3.svg
   sourceType: file
-  releaseStage: alpha
+  releaseStage: beta
 - name: SalesLoft
   sourceDefinitionId: 41991d12-d4b5-439e-afd0-260a31d4c53f
   dockerRepository: airbyte/source-salesloft

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -6700,7 +6700,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:0.4.18"
+- dockerImage: "airbyte/source-postgres:0.4.19"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -7331,7 +7331,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-s3:0.1.14"
+- dockerImage: "airbyte/source-s3:0.1.15"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/s3"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/s3"

diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile
@@ -28,7 +28,7 @@ RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] htt
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y kubectl
 
 # Don't change this manually.  Bump version expects to make moves based on this string
-ARG VERSION=0.39.5-alpha
+ARG VERSION=0.39.7-alpha
 
 ENV APPLICATION airbyte-container-orchestrator
 ENV VERSION=${VERSION}

diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-db/lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java
@@ -20,6 +20,10 @@
 import java.sql.Timestamp;
 import java.text.ParseException;
 import java.time.Instant;
+import java.time.LocalDate;
+import java.time.OffsetDateTime;
+import java.time.OffsetTime;
+import java.time.chrono.IsoEra;
 import java.util.Collections;
 import java.util.List;
 import java.util.StringJoiner;
@@ -246,4 +250,27 @@ public String getFullyQualifiedTableNameWithQuoting(final Connection connection,
     return schemaName != null ? enquoteIdentifier(connection, schemaName) + "." + quotedTableName : quotedTableName;
   }
 
+  protected <DateTime> DateTime getDateTimeObject(ResultSet resultSet, int index, Class<DateTime> clazz) throws SQLException {
+    return resultSet.getObject(index, clazz);
+  }
+
+  protected void putTimeWithTimezone(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException {
+    OffsetTime timetz = getDateTimeObject(resultSet, index, OffsetTime.class);
+    node.put(columnName, timetz.toString());
+  }
+
+  protected void putTimestampWithTimezone(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException {
+    OffsetDateTime timestamptz = getDateTimeObject(resultSet, index, OffsetDateTime.class);
+    LocalDate localDate = timestamptz.toLocalDate();
+    node.put(columnName, resolveEra(localDate, timestamptz.toString()));
+  }
+
+  protected String resolveEra(LocalDate date, String value) {
+    return isBCE(date) ? value.substring(1) + " BC" : value;
+  }
+
+  public static boolean isBCE(LocalDate date) {
+    return date.getEra().equals(IsoEra.BCE);
+  }
+
 }
diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile
@@ -20,5 +20,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1
 
 ENV ENABLE_SENTRY true
 
-LABEL io.airbyte.version=0.4.26
+LABEL io.airbyte.version=0.4.28
 LABEL io.airbyte.name=airbyte/destination-snowflake
diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle
@@ -38,6 +38,7 @@ dependencies {
     implementation project(':airbyte-db:lib')
     implementation project(':airbyte-integrations:bases:base-java')
     implementation project(':airbyte-integrations:connectors:destination-jdbc')
+    implementation project(':airbyte-integrations:connectors:destination-gcs')
     implementation project(':airbyte-integrations:connectors:destination-s3')
     implementation project(':airbyte-protocol:models')
 

diff --git a/.../main/java/io/airbyte/integrations/destination/snowflake/SnowflakeCopyGcsDestination.java b/.../main/java/io/airbyte/integrations/destination/snowflake/SnowflakeCopyGcsDestination.java
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
- */
-
-package io.airbyte.integrations.destination.snowflake;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import io.airbyte.db.jdbc.JdbcDatabase;
-import io.airbyte.integrations.base.AirbyteMessageConsumer;
-import io.airbyte.integrations.destination.ExtendedNameTransformer;
-import io.airbyte.integrations.destination.jdbc.SqlOperations;
-import io.airbyte.integrations.destination.jdbc.copy.CopyConsumerFactory;
-import io.airbyte.integrations.destination.jdbc.copy.CopyDestination;
-import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsConfig;
-import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsStreamCopier;
-import io.airbyte.protocol.models.AirbyteMessage;
-import io.airbyte.protocol.models.ConfiguredAirbyteCatalog;
-import java.util.function.Consumer;
-import javax.sql.DataSource;
-
-public class SnowflakeCopyGcsDestination extends CopyDestination {
-
-  @Override
-  public AirbyteMessageConsumer getConsumer(final JsonNode config,
-                                            final ConfiguredAirbyteCatalog catalog,
-                                            final Consumer<AirbyteMessage> outputRecordCollector) {
-    final DataSource dataSource = getDataSource(config);
-    return CopyConsumerFactory.create(
-        outputRecordCollector,
-        dataSource,
-        getDatabase(dataSource),
-        getSqlOperations(),
-        getNameTransformer(),
-        GcsConfig.getGcsConfig(config),
-        catalog,
-        new SnowflakeGcsStreamCopierFactory(),
-        getConfiguredSchema(config));
-  }
-
-  @Override
-  public void checkPersistence(final JsonNode config) throws Exception {
-    GcsStreamCopier.attemptWriteToPersistence(GcsConfig.getGcsConfig(config));
-  }
-
-  @Override
-  public ExtendedNameTransformer getNameTransformer() {
-    return new SnowflakeSQLNameTransformer();
-  }
-
-  @Override
-  public DataSource getDataSource(final JsonNode config) {
-    return SnowflakeDatabase.createDataSource(config);
-  }
-
-  @Override
-  public JdbcDatabase getDatabase(final DataSource dataSource) {
-    return SnowflakeDatabase.getDatabase(dataSource);
-  }
-
-  @Override
-  public SqlOperations getSqlOperations() {
-    return new SnowflakeSqlOperations();
-  }
-
-  private String getConfiguredSchema(final JsonNode config) {
-    return config.get("schema").asText();
-  }
-
-}

diff --git a/...main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java b/...main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java
@@ -39,13 +39,13 @@ public static boolean isAzureBlobCopy(final JsonNode config) {
 
   public static Map<DestinationType, Destination> getTypeToDestination() {
     final SnowflakeS3StagingDestination s3StagingDestination = new SnowflakeS3StagingDestination();
-    final SnowflakeCopyGcsDestination copyGcsDestination = new SnowflakeCopyGcsDestination();
+    final SnowflakeGcsStagingDestination gcsStagingDestination = new SnowflakeGcsStagingDestination();
     final SnowflakeInternalStagingDestination internalStagingDestination = new SnowflakeInternalStagingDestination();
     final SnowflakeCopyAzureBlobStorageDestination azureBlobStorageDestination = new SnowflakeCopyAzureBlobStorageDestination();
 
     return ImmutableMap.of(
         DestinationType.COPY_S3, s3StagingDestination,
-        DestinationType.COPY_GCS, copyGcsDestination,
+        DestinationType.COPY_GCS, gcsStagingDestination,
         DestinationType.COPY_AZURE_BLOB, azureBlobStorageDestination,
         DestinationType.INTERNAL_STAGING, internalStagingDestination);
   }