JDBC Sources: remove redundant date-time datatypes formatting (#20760)

* JDBC sources: consolidate date/time values mapping * fixed Snowflake acceptance tests * fixed TestJdbcUtils * refactoring * update setDate for incremental sync * fixed CockroachDbJdbcSourceAcceptanceTest * fixed CockroachDbSourceDatatypeTest * fixed CockroachDbSourceDatatypeTest * fixed MsSQLSourceDatatypeTest * fixed MsSQL cdc datatype test * refactoring * fixed mssql cdc test * fixed Redshift tests * fixed Redshift tests * updated db2 datatype tests * fixed DB2 tests * fixed DB2 strict-encrypt tests * fixed Oracle strict-encrypt tests * refactoring * updated changelog * refactoring * removed redundant overrides * bump versions * auto-bump connector version * JDBC Sources: remove redundant date parsing * updated Snowflake test * updated cockroachdb test * updated date-time datatypes parsing in Postgres Source * updated formatting in JDBC sources * updated test cases * updated Snowflake test * updated Snowflake, Redshift, Cockroach, DB2 tests * updated Snowflake, Postgres tests * updated Redshift tests * updated MySQL, MsSQL, DB2 tests * updated MsSQL tests * updated Oracle tests * updated Oracle tests * updated Oracle tests * fixed some tests * refactoring * fixed checkstyle * fixed TestJdbcUtils * randomize Reshift test username * removed redundant test * updated sanity test * Automated Change * format time and bump versions * auto-bump connector version * bump versions * auto-bump connector version * auto-bump connector version * manually update definitions * Automated Change --------- Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com> Co-authored-by: ievgeniit <etsybaev@gmail.com> Co-authored-by: VitaliiMaltsev <VitaliiMaltsev@users.noreply.github.com>
airbytehq · Mar 23, 2023 · 12eac42 · 12eac42
1 parent 69736d9
commit 12eac42
Show file tree

Hide file tree

Showing 55 changed files with 207 additions and 322 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -45,7 +45,7 @@
 - name: AlloyDB for PostgreSQL
   sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
   dockerRepository: airbyte/source-alloydb
-  dockerImageTag: 2.0.6
+  dockerImageTag: 2.0.9
   documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   icon: alloydb.svg
   sourceType: database
@@ -280,7 +280,7 @@
 - name: ClickHouse
   sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c
   dockerRepository: airbyte/source-clickhouse
-  dockerImageTag: 0.1.16
+  dockerImageTag: 0.1.17
   documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse
   icon: clickhouse.svg
   sourceType: database
@@ -319,7 +319,7 @@
 - name: Cockroachdb
   sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
   dockerRepository: airbyte/source-cockroachdb
-  dockerImageTag: 0.1.21
+  dockerImageTag: 0.1.22
   documentationUrl: https://docs.airbyte.com/integrations/sources/cockroachdb
   icon: cockroachdb.svg
   sourceType: database
@@ -922,7 +922,7 @@
 - name: IBM Db2
   sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2
   dockerRepository: airbyte/source-db2
-  dockerImageTag: 0.1.18
+  dockerImageTag: 0.1.19
   documentationUrl: https://docs.airbyte.com/integrations/sources/db2
   icon: db2.svg
   sourceType: database
@@ -1190,7 +1190,7 @@
 - name: Microsoft SQL Server (MSSQL)
   sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
   dockerRepository: airbyte/source-mssql
-  dockerImageTag: 1.0.5
+  dockerImageTag: 1.0.6
   documentationUrl: https://docs.airbyte.com/integrations/sources/mssql
   icon: mssql.svg
   sourceType: database
@@ -1253,7 +1253,7 @@
 - name: MySQL
   sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
   dockerRepository: airbyte/source-mysql
-  dockerImageTag: 2.0.7
+  dockerImageTag: 2.0.8
   documentationUrl: https://docs.airbyte.com/integrations/sources/mysql
   icon: mysql.svg
   sourceType: database
@@ -1373,7 +1373,7 @@
 - name: Oracle DB
   sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83
   dockerRepository: airbyte/source-oracle
-  dockerImageTag: 0.3.23
+  dockerImageTag: 0.3.24
   documentationUrl: https://docs.airbyte.com/integrations/sources/oracle
   icon: oracle.svg
   sourceType: database
@@ -1565,7 +1565,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 2.0.8
+  dockerImageTag: 2.0.9
   documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database
@@ -1904,7 +1904,7 @@
 - name: Snowflake
   sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2
   dockerRepository: airbyte/source-snowflake
-  dockerImageTag: 0.1.31
+  dockerImageTag: 0.1.32
   documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake
   icon: snowflake.svg
   sourceType: database
@@ -2019,7 +2019,7 @@
 - name: TiDB
   sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae
   dockerRepository: airbyte/source-tidb
-  dockerImageTag: 0.2.3
+  dockerImageTag: 0.2.4
   documentationUrl: https://docs.airbyte.com/integrations/sources/tidb
   icon: tidb.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -370,7 +370,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-alloydb:2.0.6"
+- dockerImage: "airbyte/source-alloydb:2.0.9"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -388,6 +388,7 @@
           description: "Hostname of the database."
           type: "string"
           order: 0
+          group: "db"
         port:
           title: "Port"
           description: "Port of the database."
@@ -398,11 +399,13 @@
           examples:
           - "5432"
           order: 1
+          group: "db"
         database:
           title: "Database Name"
           description: "Name of the database."
           type: "string"
           order: 2
+          group: "db"
         schemas:
           title: "Schemas"
           description: "The list of schemas (case sensitive) to sync from. Defaults\
@@ -415,17 +418,21 @@
           default:
           - "public"
           order: 3
+          group: "db"
         username:
           title: "Username"
           description: "Username to access the database."
           type: "string"
           order: 4
+          group: "auth"
         password:
           title: "Password"
           description: "Password associated with the username."
           type: "string"
           airbyte_secret: true
           order: 5
+          group: "auth"
+          always_show: true
         jdbc_url_params:
           description: "Additional properties to pass to the JDBC URL string when\
             \ connecting to the database formatted as 'key=value' pairs separated\
@@ -435,19 +442,23 @@
           title: "JDBC URL Parameters (Advanced)"
           type: "string"
           order: 6
+          group: "advanced"
         ssl:
           title: "Connect using SSL"
           description: "Encrypt data using SSL. When activating SSL, please select\
             \ one of the connection modes."
           type: "boolean"
           default: false
           order: 7
+          group: "security"
+          always_show: true
         ssl_mode:
           title: "SSL Modes"
           description: "SSL connection modes. \n  Read more <a href=\"https://jdbc.postgresql.org/documentation/head/ssl-client.html\"\
             > in the docs</a>."
           type: "object"
-          order: 7
+          order: 8
+          group: "security"
           oneOf:
           - title: "disable"
             additionalProperties: true
@@ -576,7 +587,8 @@
           type: "object"
           title: "Replication Method"
           description: "Replication method for extracting data from the database."
-          order: 8
+          order: 9
+          group: "advanced"
           oneOf:
           - title: "Standard"
             description: "Standard replication requires no setup on the DB side but\
@@ -753,6 +765,14 @@
                 type: "string"
                 airbyte_secret: true
                 order: 4
+          group: "security"
+      groups:
+      - id: "db"
+      - id: "auth"
+      - id: "security"
+        title: "Security"
+      - id: "advanced"
+        title: "Advanced"
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
@@ -2326,7 +2346,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-clickhouse:0.1.16"
+- dockerImage: "airbyte/source-clickhouse:0.1.17"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/clickhouse"
     connectionSpecification:
@@ -2625,7 +2645,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-cockroachdb:0.1.21"
+- dockerImage: "airbyte/source-cockroachdb:0.1.22"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/cockroachdb"
     connectionSpecification:
@@ -6880,7 +6900,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-db2:0.1.18"
+- dockerImage: "airbyte/source-db2:0.1.19"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/db2"
     connectionSpecification:
@@ -8461,7 +8481,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mssql:1.0.5"
+- dockerImage: "airbyte/source-mssql:1.0.6"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/destinations/mssql"
     connectionSpecification:
@@ -9336,7 +9356,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mysql:2.0.7"
+- dockerImage: "airbyte/source-mysql:2.0.8"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql"
     connectionSpecification:
@@ -10690,7 +10710,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-oracle:0.3.23"
+- dockerImage: "airbyte/source-oracle:0.3.24"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/oracle"
     connectionSpecification:
@@ -11912,7 +11932,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:2.0.8"
+- dockerImage: "airbyte/source-postgres:2.0.9"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -14405,7 +14425,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-snowflake:0.1.31"
+- dockerImage: "airbyte/source-snowflake:0.1.32"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/snowflake"
     connectionSpecification:
@@ -15261,7 +15281,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-tidb:0.2.3"
+- dockerImage: "airbyte/source-tidb:0.2.4"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/tidb"
     connectionSpecification:

diff --git a/...te-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/...te-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java
@@ -5,7 +5,6 @@
 package io.airbyte.db.jdbc;
 
 import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER;
-import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -21,11 +20,7 @@
 import java.sql.SQLException;
 import java.sql.Timestamp;
 import java.text.ParseException;
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.LocalTime;
-import java.time.OffsetDateTime;
-import java.time.OffsetTime;
+import java.time.*;
 import java.time.chrono.IsoEra;
 import java.time.format.DateTimeParseException;
 import java.util.Collections;
@@ -123,17 +118,21 @@ protected void putString(final ObjectNode node, final String columnName, final R
   }
 
   protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
-    node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, LocalDate.class)));
+    node.put(columnName, resultSet.getString(index));
   }
 
   protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
     node.put(columnName, DateTimeConverter.convertToTime(getObject(resultSet, index, LocalTime.class)));
   }
 
   protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
-    // https://www.cis.upenn.edu/~bcpierce/courses/629/jdkdocs/guide/jdbc/getstart/mapping.doc.html
-    final Instant instant = resultSet.getTimestamp(index).toInstant();
-    node.put(columnName, DataTypeUtils.toISO8601StringWithMicroseconds(instant));
+    try {
+      node.put(columnName, DateTimeConverter.convertToTimestamp(getObject(resultSet, index, LocalDateTime.class)));
+    } catch (Exception e) {
+      // for backward compatibility
+      final Instant instant = resultSet.getTimestamp(index).toInstant();
+      node.put(columnName, DataTypeUtils.toISO8601StringWithMicroseconds(instant));
+    }
   }
 
   protected void putBinary(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
@@ -153,29 +152,10 @@ protected void setTime(final PreparedStatement preparedStatement, final int para
   }
 
   protected void setTimestamp(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException {
-    // parse time, and timestamp the same way. this seems to not cause an problems and allows us
-    // to treat them all as ISO8601. if this causes any problems down the line, we can adjust.
-    // Parsing TIME as a TIMESTAMP might potentially break for ClickHouse cause it doesn't expect TIME
-    // value in the following format
     try {
-      var valueWithoutMicros = value;
-      final StringBuilder nanos = new StringBuilder();
-      final var dotIndex = value.indexOf(".");
-      if (dotIndex > 0) {
-        final var micro = value.substring(value.lastIndexOf('.') + 1, value.length() - 1);
-        nanos.append(micro);
-        valueWithoutMicros = value.replace("." + micro, "");
-      }
-      while (nanos.length() != 9) {
-        nanos.append("0");
-      }
-
-      final var timestamp = Timestamp
-          .from(DataTypeUtils.getDateFormat().parse(valueWithoutMicros).toInstant());
-      timestamp.setNanos(Integer.parseInt(nanos.toString()));
-      preparedStatement.setTimestamp(parameterIndex, timestamp);
-    } catch (final ParseException e) {
-      throw new RuntimeException(e);
+      preparedStatement.setObject(parameterIndex, LocalDateTime.parse(value));
+    } catch (final DateTimeParseException e) {
+      preparedStatement.setObject(parameterIndex, OffsetDateTime.parse(value));
     }
   }
 
@@ -246,7 +226,7 @@ protected <ObjectType> ObjectType getObject(final ResultSet resultSet, final int
 
   protected void putTimeWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException {
     final OffsetTime timetz = getObject(resultSet, index, OffsetTime.class);
-    node.put(columnName, timetz.format(TIMETZ_FORMATTER));
+    node.put(columnName, DateTimeConverter.convertToTimeWithTimezone(timetz));
   }
 
   protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index)