From 12eac421de4414d4b216dc66d55f93e02f6fecad Mon Sep 17 00:00:00 2001 From: VitaliiMaltsev <39538064+VitaliiMaltsev@users.noreply.github.com> Date: Thu, 23 Mar 2023 16:31:50 +0200 Subject: [PATCH] JDBC Sources: remove redundant date-time datatypes formatting (#20760) * JDBC sources: consolidate date/time values mapping * fixed Snowflake acceptance tests * fixed TestJdbcUtils * refactoring * update setDate for incremental sync * fixed CockroachDbJdbcSourceAcceptanceTest * fixed CockroachDbSourceDatatypeTest * fixed CockroachDbSourceDatatypeTest * fixed MsSQLSourceDatatypeTest * fixed MsSQL cdc datatype test * refactoring * fixed mssql cdc test * fixed Redshift tests * fixed Redshift tests * updated db2 datatype tests * fixed DB2 tests * fixed DB2 strict-encrypt tests * fixed Oracle strict-encrypt tests * refactoring * updated changelog * refactoring * removed redundant overrides * bump versions * auto-bump connector version * JDBC Sources: remove redundant date parsing * updated Snowflake test * updated cockroachdb test * updated date-time datatypes parsing in Postgres Source * updated formatting in JDBC sources * updated test cases * updated Snowflake test * updated Snowflake, Redshift, Cockroach, DB2 tests * updated Snowflake, Postgres tests * updated Redshift tests * updated MySQL, MsSQL, DB2 tests * updated MsSQL tests * updated Oracle tests * updated Oracle tests * updated Oracle tests * fixed some tests * refactoring * fixed checkstyle * fixed TestJdbcUtils * randomize Reshift test username * removed redundant test * updated sanity test * Automated Change * format time and bump versions * auto-bump connector version * bump versions * auto-bump connector version * auto-bump connector version * manually update definitions * Automated Change --------- Co-authored-by: Octavia Squidington III Co-authored-by: ievgeniit Co-authored-by: VitaliiMaltsev --- .../resources/seed/source_definitions.yaml | 20 +-- .../src/main/resources/seed/source_specs.yaml | 44 ++++-- ...bstractJdbcCompatibleSourceOperations.java | 46 ++----- .../io/airbyte/db/jdbc/DateTimeConverter.java | 36 +++-- .../io/airbyte/db/jdbc/TestJdbcUtils.java | 4 +- .../debezium/internals/MSSQLConverter.java | 2 +- .../debezium/internals/PostgresConverter.java | 3 +- .../jdbc/SqlOperationsUtilsTest.java | 126 ------------------ .../source-alloydb-strict-encrypt/Dockerfile | 2 +- .../connectors/source-alloydb/Dockerfile | 2 +- .../Dockerfile | 2 +- .../connectors/source-clickhouse/Dockerfile | 2 +- .../Dockerfile | 2 +- .../connectors/source-cockroachdb/Dockerfile | 2 +- .../CockroachDbSourceDatatypeTest.java | 11 +- .../source-db2-strict-encrypt/Dockerfile | 2 +- .../connectors/source-db2/Dockerfile | 2 +- .../Db2SourceOperations.java | 23 ++-- .../sources/Db2SourceDatatypeTest.java | 6 +- .../source-mssql-strict-encrypt/Dockerfile | 2 +- .../connectors/source-mssql/Dockerfile | 2 +- .../source/mssql/MssqlSourceOperations.java | 6 - .../AbstractMssqlSourceDatatypeTest.java | 14 +- .../source-mysql-strict-encrypt/Dockerfile | 2 +- .../connectors/source-mysql/Dockerfile | 2 +- .../source/mysql/MySqlSourceOperations.java | 6 - .../AbstractMySqlSourceDatatypeTest.java | 8 +- .../source-oracle-strict-encrypt/Dockerfile | 2 +- .../connectors/source-oracle/Dockerfile | 2 +- .../OracleJdbcSourceAcceptanceTest.java | 18 +-- .../oracle/OracleSourceDatatypeTest.java | 4 +- .../source-postgres-strict-encrypt/Dockerfile | 2 +- .../connectors/source-postgres/Dockerfile | 2 +- .../postgres/PostgresSourceOperations.java | 6 +- .../AbstractPostgresSourceDatatypeTest.java | 22 +-- .../CdcWalLogsPostgresSourceDatatypeTest.java | 4 +- .../integration_tests/expected_records.jsonl | 4 +- .../redshift/RedshiftSourceOperations.java | 23 ++-- .../sources/RedshiftSourceAcceptanceTest.java | 2 +- .../connectors/source-snowflake/Dockerfile | 2 +- .../integration_tests/expected_records.jsonl | 4 +- .../SnowflakeSourceOperations.java | 10 -- .../sources/SnowflakeSourceDatatypeTest.java | 10 +- .../connectors/source-tidb/Dockerfile | 2 +- connectors.md | 20 +-- docs/integrations/sources/alloydb.md | 1 + docs/integrations/sources/clickhouse.md | 2 + docs/integrations/sources/cockroachdb.md | 1 + docs/integrations/sources/db2.md | 1 + docs/integrations/sources/mssql.md | 1 + docs/integrations/sources/mysql.md | 1 + docs/integrations/sources/oracle.md | 1 + docs/integrations/sources/postgres.md | 1 + docs/integrations/sources/snowflake.md | 1 + docs/integrations/sources/tidb.md | 1 + 55 files changed, 207 insertions(+), 322 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4e936598152e..4db082b77940 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -45,7 +45,7 @@ - name: AlloyDB for PostgreSQL sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 dockerRepository: airbyte/source-alloydb - dockerImageTag: 2.0.6 + dockerImageTag: 2.0.9 documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb icon: alloydb.svg sourceType: database @@ -280,7 +280,7 @@ - name: ClickHouse sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c dockerRepository: airbyte/source-clickhouse - dockerImageTag: 0.1.16 + dockerImageTag: 0.1.17 documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse icon: clickhouse.svg sourceType: database @@ -319,7 +319,7 @@ - name: Cockroachdb sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 dockerRepository: airbyte/source-cockroachdb - dockerImageTag: 0.1.21 + dockerImageTag: 0.1.22 documentationUrl: https://docs.airbyte.com/integrations/sources/cockroachdb icon: cockroachdb.svg sourceType: database @@ -922,7 +922,7 @@ - name: IBM Db2 sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 dockerRepository: airbyte/source-db2 - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.19 documentationUrl: https://docs.airbyte.com/integrations/sources/db2 icon: db2.svg sourceType: database @@ -1190,7 +1190,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 1.0.5 + dockerImageTag: 1.0.6 documentationUrl: https://docs.airbyte.com/integrations/sources/mssql icon: mssql.svg sourceType: database @@ -1253,7 +1253,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 2.0.7 + dockerImageTag: 2.0.8 documentationUrl: https://docs.airbyte.com/integrations/sources/mysql icon: mysql.svg sourceType: database @@ -1373,7 +1373,7 @@ - name: Oracle DB sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 dockerRepository: airbyte/source-oracle - dockerImageTag: 0.3.23 + dockerImageTag: 0.3.24 documentationUrl: https://docs.airbyte.com/integrations/sources/oracle icon: oracle.svg sourceType: database @@ -1565,7 +1565,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 2.0.8 + dockerImageTag: 2.0.9 documentationUrl: https://docs.airbyte.com/integrations/sources/postgres icon: postgresql.svg sourceType: database @@ -1904,7 +1904,7 @@ - name: Snowflake sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 dockerRepository: airbyte/source-snowflake - dockerImageTag: 0.1.31 + dockerImageTag: 0.1.32 documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake icon: snowflake.svg sourceType: database @@ -2019,7 +2019,7 @@ - name: TiDB sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae dockerRepository: airbyte/source-tidb - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.com/integrations/sources/tidb icon: tidb.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ae4913dce0ab..145678a67b95 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -370,7 +370,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-alloydb:2.0.6" +- dockerImage: "airbyte/source-alloydb:2.0.9" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -388,6 +388,7 @@ description: "Hostname of the database." type: "string" order: 0 + group: "db" port: title: "Port" description: "Port of the database." @@ -398,11 +399,13 @@ examples: - "5432" order: 1 + group: "db" database: title: "Database Name" description: "Name of the database." type: "string" order: 2 + group: "db" schemas: title: "Schemas" description: "The list of schemas (case sensitive) to sync from. Defaults\ @@ -415,17 +418,21 @@ default: - "public" order: 3 + group: "db" username: title: "Username" description: "Username to access the database." type: "string" order: 4 + group: "auth" password: title: "Password" description: "Password associated with the username." type: "string" airbyte_secret: true order: 5 + group: "auth" + always_show: true jdbc_url_params: description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ @@ -435,6 +442,7 @@ title: "JDBC URL Parameters (Advanced)" type: "string" order: 6 + group: "advanced" ssl: title: "Connect using SSL" description: "Encrypt data using SSL. When activating SSL, please select\ @@ -442,12 +450,15 @@ type: "boolean" default: false order: 7 + group: "security" + always_show: true ssl_mode: title: "SSL Modes" description: "SSL connection modes. \n Read more in the docs." type: "object" - order: 7 + order: 8 + group: "security" oneOf: - title: "disable" additionalProperties: true @@ -576,7 +587,8 @@ type: "object" title: "Replication Method" description: "Replication method for extracting data from the database." - order: 8 + order: 9 + group: "advanced" oneOf: - title: "Standard" description: "Standard replication requires no setup on the DB side but\ @@ -753,6 +765,14 @@ type: "string" airbyte_secret: true order: 4 + group: "security" + groups: + - id: "db" + - id: "auth" + - id: "security" + title: "Security" + - id: "advanced" + title: "Advanced" supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] @@ -2326,7 +2346,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-clickhouse:0.1.16" +- dockerImage: "airbyte/source-clickhouse:0.1.17" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/clickhouse" connectionSpecification: @@ -2625,7 +2645,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-cockroachdb:0.1.21" +- dockerImage: "airbyte/source-cockroachdb:0.1.22" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/cockroachdb" connectionSpecification: @@ -6880,7 +6900,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-db2:0.1.18" +- dockerImage: "airbyte/source-db2:0.1.19" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/db2" connectionSpecification: @@ -8461,7 +8481,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:1.0.5" +- dockerImage: "airbyte/source-mssql:1.0.6" spec: documentationUrl: "https://docs.airbyte.com/integrations/destinations/mssql" connectionSpecification: @@ -9336,7 +9356,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:2.0.7" +- dockerImage: "airbyte/source-mysql:2.0.8" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/mysql" connectionSpecification: @@ -10690,7 +10710,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-oracle:0.3.23" +- dockerImage: "airbyte/source-oracle:0.3.24" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/oracle" connectionSpecification: @@ -11912,7 +11932,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:2.0.8" +- dockerImage: "airbyte/source-postgres:2.0.9" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: @@ -14405,7 +14425,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-snowflake:0.1.31" +- dockerImage: "airbyte/source-snowflake:0.1.32" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/snowflake" connectionSpecification: @@ -15261,7 +15281,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tidb:0.2.3" +- dockerImage: "airbyte/source-tidb:0.2.4" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/tidb" connectionSpecification: diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index 079b69ea81bf..33ed0f45688c 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -5,7 +5,6 @@ package io.airbyte.db.jdbc; import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -21,11 +20,7 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.text.ParseException; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.OffsetDateTime; -import java.time.OffsetTime; +import java.time.*; import java.time.chrono.IsoEra; import java.time.format.DateTimeParseException; import java.util.Collections; @@ -123,7 +118,7 @@ protected void putString(final ObjectNode node, final String columnName, final R } protected void putDate(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DateTimeConverter.convertToDate(getObject(resultSet, index, LocalDate.class))); + node.put(columnName, resultSet.getString(index)); } protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { @@ -131,9 +126,13 @@ protected void putTime(final ObjectNode node, final String columnName, final Res } protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - // https://www.cis.upenn.edu/~bcpierce/courses/629/jdkdocs/guide/jdbc/getstart/mapping.doc.html - final Instant instant = resultSet.getTimestamp(index).toInstant(); - node.put(columnName, DataTypeUtils.toISO8601StringWithMicroseconds(instant)); + try { + node.put(columnName, DateTimeConverter.convertToTimestamp(getObject(resultSet, index, LocalDateTime.class))); + } catch (Exception e) { + // for backward compatibility + final Instant instant = resultSet.getTimestamp(index).toInstant(); + node.put(columnName, DataTypeUtils.toISO8601StringWithMicroseconds(instant)); + } } protected void putBinary(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { @@ -153,29 +152,10 @@ protected void setTime(final PreparedStatement preparedStatement, final int para } protected void setTimestamp(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - // parse time, and timestamp the same way. this seems to not cause an problems and allows us - // to treat them all as ISO8601. if this causes any problems down the line, we can adjust. - // Parsing TIME as a TIMESTAMP might potentially break for ClickHouse cause it doesn't expect TIME - // value in the following format try { - var valueWithoutMicros = value; - final StringBuilder nanos = new StringBuilder(); - final var dotIndex = value.indexOf("."); - if (dotIndex > 0) { - final var micro = value.substring(value.lastIndexOf('.') + 1, value.length() - 1); - nanos.append(micro); - valueWithoutMicros = value.replace("." + micro, ""); - } - while (nanos.length() != 9) { - nanos.append("0"); - } - - final var timestamp = Timestamp - .from(DataTypeUtils.getDateFormat().parse(valueWithoutMicros).toInstant()); - timestamp.setNanos(Integer.parseInt(nanos.toString())); - preparedStatement.setTimestamp(parameterIndex, timestamp); - } catch (final ParseException e) { - throw new RuntimeException(e); + preparedStatement.setObject(parameterIndex, LocalDateTime.parse(value)); + } catch (final DateTimeParseException e) { + preparedStatement.setObject(parameterIndex, OffsetDateTime.parse(value)); } } @@ -246,7 +226,7 @@ protected ObjectType getObject(final ResultSet resultSet, final int protected void putTimeWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { final OffsetTime timetz = getObject(resultSet, index, OffsetTime.class); - node.put(columnName, timetz.format(TIMETZ_FORMATTER)); + node.put(columnName, DateTimeConverter.convertToTimeWithTimezone(timetz)); } protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java index 736e1b007c8e..98d08a2ea7c4 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java @@ -44,14 +44,14 @@ public class DateTimeConverter { public static String convertToTimeWithTimezone(final Object time) { if (time instanceof final java.time.OffsetTime timetz) { - return timetz.format(TIMETZ_FORMATTER); + return hasZeroSecondsAndNanos(timetz.toLocalTime()) ? timetz.format(TIMETZ_FORMATTER) : timetz.toString(); } else { if (!loggedUnknownTimeWithTimeZoneClass) { LOGGER.info("Unknown class for Time with timezone data type" + time.getClass()); loggedUnknownTimeWithTimeZoneClass = true; } final OffsetTime timetz = OffsetTime.parse(time.toString(), TIME_WITH_TIMEZONE_FORMATTER); - return timetz.format(TIMETZ_FORMATTER); + return hasZeroSecondsAndNanos(timetz.toLocalTime()) ? timetz.format(TIMETZ_FORMATTER) : timetz.toString(); } } @@ -98,15 +98,15 @@ public static String convertToTimestamp(final Object timestamp) { if (timestamp instanceof final Timestamp t) { // Snapshot mode final LocalDateTime localDateTime = t.toLocalDateTime(); - final String value = localDateTime.format(TIMESTAMP_FORMATTER); - return resolveEra(t, value); + return resolveEra(t, + hasZeroSecondsAndNanos(localDateTime.toLocalTime()) ? localDateTime.format(TIMESTAMP_FORMATTER) : localDateTime.toString()); } else if (timestamp instanceof final Instant i) { // Incremental mode return resolveEra(i.atZone(UTC).toLocalDate(), i.atOffset(UTC).toLocalDateTime().format(TIMESTAMP_FORMATTER)); } else if (timestamp instanceof final LocalDateTime localDateTime) { final LocalDate date = localDateTime.toLocalDate(); - final String value = localDateTime.format(TIMESTAMP_FORMATTER); - return resolveEra(date, value); + return resolveEra(date, + hasZeroSecondsAndNanos(localDateTime.toLocalTime()) ? localDateTime.format(TIMESTAMP_FORMATTER) : localDateTime.toString()); } else { if (!loggedUnknownTimestampClass) { LOGGER.info("Unknown class for Timestamp data type" + timestamp.getClass()); @@ -114,8 +114,8 @@ public static String convertToTimestamp(final Object timestamp) { } final LocalDateTime localDateTime = LocalDateTime.parse(timestamp.toString()); final LocalDate date = localDateTime.toLocalDate(); - final String value = localDateTime.format(TIMESTAMP_FORMATTER); - return resolveEra(date, value); + return resolveEra(date, + hasZeroSecondsAndNanos(localDateTime.toLocalTime()) ? localDateTime.format(TIMESTAMP_FORMATTER) : localDateTime.toString()); } } @@ -143,18 +143,18 @@ public static String convertToDate(final Object date) { public static String convertToTime(final Object time) { if (time instanceof final Time sqlTime) { - return sqlTime.toLocalTime().format(TIME_FORMATTER); + return formatTime(sqlTime.toLocalTime()); } else if (time instanceof final LocalTime localTime) { - return localTime.format(TIME_FORMATTER); + return formatTime(localTime); } else if (time instanceof java.time.Duration) { long value = ((Duration) time).toNanos(); if (value >= 0 && value < TimeUnit.DAYS.toNanos(1)) { - return LocalTime.ofNanoOfDay(value).format(TIME_FORMATTER); + return formatTime(LocalTime.ofNanoOfDay(value)); } else { final long updatedValue = Math.min(Math.abs(value), LocalTime.MAX.toNanoOfDay()); LOGGER.debug("Time values must use number of nanoseconds greater than 0 and less than 86400000000000 but its {}, converting to {} ", value, updatedValue); - return LocalTime.ofNanoOfDay(updatedValue).format(TIME_FORMATTER); + return formatTime(LocalTime.ofNanoOfDay(updatedValue)); } } else { if (!loggedUnknownTimeClass) { @@ -165,12 +165,20 @@ public static String convertToTime(final Object time) { final String valueAsString = time.toString(); if (valueAsString.startsWith("24")) { LOGGER.debug("Time value {} is above range, converting to 23:59:59", valueAsString); - return LocalTime.MAX.format(TIME_FORMATTER); + return LocalTime.MAX.toString(); } - return LocalTime.parse(valueAsString).format(TIME_FORMATTER); + return formatTime(LocalTime.parse(valueAsString)); } } + private static String formatTime(LocalTime localTime) { + return hasZeroSecondsAndNanos(localTime) ? localTime.format(TIME_FORMATTER) : localTime.toString(); + } + + public static boolean hasZeroSecondsAndNanos(LocalTime localTime) { + return (localTime.getSecond() == 0 && localTime.getNano() == 0); + } + public static void putJavaSQLDate(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { final Date date = resultSet.getDate(index); node.put(columnName, convertToDate(date)); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java index d8f68e938908..4c2ffd9dec13 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java @@ -167,7 +167,7 @@ void testSetStatementField() throws SQLException { sourceOperations.setCursorField(ps, 12, JDBCType.VARCHAR, "a"); sourceOperations.setCursorField(ps, 13, JDBCType.DATE, "2020-11-01"); sourceOperations.setCursorField(ps, 14, JDBCType.TIME, "05:00:00.000"); - sourceOperations.setCursorField(ps, 15, JDBCType.TIMESTAMP, "2001-09-29T03:00:00.000Z"); + sourceOperations.setCursorField(ps, 15, JDBCType.TIMESTAMP, "2001-09-29T03:00:00.000"); sourceOperations.setCursorField(ps, 16, JDBCType.BINARY, "61616161"); ps.execute(); @@ -395,7 +395,7 @@ private ObjectNode expectedValues() { expected.put("varchar", "a"); expected.put("date", "2020-11-01"); expected.put("time", "05:00:00.000000"); - expected.put("timestamp", "2001-09-29T03:00:00.000000Z"); + expected.put("timestamp", "2001-09-29T03:00:00.000000"); expected.put("binary1", "aaaa".getBytes(Charsets.UTF_8)); return expected; } diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java index a364b2dbbaa6..875434a287d9 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/MSSQLConverter.java @@ -111,7 +111,7 @@ private void registerDate(final RelationalColumn field, if (field.typeName().equalsIgnoreCase("DATE")) { return DateTimeConverter.convertToDate(input); } - return DebeziumConverterUtils.convertDate(input); + return DateTimeConverter.convertToTimestamp(input); }); } diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java index 25113b631dc9..ae2c259c629d 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/PostgresConverter.java @@ -4,7 +4,6 @@ package io.airbyte.integrations.debezium.internals; -import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; import static io.airbyte.db.jdbc.DateTimeConverter.convertToDate; import static io.airbyte.db.jdbc.DateTimeConverter.convertToTime; import static io.airbyte.db.jdbc.DateTimeConverter.convertToTimestamp; @@ -186,7 +185,7 @@ private Object convertArray(final Object x, final RelationalColumn field) { timetzArr.add(null); } else { final OffsetTime parsed = OffsetTime.parse(s, format); - timetzArr.add(parsed.format(TIMETZ_FORMATTER)); + timetzArr.add(DateTimeConverter.convertToTimeWithTimezone(parsed)); } }); return timetzArr; diff --git a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java b/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java deleted file mode 100644 index 6fdc8f1592d7..000000000000 --- a/airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtilsTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.jdbc; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.sql.SQLException; -import java.time.Instant; -import java.util.List; -import java.util.UUID; -import java.util.function.Supplier; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; - -class SqlOperationsUtilsTest { - - private static final Instant NOW = Instant.ofEpochSecond(900); - private static final String SCHEMA_NAME = "public"; - private static final String STREAM_NAME = "rivers"; - - private PostgreSQLContainer container; - private JdbcDatabase database; - private Supplier uuidSupplier; - - @SuppressWarnings("unchecked") - @BeforeEach - void setup() { - container = new PostgreSQLContainer<>("postgres:13-alpine"); - container.start(); - - final JsonNode config = createConfig(); - - database = new DefaultJdbcDatabase( - DataSourceFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - config.get(JdbcUtils.JDBC_URL_KEY).asText())); - - uuidSupplier = mock(Supplier.class); - } - - @Test - void testInsertRawRecordsInSingleQuery() throws SQLException { - final UUID RECORD1_UUID = UUID.randomUUID(); - final UUID RECORD2_UUID = UUID.randomUUID(); - when(uuidSupplier.get()).thenReturn(RECORD1_UUID).thenReturn(RECORD2_UUID); - - new TestJdbcSqlOperations().createTableIfNotExists(database, SCHEMA_NAME, STREAM_NAME); - - final String insertQueryComponent = String.format( - "INSERT INTO %s.%s (%s, %s, %s) VALUES\n", - SCHEMA_NAME, - STREAM_NAME, - JavaBaseConstants.COLUMN_NAME_AB_ID, - JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - final String recordQueryComponent = "(?, ?::jsonb, ?),\n"; - - final List records = Lists.newArrayList( - new AirbyteRecordMessage() - .withStream("rivers") - .withEmittedAt(NOW.toEpochMilli()) - .withData(Jsons.jsonNode(ImmutableMap.of("name", "rio grande", "width", 10))), - new AirbyteRecordMessage() - .withStream("rivers") - .withEmittedAt(NOW.toEpochMilli()) - .withData(Jsons.jsonNode(ImmutableMap.of("name", "mississippi", "width", 20)))); - - SqlOperationsUtils.insertRawRecordsInSingleQuery(insertQueryComponent, recordQueryComponent, database, records, uuidSupplier, true); - - final List actualRecords = database.bufferedResultSetQuery( - connection -> connection.createStatement().executeQuery("SELECT * FROM RIVERS"), - JdbcUtils.getDefaultSourceOperations()::rowToJson); - - final List expectedRecords = Lists.newArrayList( - Jsons.jsonNode(ImmutableMap.builder() - .put(JavaBaseConstants.COLUMN_NAME_AB_ID, RECORD1_UUID) - .put(JavaBaseConstants.COLUMN_NAME_DATA, records.get(0).getData()) - .put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, DataTypeUtils - .toISO8601StringWithMicroseconds(Instant.ofEpochMilli(records.get(0).getEmittedAt()))) - .build()), - Jsons.jsonNode(ImmutableMap.builder() - .put(JavaBaseConstants.COLUMN_NAME_AB_ID, RECORD2_UUID) - .put(JavaBaseConstants.COLUMN_NAME_DATA, records.get(1).getData()) - .put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, DataTypeUtils - .toISO8601StringWithMicroseconds(Instant.ofEpochMilli(records.get(1).getEmittedAt()))) - .build())); - - actualRecords.forEach( - r -> ((ObjectNode) r).put(JavaBaseConstants.COLUMN_NAME_DATA, Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText()))); - - assertEquals(expectedRecords, actualRecords); - } - - private JsonNode createConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, SCHEMA_NAME) - .put(JdbcUtils.JDBC_URL_KEY, String.format("jdbc:postgresql://%s:%s/%s", - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName())) - .build()); - } - -} diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile index 43584c2e8526..86f36fcb0b3b 100644 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.6 +LABEL io.airbyte.version=2.0.9 LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile index 5fb54c2519e7..4d285e9a7541 100644 --- a/airbyte-integrations/connectors/source-alloydb/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.6 +LABEL io.airbyte.version=2.0.9 LABEL io.airbyte.name=airbyte/source-alloydb diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index 32c8f3e939db..63e085101334 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile index 35c40b88a08f..2bdaa7dedb37 100644 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-clickhouse diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile index d2bd3f9bcf08..422957b9bb8f 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.21 +LABEL io.airbyte.version=0.1.22 LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile index 8a14e561eabf..a9cc6d8b9de1 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.21 +LABEL io.airbyte.version=0.1.22 LABEL io.airbyte.name=airbyte/source-cockroachdb diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index d6233b981cdb..e1c1bb66ddbc 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -17,6 +17,7 @@ import io.airbyte.protocol.models.JsonSchemaType; import java.sql.SQLException; import java.util.Objects; +import java.util.Set; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.slf4j.Logger; @@ -211,13 +212,15 @@ protected void initTests() { .addExpectedValues("a ", "abc ", "Миші йдуть; ", "櫻花分店 ", " ", null) .build()); + for (final String type : Set.of("date", "date not null default '0000-00-00'")) { + } addDataTypeTestData( TestDataHolder.builder() .sourceType("date") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'1999-01-08'", "null") - .addExpectedValues("1999-01-08", null) + .addInsertValues("'1999-01-08'", "null", "'16-12-06'") + .addExpectedValues("1999-01-08", null, "2016-12-06") .build()); addDataTypeTestData( @@ -318,7 +321,7 @@ protected void initTests() { .sourceType("time") .airbyteType(JsonSchemaType.STRING) .addInsertValues("'04:05:06'", null) - .addExpectedValues("04:05:06.000000") + .addExpectedValues("04:05:06") .addNullExpectedValue() .build()); @@ -337,7 +340,7 @@ protected void initTests() { .sourceType("timestamp") .airbyteType(JsonSchemaType.STRING) .addInsertValues("TIMESTAMP '2004-10-19 10:23:54'", "TIMESTAMP '2004-10-19 10:23:54.123456'", "null") - .addExpectedValues("2004-10-19T10:23:54.000000Z", "2004-10-19T10:23:54.123456Z", null) + .addExpectedValues("2004-10-19T10:23:54", "2004-10-19T10:23:54.123456", null) .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile index e9ce890464d2..7c576094b47c 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile index 8ec651e4eff9..019f26e4c4f5 100644 --- a/airbyte-integrations/connectors/source-db2/Dockerfile +++ b/airbyte-integrations/connectors/source-db2/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/source-db2 diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java index a9f2e761adf6..bd5cc47b5132 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java @@ -4,18 +4,20 @@ package io.airbyte.integrations.source.db2; -import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLDate; import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; +import io.airbyte.db.jdbc.DateTimeConverter; import io.airbyte.db.jdbc.JdbcSourceOperations; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.Collections; import java.util.List; import org.slf4j.Logger; @@ -76,21 +78,24 @@ private void putDecfloat(final ObjectNode node, } @Override - protected void putDate(final ObjectNode node, + protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - putJavaSQLDate(node, columnName, resultSet, index); + putJavaSQLTime(node, columnName, resultSet, index); } @Override - protected void putTime(final ObjectNode node, - final String columnName, - final ResultSet resultSet, - final int index) - throws SQLException { - putJavaSQLTime(node, columnName, resultSet, index); + protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + final Timestamp timestamp = resultSet.getTimestamp(index); + node.put(columnName, DateTimeConverter.convertToTimestamp(timestamp)); + } + + @Override + protected void setTimestamp(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { + final LocalDateTime date = LocalDateTime.parse(value); + preparedStatement.setTimestamp(parameterIndex, Timestamp.valueOf(date)); } @Override diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index 3b0d7679fe6c..0e4dbcfde53e 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -311,7 +311,7 @@ protected void initTests() { .sourceType("TIME") .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'00.00.00'", "'1:59 PM'", "'23.59.59'") - .addExpectedValues(null, "00:00:00.000000", "13:59:00.000000", "23:59:59.000000") + .addExpectedValues(null, "00:00:00.000000", "13:59:00.000000", "23:59:59") .build()); addDataTypeTestData( TestDataHolder.builder() @@ -319,8 +319,8 @@ protected void initTests() { .sourceType("TIMESTAMP") .airbyteType(JsonSchemaType.STRING) .addInsertValues("null", "'2018-03-22-12.00.00.123'", "'2018-03-22-12.00.00.123456'", "'20180322125959'", "'20180101 12:00:59 PM'") - .addExpectedValues(null, "2018-03-22T12:00:00.123000Z", "2018-03-22T12:00:00.123456Z", "2018-03-22T12:59:59.000000Z", - "2018-01-01T12:00:59.000000Z") + .addExpectedValues(null, "2018-03-22T12:00:00.123", "2018-03-22T12:00:00.123456", "2018-03-22T12:59:59", + "2018-01-01T12:00:59") .build()); } diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index 7e1dbb089a8f..22bceacb7354 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.5 +LABEL io.airbyte.version=1.0.6 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 107aaf439a6b..11680a73522a 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.5 +LABEL io.airbyte.version=1.0.6 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java index d254d2525f63..95728cfc2e7e 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java @@ -15,7 +15,6 @@ import com.microsoft.sqlserver.jdbc.Geography; import com.microsoft.sqlserver.jdbc.Geometry; import com.microsoft.sqlserver.jdbc.SQLServerResultSetMetaData; -import io.airbyte.db.DataTypeUtils; import io.airbyte.db.jdbc.JdbcSourceOperations; import java.nio.charset.Charset; import java.sql.JDBCType; @@ -111,11 +110,6 @@ protected void putBinary(final ObjectNode node, node.put(columnName, value); } - @Override - protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DataTypeUtils.toISOTimeString(resultSet.getTimestamp(index).toLocalDateTime())); - } - protected void putGeometry(final ObjectNode node, final String columnName, final ResultSet resultSet, diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java index 42ecf836b932..902f9bd39735 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java @@ -153,7 +153,7 @@ protected void initTests() { .sourceType("smalldatetime") .airbyteType(JsonSchemaType.STRING) .addInsertValues("'1900-01-01'", "'2079-06-06'", "null") - .addExpectedValues("1900-01-01T00:00:00.000000Z", "2079-06-06T00:00:00.000000Z", null) + .addExpectedValues("1900-01-01T00:00:00.000000", "2079-06-06T00:00:00.000000", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); @@ -161,10 +161,10 @@ protected void initTests() { TestDataHolder.builder() .sourceType("datetime") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'1753-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04Z'", - "'9999-12-31T13:00:04.123Z'", "null") - .addExpectedValues("1753-01-01T00:00:00.000000Z", "9999-12-31T00:00:00.000000Z", "9999-12-31T13:00:04.000000Z", - "9999-12-31T13:00:04.123000Z", null) + .addInsertValues("'1753-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04'", + "'9999-12-31T13:00:04.123'", "null") + .addExpectedValues("1753-01-01T00:00:00.000000", "9999-12-31T00:00:00.000000", "9999-12-31T13:00:04", + "9999-12-31T13:00:04.123", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); @@ -172,8 +172,8 @@ protected void initTests() { TestDataHolder.builder() .sourceType("datetime2") .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'0001-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04.123456Z'", "null") - .addExpectedValues("0001-01-01T00:00:00.000000Z", "9999-12-31T00:00:00.000000Z", "9999-12-31T13:00:04.123456Z", null) + .addInsertValues("'0001-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04.123456'", "null") + .addExpectedValues("0001-01-01T00:00:00.000000", "9999-12-31T00:00:00.000000", "9999-12-31T13:00:04.123456", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index 4f727944f972..44c23fca5881 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.7 +LABEL io.airbyte.version=2.0.8 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index a09529e1be4d..a555f9d8a57a 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,6 +16,6 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.7 +LABEL io.airbyte.version=2.0.8 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java index b32410d609d6..77029d12f1fd 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java @@ -46,7 +46,6 @@ import io.airbyte.db.DataTypeUtils; import io.airbyte.db.SourceOperations; import io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations; -import io.airbyte.db.jdbc.DateTimeConverter; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -211,11 +210,6 @@ public boolean isCursorType(final MysqlType type) { return ALLOWED_CURSOR_TYPES.contains(type); } - @Override - protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DateTimeConverter.convertToTimestamp(getObject(resultSet, index, LocalDateTime.class))); - } - @Override public JsonSchemaType getAirbyteType(final MysqlType mysqlType) { return switch (mysqlType) { diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java index ca1326d9bdbb..3b64dd3229a5 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java @@ -247,8 +247,8 @@ protected void initTests() { .sourceType("date") .fullSourceDataType(type) .airbyteType(JsonSchemaType.STRING_DATE) - .addInsertValues("'1999-01-08'", "'2021-01-01'") - .addExpectedValues("1999-01-08", "2021-01-01") + .addInsertValues("'1999-01-08'", "'2021-01-01'", "'2022/11/12'", "'1987.12.01'") + .addExpectedValues("1999-01-08", "2021-01-01", "2022-11-12", "1987-12-01") .build()); } @@ -267,7 +267,7 @@ protected void initTests() { .fullSourceDataType(fullSourceType) .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) .addInsertValues("'2005-10-10 23:22:21'", "'2013-09-05T10:10:02'", "'2013-09-06T10:10:02'") - .addExpectedValues("2005-10-10T23:22:21.000000", "2013-09-05T10:10:02.000000", "2013-09-06T10:10:02.000000") + .addExpectedValues("2005-10-10T23:22:21", "2013-09-05T10:10:02", "2013-09-06T10:10:02") .build()); } @@ -289,7 +289,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE) // JDBC driver can process only "clock"(00:00:00-23:59:59) values. .addInsertValues("'-22:59:59'", "'23:59:59'", "'00:00:00'") - .addExpectedValues("22:59:59.000000", "23:59:59.000000", "00:00:00.000000") + .addExpectedValues("22:59:59", "23:59:59", "00:00:00.000000") .build()); } diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile index b8acc0fcf786..33ea624423ec 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile @@ -17,5 +17,5 @@ ENV TZ UTC COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.23 +LABEL io.airbyte.version=0.3.24 LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index 051f0dd494b6..daab4ac76e0e 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -8,5 +8,5 @@ ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.23 +LABEL io.airbyte.version=0.3.24 LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java index c3559f50ce8b..b6d73e87a600 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java @@ -127,8 +127,8 @@ protected void incrementalDateCheck() throws Exception { // standard) is concerned as it has both a date and time component. incrementalCursorCheck( COL_UPDATED_AT, - "2005-10-18T00:00:00.000000Z", - "2006-10-19T00:00:00.000000Z", + "2005-10-18T00:00:00.000000", + "2006-10-19T00:00:00.000000", Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); } @@ -202,7 +202,7 @@ protected List getTestMessages() { .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19T00:00:00.000000Z")))), + COL_UPDATED_AT, "2004-10-19T00:00:00.000000")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName) .withNamespace(getDefaultNamespace()) @@ -210,22 +210,22 @@ protected List getTestMessages() { .of(COL_ID, ID_VALUE_2, COL_NAME, "crusher", COL_UPDATED_AT, - "2005-10-19T00:00:00.000000Z")))), + "2005-10-19T00:00:00.000000")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_3, COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); } @Test void testIncrementalTimestampCheckCursor() throws Exception { incrementalCursorCheck( COL_UPDATED_AT, - "2005-10-18T00:00:00.000000Z", - "2006-10-19T00:00:00.000000Z", + "2005-10-18T00:00:00.000000", + "2006-10-19T00:00:00.000000", Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); } @@ -269,13 +269,13 @@ void testReadOneTableIncrementallyTwice() throws Exception { .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); expectedMessages.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index e029b6ef377a..5939a62bf5a9 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -203,7 +203,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.STRING) .addInsertValues("to_date('-4700/01/01','syyyy/mm/dd')", "to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss')", "null") - .addExpectedValues("4700-01-01T00:00:00.000000Z", "9999-12-31T23:59:59.000000Z", null) + .addExpectedValues("4700-01-01T00:00:00.000000 BC", "9999-12-31T23:59:59", null) // @TODO stream fails when gets Zero date value // .addInsertValues("'2021/01/00'", "'2021/00/00'", "'0000/00/00'") .build()); @@ -214,7 +214,7 @@ protected void initTests() { .airbyteType(JsonSchemaType.STRING) .addInsertValues("to_timestamp('2020-06-10 06:14:00.742', 'YYYY-MM-DD HH24:MI:SS.FF')", "to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF')") - .addExpectedValues("2020-06-10T06:14:00.742000Z", "2020-06-10T06:14:00.742123Z") + .addExpectedValues("2020-06-10T06:14:00.742", "2020-06-10T06:14:00.742123") .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 9fb1be84a92c..3053e6cd2d54 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.8 +LABEL io.airbyte.version=2.0.9 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 783fa6f2b8dd..512a1dce857a 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.8 +LABEL io.airbyte.version=2.0.9 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index 989f69ab105c..ed7fa3e1bcb5 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -5,7 +5,6 @@ package io.airbyte.integrations.source.postgres; import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; @@ -34,6 +33,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Timestamp; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -231,7 +231,7 @@ private void putTimeTzArray(final ObjectNode node, final String columnName, fina if (timetz == null) { arrayNode.add(NullNode.getInstance()); } else { - arrayNode.add(timetz.format(TIMETZ_FORMATTER)); + arrayNode.add(DateTimeConverter.convertToTimeWithTimezone(timetz)); } } node.set(columnName, arrayNode); @@ -241,7 +241,7 @@ private void putTimestampArray(final ObjectNode node, final String columnName, f final ArrayNode arrayNode = Jsons.arrayNode(); final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); while (arrayResultSet.next()) { - final LocalDateTime timestamp = getObject(arrayResultSet, 2, LocalDateTime.class); + final Timestamp timestamp = arrayResultSet.getTimestamp(2); if (timestamp == null) { arrayNode.add(NullNode.getInstance()); } else { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java index d86693c15ffb..1a7a79685aed 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java @@ -189,8 +189,8 @@ protected void initTests() { .sourceType("date") .fullSourceDataType(type) .airbyteType(JsonSchemaType.STRING_DATE) - .addInsertValues("'1999-01-08'", "'1991-02-10 BC'") - .addExpectedValues("1999-01-08", "1991-02-10 BC") + .addInsertValues("'1999-01-08'", "'1991-02-10 BC'", "'2022/11/12'", "'1987.12.01'") + .addExpectedValues("1999-01-08", "1991-02-10 BC", "2022-11-12", "1987-12-01") .build()); } @@ -389,9 +389,10 @@ protected void initTests() { .fullSourceDataType(fullSourceType) .airbyteType(JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE) // time column will ignore time zone - .addInsertValues("'13:00:01'", "'13:00:02+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.01234Z+8'", "'13:00:00Z-8'", "'24:00:00'") - .addExpectedValues("13:00:01.000000", "13:00:02.000000", "13:00:03.000000", "13:00:04.000000", "13:00:05.012340", - "13:00:00.000000", "23:59:59.999999") + .addInsertValues("'13:00:01.010'", "'13:00:02.000001+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.01234Z+8'", "'13:00:00Z-8'", + "'24:00:00'") + .addExpectedValues("13:00:01.010", "13:00:02.000001", "13:00:03", "13:00:04", "13:00:05.012340", + "13:00:00.000000", "23:59:59.999999999") .build()); } @@ -591,11 +592,12 @@ protected void addTimeWithTimeZoneTest() { .sourceType("timetz") .fullSourceDataType(fullSourceType) .airbyteType(JsonSchemaType.STRING_TIME_WITH_TIMEZONE) - .addInsertValues("null", "'13:00:01'", "'13:00:00+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.012345Z+8'", "'13:00:06.00000Z-8'") + .addInsertValues("null", "'13:00:01.123456'", "'13:00:01+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.012345Z+8'", + "'13:00:06.00000Z-8'") // A time value without time zone will use the time zone set on the database, which is Z-7, // so 13:00:01 is returned as 13:00:01-07. - .addExpectedValues(null, "13:00:01.000000-07:00", "13:00:00.000000+08:00", "13:00:03.000000-08:00", "13:00:04.000000Z", - "13:00:05.012345-08:00", "13:00:06.000000+08:00") + .addExpectedValues(null, "13:00:01.123456-07:00", "13:00:01+08:00", "13:00:03-08:00", "13:00:04Z", + "13:00:05.012345-08:00", "13:00:06+08:00") .build()); } } @@ -842,7 +844,7 @@ private void addArraysTestData() { .build()) .addInsertValues("'{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}'") .addExpectedValues( - "[\"13:00:01.000000\",\"13:00:02.000000\",\"13:00:03.000000\",\"13:00:04.000000\",\"13:00:05.000000\",\"13:00:00.000000\"]") + "[\"13:00:01\",\"13:00:02\",\"13:00:03\",\"13:00:04\",\"13:00:05\",\"13:00:00.000000\"]") .build()); addDataTypeTestData( @@ -854,7 +856,7 @@ private void addArraysTestData() { .build()) .addInsertValues("'{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}'") .addExpectedValues( - "[null,\"13:00:01.000000-07:00\",\"13:00:00.000000+08:00\",\"13:00:03.000000-08:00\",\"13:00:04.000000Z\",\"13:00:05.012345-08:00\",\"13:00:06.000000+08:00\",\"13:00:00.000000-07:00\"]") + "[null,\"13:00:01-07:00\",\"13:00:00.000000+08:00\",\"13:00:03-08:00\",\"13:00:04Z\",\"13:00:05.012345-08:00\",\"13:00:06+08:00\",\"13:00:00.000000-07:00\"]") .build()); addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java index 5e69c40d42c2..ef8230a7505c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java @@ -183,8 +183,8 @@ protected void addTimeWithTimeZoneTest() { .addInsertValues("null", "'13:00:01'", "'13:00:00+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.012345Z+8'", "'13:00:06.00000Z-8'") // A time value without time zone will use the time zone set on the database, which is Z-7, // so 13:00:01 is returned as 13:00:01-07. - .addExpectedValues(null, "20:00:01.000000Z", "05:00:00.000000Z", "21:00:03.000000Z", "13:00:04.000000Z", "21:00:05.012345Z", - "05:00:06.000000Z") + .addExpectedValues(null, "20:00:01Z", "05:00:00.000000Z", "21:00:03Z", "13:00:04Z", "21:00:05.012345Z", + "05:00:06Z") .build()); } } diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.jsonl index 28c55a076ada..a1c7c7e6c319 100644 --- a/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.jsonl @@ -1,2 +1,2 @@ -{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31","test_column_11":"2008-06-01T09:59:59.000000Z","test_column_12":"2008-06-01T14:59:59.000000Z","test_column_13":"04:05:06.000000","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 } -{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10","test_column_11":"2005-06-21T12:00:59.000000Z","test_column_12":"2003-10-15T14:59:59.000000Z","test_column_13":"04:05:00.000000","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 } +{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31","test_column_11":"2008-06-01T09:59:59","test_column_12":"2008-06-01T14:59:59","test_column_13":"04:05:06","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 } +{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10","test_column_11":"2005-06-21T12:00:59","test_column_12":"2003-10-15T14:59:59","test_column_13":"04:05:00.000000","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 } diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java index 08022da706fa..ee16ddccf750 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java @@ -4,16 +4,18 @@ package io.airbyte.integrations.source.redshift; -import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLDate; import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.db.jdbc.DateTimeConverter; import io.airbyte.db.jdbc.JdbcSourceOperations; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; import java.time.LocalDate; +import java.time.LocalDateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,21 +24,24 @@ public class RedshiftSourceOperations extends JdbcSourceOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftSourceOperations.class); @Override - protected void putDate(final ObjectNode node, + protected void putTime(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - putJavaSQLDate(node, columnName, resultSet, index); + putJavaSQLTime(node, columnName, resultSet, index); } @Override - protected void putTime(final ObjectNode node, - final String columnName, - final ResultSet resultSet, - final int index) - throws SQLException { - putJavaSQLTime(node, columnName, resultSet, index); + protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + final Timestamp timestamp = resultSet.getTimestamp(index); + node.put(columnName, DateTimeConverter.convertToTimestamp(timestamp)); + } + + @Override + protected void setTimestamp(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { + final LocalDateTime date = LocalDateTime.parse(value); + preparedStatement.setTimestamp(parameterIndex, Timestamp.valueOf(date)); } @Override diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java index 2a2f3e941fc7..2c9be009042f 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java @@ -57,7 +57,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config = getStaticConfig(); database = createDatabase(config); - testUserName = "foo"; + testUserName = Strings.addRandomSuffix("foo", "_", 5).toLowerCase(); testUserPassword = "BarBarBar1&"; createTestUser(database, config, testUserName, testUserPassword); schemaName = Strings.addRandomSuffix("integration_test", "_", 5).toLowerCase(); diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 50f1d0b2f474..8864ebe88cfe 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.31 +LABEL io.airbyte.version=0.1.32 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.jsonl index bb63fbe99f5a..1e489309c367 100644 --- a/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.jsonl @@ -1,5 +1,5 @@ -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 1, "TEST_COLUMN_1" : 99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : -9007199254740990.0, "TEST_COLUMN_12" : 1e-307, "TEST_COLUMN_14" : "\u0442\u0435\u0441\u0442", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-!-", "TEST_COLUMN_17" : "a", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : 9223372036854775807, "TEST_COLUMN_20" : "0001-01-01", "TEST_COLUMN_21" : "0001-01-01T00:00:00.000000", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123000", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123000Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123000", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123000Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n -122.35,\n 37.55\n ],\n \"type\": \"Point\"\n}", "TEST_COLUMN_4" : 99999999999999999999999999999999999999, "TEST_COLUMN_5" : 9223372036854775807, "TEST_COLUMN_6" : 9223372036854775807, "TEST_COLUMN_7" : 9223372036854775807, "TEST_COLUMN_8" : 9223372036854775807, "TEST_COLUMN_9" : 9223372036854775807 }, "emitted_at" : 1670334357227} -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 2, "TEST_COLUMN_1" : -99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "\u26a1 test \ufffd\ufffd", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u30b9", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.000000", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 1, "TEST_COLUMN_1" : 99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : -9007199254740990.0, "TEST_COLUMN_12" : 1e-307, "TEST_COLUMN_14" : "\u0442\u0435\u0441\u0442", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-!-", "TEST_COLUMN_17" : "a", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : 9223372036854775807, "TEST_COLUMN_20" : "0001-01-01", "TEST_COLUMN_21" : "0001-01-01T00:00:00.000000", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123000Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123000Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n -122.35,\n 37.55\n ],\n \"type\": \"Point\"\n}", "TEST_COLUMN_4" : 99999999999999999999999999999999999999, "TEST_COLUMN_5" : 9223372036854775807, "TEST_COLUMN_6" : 9223372036854775807, "TEST_COLUMN_7" : 9223372036854775807, "TEST_COLUMN_8" : 9223372036854775807, "TEST_COLUMN_9" : 9223372036854775807 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 2, "TEST_COLUMN_1" : -99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "\u26a1 test \ufffd\ufffd", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u30b9", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} { "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 3, "TEST_COLUMN_1" : 9223372036854775807, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "!", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 9223372036854775807, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} { "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 4, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} { "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 5, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java index fd45cb3d29a8..5b4268803e24 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java @@ -4,7 +4,6 @@ package io.airbyte.integrations.source.snowflake; -import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLDate; import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; @@ -126,15 +125,6 @@ protected void putTimestamp(final ObjectNode node, final String columnName, fina node.put(columnName, DateTimeConverter.convertToTimestamp(timestamp)); } - @Override - protected void putDate(final ObjectNode node, - final String columnName, - final ResultSet resultSet, - final int index) - throws SQLException { - putJavaSQLDate(node, columnName, resultSet, index); - } - @Override protected void putTime(final ObjectNode node, final String columnName, diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java index 24e57d1786be..e2521c7290ed 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java @@ -263,15 +263,15 @@ protected void initTests() { TestDataHolder.builder() .sourceType("DATE") .airbyteType(JsonSchemaType.STRING_DATE) - .addInsertValues("null", "'0001-01-01'", "'9999-12-31'") - .addExpectedValues(null, "0001-01-01", "9999-12-31") + .addInsertValues("null", "'0001-01-01'", "'9999-12-31'", "'12-JAN-2022'") + .addExpectedValues(null, "0001-01-01", "9999-12-31", "2022-01-12") .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("DATETIME") .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) .addInsertValues("null", "'0001-01-01 00:00:00'", "'9999-12-31 23:59:59'", "'9999-12-31 23:59:59.123456'") - .addExpectedValues(null, "0001-01-01T00:00:00.000000", "9999-12-31T23:59:59.000000", "9999-12-31T23:59:59.123456") + .addExpectedValues(null, "0001-01-01T00:00:00.000000", "9999-12-31T23:59:59", "9999-12-31T23:59:59.123456") .build()); addDataTypeTestData( TestDataHolder.builder() @@ -286,7 +286,7 @@ protected void initTests() { .sourceType("TIMESTAMP") .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) .addInsertValues("null", "'2018-03-22 12:00:00.123'", "'2018-03-22 12:00:00.123456'") - .addExpectedValues(null, "2018-03-22T12:00:00.123000", "2018-03-22T12:00:00.123456") + .addExpectedValues(null, "2018-03-22T12:00:00.123", "2018-03-22T12:00:00.123456") .build()); addDataTypeTestData( TestDataHolder.builder() @@ -300,7 +300,7 @@ protected void initTests() { .sourceType("TIMESTAMP_NTZ") .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'", "'2018-03-22 12:00:00.123456 +05:00'") - .addExpectedValues(null, "2018-03-22T12:00:00.123000", "2018-03-22T12:00:00.123456") + .addExpectedValues(null, "2018-03-22T12:00:00.123", "2018-03-22T12:00:00.123456") .build()); addDataTypeTestData( TestDataHolder.builder() diff --git a/airbyte-integrations/connectors/source-tidb/Dockerfile b/airbyte-integrations/connectors/source-tidb/Dockerfile index 5b3c97096bfc..9f8734640fbb 100755 --- a/airbyte-integrations/connectors/source-tidb/Dockerfile +++ b/airbyte-integrations/connectors/source-tidb/Dockerfile @@ -17,5 +17,5 @@ ENV APPLICATION source-tidb COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/source-tidb diff --git a/connectors.md b/connectors.md index 2efe4fa39612..ad0ae9dd24a8 100644 --- a/connectors.md +++ b/connectors.md @@ -9,7 +9,7 @@ | **Adjust** | Adjust icon | Source | airbyte/source-adjust:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/adjust) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-adjust) | `d3b7fa46-111b-419a-998a-d7f046f6d66d` | | **Aha** | Aha icon | Source | airbyte/source-aha:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/aha) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-aha) | `81ca39dc-4534-4dd2-b848-b0cfd2c11fce` | | **Airtable** | Airtable icon | Source | airbyte/source-airtable:3.0.0 | beta | [link](https://docs.airbyte.com/integrations/sources/airtable) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-airtable) | `14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212` | -| **AlloyDB for PostgreSQL** | AlloyDB for PostgreSQL icon | Source | airbyte/source-alloydb:2.0.6 | generally_available | [link](https://docs.airbyte.com/integrations/sources/alloydb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-alloydb) | `1fa90628-2b9e-11ed-a261-0242ac120002` | +| **AlloyDB for PostgreSQL** | AlloyDB for PostgreSQL icon | Source | airbyte/source-alloydb:2.0.9 | generally_available | [link](https://docs.airbyte.com/integrations/sources/alloydb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-alloydb) | `1fa90628-2b9e-11ed-a261-0242ac120002` | | **Alpha Vantage** | Alpha Vantage icon | Source | airbyte/source-alpha-vantage:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/alpha-vantage) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-alpha-vantage) | `db385323-9333-4fec-bec3-9e0ca9326c90` | | **Amazon Ads** | Amazon Ads icon | Source | airbyte/source-amazon-ads:1.0.1 | generally_available | [link](https://docs.airbyte.com/integrations/sources/amazon-ads) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-ads) | `c6b0a29e-1da9-4512-9002-7bfd0cba2246` | | **Amazon SQS** | Amazon SQS icon | Source | airbyte/source-amazon-sqs:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/amazon-sqs) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-amazon-sqs) | `983fd355-6bf3-4709-91b5-37afa391eeb6` | @@ -36,11 +36,11 @@ | **Chargebee** | Chargebee icon | Source | airbyte/source-chargebee:0.2.3 | generally_available | [link](https://docs.airbyte.com/integrations/sources/chargebee) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-chargebee) | `686473f1-76d9-4994-9cc7-9b13da46147c` | | **Chargify** | Chargify icon | Source | airbyte/source-chargify:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/chargify) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-chargify) | `9b2d3607-7222-4709-9fa2-c2abdebbdd88` | | **Chartmogul** | Chartmogul icon | Source | airbyte/source-chartmogul:0.2.1 | beta | [link](https://docs.airbyte.com/integrations/sources/chartmogul) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-chartmogul) | `b6604cbd-1b12-4c08-8767-e140d0fb0877` | -| **ClickHouse** | ClickHouse icon | Source | airbyte/source-clickhouse:0.1.16 | alpha | [link](https://docs.airbyte.com/integrations/sources/clickhouse) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-clickhouse) | `bad83517-5e54-4a3d-9b53-63e85fbd4d7c` | +| **ClickHouse** | ClickHouse icon | Source | airbyte/source-clickhouse:0.1.17 | alpha | [link](https://docs.airbyte.com/integrations/sources/clickhouse) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-clickhouse) | `bad83517-5e54-4a3d-9b53-63e85fbd4d7c` | | **ClickUp** | ClickUp icon | Source | airbyte/source-clickup-api:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/click-up) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-clickup-api) | `311a7a27-3fb5-4f7e-8265-5e4afe258b66` | | **Clockify** | Clockify icon | Source | airbyte/source-clockify:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/clockify) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-clockify) | `e71aae8a-5143-11ed-bdc3-0242ac120002` | | **Close.com** | Close.com icon | Source | airbyte/source-close-com:0.2.1 | beta | [link](https://docs.airbyte.com/integrations/sources/close-com) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-close-com) | `dfffecb7-9a13-43e9-acdc-b92af7997ca9` | -| **Cockroachdb** | Cockroachdb icon | Source | airbyte/source-cockroachdb:0.1.21 | alpha | [link](https://docs.airbyte.com/integrations/sources/cockroachdb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-cockroachdb) | `9fa5862c-da7c-11eb-8d19-0242ac130003` | +| **Cockroachdb** | Cockroachdb icon | Source | airbyte/source-cockroachdb:0.1.22 | alpha | [link](https://docs.airbyte.com/integrations/sources/cockroachdb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-cockroachdb) | `9fa5862c-da7c-11eb-8d19-0242ac130003` | | **Coda** | Coda icon | Source | airbyte/source-coda:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/coda) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-coda) | `27f910fd-f832-4b2e-bcfd-6ab342e434d8` | | **Coin API** | Coin API icon | Source | airbyte/source-coin-api:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/coin-api) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-coin-api) | `919984ef-53a2-479b-8ffe-9c1ddb9fc3f3` | | **CoinGecko Coins** | CoinGecko Coins icon | Source | airbyte/source-coingecko-coins:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/coingecko-coins) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-coingecko-coins) | `9cdd4183-d0ba-40c3-aad3-6f46d4103974` | @@ -105,7 +105,7 @@ | **Hellobaton** | Hellobaton icon | Source | airbyte/source-hellobaton:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/hellobaton) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-hellobaton) | `492b56d1-937c-462e-8076-21ad2031e784` | | **HubSpot** | HubSpot icon | Source | airbyte/source-hubspot:0.3.2 | generally_available | [link](https://docs.airbyte.com/integrations/sources/hubspot) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-hubspot) | `36c891d9-4bd9-43ac-bad2-10e12756272c` | | **Hubplanner** | Hubplanner icon | Source | airbyte/source-hubplanner:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/hubplanner) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-hubplanner) | `8097ceb9-383f-42f6-9f92-d3fd4bcc7689` | -| **IBM Db2** | IBM Db2 icon | Source | airbyte/source-db2:0.1.18 | alpha | [link](https://docs.airbyte.com/integrations/sources/db2) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-db2) | `447e0381-3780-4b46-bb62-00a4e3c8b8e2` | +| **IBM Db2** | IBM Db2 icon | Source | airbyte/source-db2:0.1.19 | alpha | [link](https://docs.airbyte.com/integrations/sources/db2) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-db2) | `447e0381-3780-4b46-bb62-00a4e3c8b8e2` | | **IP2Whois** | IP2Whois icon | Source | airbyte/source-ip2whois:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/ip2whois) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-ip2whois) | `f23b7b7c-d705-49a3-9042-09add3b104a5` | | **Insightly** | Insightly icon | Source | airbyte/source-insightly:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/insightly) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-insightly) | `38f84314-fe6a-4257-97be-a8dcd942d693` | | **Instagram** | Instagram icon | Source | airbyte/source-instagram:1.0.4 | generally_available | [link](https://docs.airbyte.com/integrations/sources/instagram) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-instagram) | `6acf6b55-4f1e-4fca-944e-1a3caef8aba8` | @@ -138,13 +138,13 @@ | **Marketo** | Marketo icon | Source | airbyte/source-marketo:1.0.2 | generally_available | [link](https://docs.airbyte.com/integrations/sources/marketo) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-marketo) | `9e0556f4-69df-4522-a3fb-03264d36b348` | | **Metabase** | Metabase icon | Source | airbyte/source-metabase:0.3.1 | beta | [link](https://docs.airbyte.com/integrations/sources/metabase) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-metabase) | `c7cb421b-942e-4468-99ee-e369bcabaec5` | | **Microsoft Dataverse** | Microsoft Dataverse icon | Source | airbyte/source-microsoft-dataverse:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/microsoft-dataverse) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-microsoft-dataverse) | `9220e3de-3b60-4bb2-a46f-046d59ea235a` | -| **Microsoft SQL Server (MSSQL)** | Microsoft SQL Server (MSSQL) icon | Source | airbyte/source-mssql:1.0.5 | alpha | [link](https://docs.airbyte.com/integrations/sources/mssql) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-mssql) | `b5ea17b1-f170-46dc-bc31-cc744ca984c1` | +| **Microsoft SQL Server (MSSQL)** | Microsoft SQL Server (MSSQL) icon | Source | airbyte/source-mssql:1.0.6 | alpha | [link](https://docs.airbyte.com/integrations/sources/mssql) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-mssql) | `b5ea17b1-f170-46dc-bc31-cc744ca984c1` | | **Microsoft teams** | Microsoft teams icon | Source | airbyte/source-microsoft-teams:0.2.5 | alpha | [link](https://docs.airbyte.com/integrations/sources/microsoft-teams) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-microsoft-teams) | `eaf50f04-21dd-4620-913b-2a83f5635227` | | **Mixpanel** | Mixpanel icon | Source | airbyte/source-mixpanel:0.1.30 | generally_available | [link](https://docs.airbyte.com/integrations/sources/mixpanel) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-mixpanel) | `12928b32-bf0a-4f1e-964f-07e12e37153a` | | **Monday** | Monday icon | Source | airbyte/source-monday:0.2.3 | beta | [link](https://docs.airbyte.com/integrations/sources/monday) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-monday) | `80a54ea2-9959-4040-aac1-eee42423ec9b` | | **MongoDb** | MongoDb icon | Source | airbyte/source-mongodb-v2:0.1.19 | alpha | [link](https://docs.airbyte.com/integrations/sources/mongodb-v2) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-mongodb-v2) | `b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e` | | **My Hours** | My Hours icon | Source | airbyte/source-my-hours:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/my-hours) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-my-hours) | `722ba4bf-06ec-45a4-8dd5-72e4a5cf3903` | -| **MySQL** | MySQL icon | Source | airbyte/source-mysql:2.0.7 | beta | [link](https://docs.airbyte.com/integrations/sources/mysql) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-mysql) | `435bb9a5-7887-4809-aa58-28c27df0d7ad` | +| **MySQL** | MySQL icon | Source | airbyte/source-mysql:2.0.8 | beta | [link](https://docs.airbyte.com/integrations/sources/mysql) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-mysql) | `435bb9a5-7887-4809-aa58-28c27df0d7ad` | | **NASA** | NASA icon | Source | airbyte/source-nasa:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/nasa) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-nasa) | `1a8667d7-7978-43cd-ba4d-d32cbd478971` | | **Netsuite** | Netsuite icon | Source | airbyte/source-netsuite:0.1.3 | alpha | [link](https://docs.airbyte.com/integrations/sources/netsuite) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-netsuite) | `4f2f093d-ce44-4121-8118-9d13b7bfccd0` | | **New York Times** | New York Times icon | Source | airbyte/source-nytimes:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/nytimes) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-nytimes) | `0fae6a9a-04eb-44d4-96e1-e02d3dbc1d83` | @@ -157,7 +157,7 @@ | **Open Exchange Rates** | Open Exchange Rates icon | Source | airbyte/source-open-exchange-rates:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/open-exchange-rates) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-open-exchange-rates) | `77d5ca6b-d345-4dce-ba1e-1935a75778b8` | | **OpenWeather** | OpenWeather icon | Source | airbyte/source-openweather:0.1.6 | alpha | [link](https://docs.airbyte.com/integrations/sources/openweather) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-openweather) | `d8540a80-6120-485d-b7d6-272bca477d9b` | | **Opsgenie** | x | Source | airbyte/source-opsgenie:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/opsgenie) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-opsgenie) | `06bdb480-2598-40b8-8b0f-fc2e2d2abdda` | -| **Oracle DB** | Oracle DB icon | Source | airbyte/source-oracle:0.3.23 | alpha | [link](https://docs.airbyte.com/integrations/sources/oracle) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-oracle) | `b39a7370-74c3-45a6-ac3a-380d48520a83` | +| **Oracle DB** | Oracle DB icon | Source | airbyte/source-oracle:0.3.24 | alpha | [link](https://docs.airbyte.com/integrations/sources/oracle) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-oracle) | `b39a7370-74c3-45a6-ac3a-380d48520a83` | | **Orb** | Orb icon | Source | airbyte/source-orb:1.0.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/orb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-orb) | `7f0455fb-4518-4ec0-b7a3-d808bf8081cc` | | **Orbit** | Orbit icon | Source | airbyte/source-orbit:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/orbit) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-orbit) | `95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a` | | **Oura** | Oura icon | Source | airbyte/source-oura:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/oura) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-oura) | `2bf6c581-bec5-4e32-891d-de33036bd631` | @@ -178,7 +178,7 @@ | **PokeAPI** | PokeAPI icon | Source | airbyte/source-pokeapi:0.1.5 | alpha | [link](https://docs.airbyte.com/integrations/sources/pokeapi) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-pokeapi) | `6371b14b-bc68-4236-bfbd-468e8df8e968` | | **Polygon Stock API** | Polygon Stock API icon | Source | airbyte/source-polygon-stock-api:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/polygon-stock-api) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-polygon-stock-api) | `5807d72f-0abc-49f9-8fa5-ae820007032b` | | **PostHog** | PostHog icon | Source | airbyte/source-posthog:0.1.8 | beta | [link](https://docs.airbyte.com/integrations/sources/posthog) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-posthog) | `af6d50ee-dddf-4126-a8ee-7faee990774f` | -| **Postgres** | Postgres icon | Source | airbyte/source-postgres:2.0.8 | generally_available | [link](https://docs.airbyte.com/integrations/sources/postgres) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-postgres) | `decd338e-5647-4c0b-adf4-da0e75f5a750` | +| **Postgres** | Postgres icon | Source | airbyte/source-postgres:2.0.9 | generally_available | [link](https://docs.airbyte.com/integrations/sources/postgres) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-postgres) | `decd338e-5647-4c0b-adf4-da0e75f5a750` | | **Postmark App** | Postmark App icon | Source | airbyte/source-postmarkapp:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/postmarkapp) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-postmarkapp) | `cde75ca1-1e28-4a0f-85bb-90c546de9f1f` | | **PrestaShop** | PrestaShop icon | Source | airbyte/source-prestashop:0.3.1 | beta | [link](https://docs.airbyte.com/integrations/sources/prestashop) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-prestashop) | `d60a46d4-709f-4092-a6b7-2457f7d455f5` | | **Primetric** | Primetric icon | Source | airbyte/source-primetric:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/primetric) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-primetric) | `f636c3c6-4077-45ac-b109-19fc62a283c1` | @@ -220,7 +220,7 @@ | **SmartEngage** | SmartEngage icon | Source | airbyte/source-smartengage:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/smartengage) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-smartengage) | `21cc4a17-a011-4485-8a3e-e2341a91ab9f` | | **Smartsheets** | Smartsheets icon | Source | airbyte/source-smartsheets:1.0.0 | beta | [link](https://docs.airbyte.com/integrations/sources/smartsheets) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-smartsheets) | `374ebc65-6636-4ea0-925c-7d35999a8ffc` | | **Snapchat Marketing** | Snapchat Marketing icon | Source | airbyte/source-snapchat-marketing:0.1.15 | generally_available | [link](https://docs.airbyte.com/integrations/sources/snapchat-marketing) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-snapchat-marketing) | `200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b` | -| **Snowflake** | Snowflake icon | Source | airbyte/source-snowflake:0.1.31 | alpha | [link](https://docs.airbyte.com/integrations/sources/snowflake) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-snowflake) | `e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2` | +| **Snowflake** | Snowflake icon | Source | airbyte/source-snowflake:0.1.32 | alpha | [link](https://docs.airbyte.com/integrations/sources/snowflake) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-snowflake) | `e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2` | | **Sonar Cloud** | Sonar Cloud icon | Source | airbyte/source-sonar-cloud:0.1.1 | alpha | [link](https://docs.airbyte.com/integrations/sources/sonar-cloud) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-sonar-cloud) | `3ab1d7d0-1577-4ab9-bcc4-1ff6a4c2c9f2` | | **SpaceX API** | SpaceX API icon | Source | airbyte/source-spacex-api:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/spacex-api) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-spacex-api) | `62235e65-af7a-4138-9130-0bda954eb6a8` | | **Square** | Square icon | Source | airbyte/source-square:0.2.1 | beta | [link](https://docs.airbyte.com/integrations/sources/square) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-square) | `77225a51-cd15-4a13-af02-65816bd0ecf4` | @@ -236,7 +236,7 @@ | **TalkDesk Explore** | TalkDesk Explore icon | Source | airbyte/source-talkdesk-explore:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/talkdesk-explore) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-talkdesk-explore) | `f00d2cf4-3c28-499a-ba93-b50b6f26359e` | | **Tempo** | Tempo icon | Source | airbyte/source-tempo:0.3.1 | beta | [link](https://docs.airbyte.com/integrations/sources/tempo) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tempo) | `d1aa448b-7c54-498e-ad95-263cbebcd2db` | | **The Guardian API** | The Guardian API icon | Source | airbyte/source-the-guardian-api:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/the-guardian-api) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-the-guardian-api) | `d42bd69f-6bf0-4d0b-9209-16231af07a92` | -| **TiDB** | TiDB icon | Source | airbyte/source-tidb:0.2.3 | alpha | [link](https://docs.airbyte.com/integrations/sources/tidb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tidb) | `0dad1a35-ccf8-4d03-b73e-6788c00b13ae` | +| **TiDB** | TiDB icon | Source | airbyte/source-tidb:0.2.4 | alpha | [link](https://docs.airbyte.com/integrations/sources/tidb) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tidb) | `0dad1a35-ccf8-4d03-b73e-6788c00b13ae` | | **TikTok Marketing** | TikTok Marketing icon | Source | airbyte/source-tiktok-marketing:2.0.4 | generally_available | [link](https://docs.airbyte.com/integrations/sources/tiktok-marketing) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-tiktok-marketing) | `4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35` | | **Timely** | Timely icon | Source | airbyte/source-timely:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/timely) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-timely) | `bc617b5f-1b9e-4a2d-bebe-782fd454a771` | | **Toggl** | Toggl icon | Source | airbyte/source-toggl:0.1.0 | alpha | [link](https://docs.airbyte.com/integrations/sources/toggl) | [code](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-toggl) | `7e7c844f-2300-4342-b7d3-6dd7992593cd` | diff --git a/docs/integrations/sources/alloydb.md b/docs/integrations/sources/alloydb.md index 5985cae3b345..af99e2be6133 100644 --- a/docs/integrations/sources/alloydb.md +++ b/docs/integrations/sources/alloydb.md @@ -321,6 +321,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.9 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 2.0.6 | 2022-03-21 | [24271](https://github.com/airbytehq/airbyte/pull/24271) | Fix NPE in CDC mode | | 2.0.3 | 2023-03-21 | [24147](https://github.com/airbytehq/airbyte/pull/24275) | Fix error with CDC checkpointing | | 2.0.2 | 2022-03-13 | [23112](https://github.com/airbytehq/airbyte/pull/21727) | Add state checkpointing for CDC sync. | diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index cadf5d25ac27..912065139d93 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -78,6 +78,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:-----------------------------------------------------------------| +| 0.1.17 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 0.1.16 |2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | | 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | @@ -98,6 +99,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:---| :--- |:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.17 | 2022-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 0.1.16 |2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to| | 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index 4cdac1ed86a8..a32fddf184d9 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -95,6 +95,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- | :--- | +| 0.1.22 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 0.1.21 | 2023-03-14 | [24000](https://github.com/airbytehq/airbyte/pull/24000) | Removed check method call on read. | | 0.1.20 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect | | 0.1.19 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index ba6fb389b672..33460c6a113f 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -60,6 +60,7 @@ You can also enter your own password for the keystore, but if you don't, the pas | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.19 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 0.1.18 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | | 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index f6e952bea11e..194ac7501f23 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -341,6 +341,7 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.0.6 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 1.0.5 | 2022-03-21 | [24207](https://github.com/airbytehq/airbyte/pull/24207) | Fix incorrect schema change warning in CDC mode | | 1.0.4 | 2023-03-21 | [24147](https://github.com/airbytehq/airbyte/pull/24275) | Fix error with CDC checkpointing | | 1.0.3 | 2023-03-15 | [24082](https://github.com/airbytehq/airbyte/pull/24082) | Fixed NPE during cursor values validation | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 33be0c07a33b..43f083fdf6b9 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -256,6 +256,7 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.8 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 2.0.7 | 2022-03-21 | [24207](https://github.com/airbytehq/airbyte/pull/24207) | Fix incorrect schema change warning in CDC mode | | 2.0.6 | 2023-03-21 | [23984](https://github.com/airbytehq/airbyte/pull/23984) | Support CDC heartbeats | | 2.0.5 | 2023-03-21 | [24147](https://github.com/airbytehq/airbyte/pull/24275) | Fix error with CDC checkpointing | diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index 76815e17e103..319e5763e941 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -132,6 +132,7 @@ Airbyte has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:------------------------------------------------| +| 0.3.24 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 0.3.23 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | | 0.3.22 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index c342acbf4a4f..c3492a93fa1f 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -396,6 +396,7 @@ The root causes is that the WALs needed for the incremental sync has been remove | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.9 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 2.0.8 | 2023-03-22 | [24255](https://github.com/airbytehq/airbyte/pull/24255) | Add field groups and titles to improve display of connector setup form | | 2.0.7 | 2022-03-21 | [24207](https://github.com/airbytehq/airbyte/pull/24207) | Fix incorrect schema change warning in CDC mode | | 2.0.6 | 2022-03-21 | [24271](https://github.com/airbytehq/airbyte/pull/24271) | Fix NPE in CDC mode | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 5ef13fbbbe7e..40b4d2c9c288 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -122,6 +122,7 @@ To read more please check official [Snowflake documentation](https://docs.snowfl | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.32 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 0.1.31 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | | 0.1.30 | 2023-02-21 | [22358](https://github.com/airbytehq/airbyte/pull/22358) | Improved handling of big integer cursor type values. | | 0.1.29 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources. | diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index 5833c3d9cd73..efc44043cd19 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -128,6 +128,7 @@ Now that you have set up the TiDB source connector, check out the following TiDB | Version | Date | Pull Request | Subject | | :------ | :--- | :----------- | ------- | +| 0.2.4 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | | 0.2.3 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | | 0.2.2 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |