postgres-source: handle 24:00:00 value for time column (#17782)

* postgres-source: handle 24:00:00 value for time column * address review comment * fix test + use LocalTime.MAX * bump version * update alloy db as well * auto-bump connector version [ci skip] * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Oct 11, 2022 · 4652173 · 4652173
1 parent e2a5c37
commit 4652173
Show file tree

Hide file tree

Showing 10 changed files with 26 additions and 17 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -17,7 +17,7 @@
 - name: AlloyDB for PostgreSQL
   sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002
   dockerRepository: airbyte/source-alloydb
-  dockerImageTag: 1.0.0
+  dockerImageTag: 1.0.15
   documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb
   icon: alloydb.svg
   sourceType: database
@@ -836,7 +836,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 1.0.14
+  dockerImageTag: 1.0.15
   documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -198,7 +198,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-alloydb:1.0.0"
+- dockerImage: "airbyte/source-alloydb:1.0.15"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -8590,7 +8590,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:1.0.14"
+- dockerImage: "airbyte/source-postgres:1.0.15"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:

diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java
@@ -145,11 +145,11 @@ public static String convertToTime(final Object time) {
       return localTime.format(TIME_FORMATTER);
     } else if (time instanceof java.time.Duration) {
       long value = ((Duration) time).toNanos();
-      if (value >= 0 && value <= TimeUnit.DAYS.toNanos(1)) {
+      if (value >= 0 && value < TimeUnit.DAYS.toNanos(1)) {
         return LocalTime.ofNanoOfDay(value).format(TIME_FORMATTER);
       } else {
-        final long updatedValue = 0 > value ? Math.abs(value) : TimeUnit.DAYS.toNanos(1);
-        LOGGER.debug("Time values must use number of milliseconds greater than 0 and less than 86400000000000 but its {}, converting to {} ", value,
+        final long updatedValue = Math.min(Math.abs(value), LocalTime.MAX.toNanoOfDay());
+        LOGGER.debug("Time values must use number of nanoseconds greater than 0 and less than 86400000000000 but its {}, converting to {} ", value,
             updatedValue);
         return LocalTime.ofNanoOfDay(updatedValue).format(TIME_FORMATTER);
       }
@@ -158,7 +158,13 @@ public static String convertToTime(final Object time) {
         LOGGER.info("Unknown class for Time data type" + time.getClass());
         loggedUnknownTimeClass = true;
       }
-      return LocalTime.parse(time.toString()).format(TIME_FORMATTER);
+
+      final String valueAsString = time.toString();
+      if (valueAsString.startsWith("24")) {
+        LOGGER.debug("Time value {} is above range, converting to 23:59:59", valueAsString);
+        return LocalTime.MAX.format(TIME_FORMATTER);
+      }
+      return LocalTime.parse(valueAsString).format(TIME_FORMATTER);
     }
   }
 

diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=1.0.0
+LABEL io.airbyte.version=1.0.15
 LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt
diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-alloydb
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=1.0.0
+LABEL io.airbyte.version=1.0.15
 LABEL io.airbyte.name=airbyte/source-alloydb
diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=1.0.14
+LABEL io.airbyte.version=1.0.15
 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt
diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=1.0.14
+LABEL io.airbyte.version=1.0.15
 LABEL io.airbyte.name=airbyte/source-postgres
diff --git a/...integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java b/...integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java
@@ -404,9 +404,9 @@ protected void initTests() {
               .fullSourceDataType(fullSourceType)
               .airbyteType(JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE)
               // time column will ignore time zone
-              .addInsertValues("'13:00:01'", "'13:00:02+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.01234Z+8'", "'13:00:00Z-8'")
+              .addInsertValues("'13:00:01'", "'13:00:02+8'", "'13:00:03-8'", "'13:00:04Z'", "'13:00:05.01234Z+8'", "'13:00:00Z-8'", "'24:00:00'")
               .addExpectedValues("13:00:01.000000", "13:00:02.000000", "13:00:03.000000", "13:00:04.000000", "13:00:05.012340",
-                  "13:00:00.000000")
+                  "13:00:00.000000", "23:59:59.999999")
               .build());
     }
 

diff --git a/docs/integrations/sources/alloydb.md b/docs/integrations/sources/alloydb.md
@@ -331,12 +331,14 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp
 
 | Version | Date       | Pull Request                                             | Subject                                          |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------|
+| 1.0.15  | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782)   | Align with Postgres source v.1.0.15              |
 | 1.0.0   | 2022-09-15 | [16776](https://github.com/airbytehq/airbyte/pull/16776)   | Align with strict-encrypt version                |
 | 0.1.0   | 2022-09-05 | [16323](https://github.com/airbytehq/airbyte/pull/16323)   | Initial commit. Based on source-postgres v.1.0.7 |
 
 
 ## Changelog (Strict Encrypt)
 
-| Version | Date       | Pull Request                                             | Subject                                                  |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------|
-| 1.0.0   | 2022-09-15 | [16776](https://github.com/airbytehq/airbyte/pull/16776)   | Initial commit. Based on source-postgres-strict-encrypt  |
+| Version | Date       | Pull Request                                             | Subject                                                 |
+|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------|
+| 1.0.15  | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Align with Postgres source v.1.0.15                     |
+| 1.0.0   | 2022-09-15 | [16776](https://github.com/airbytehq/airbyte/pull/16776) | Initial commit. Based on source-postgres-strict-encrypt |
diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md
@@ -398,6 +398,7 @@ The root causes is that the WALs needed for the incremental sync has been remove
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                    |
 |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.0.15  | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Handle 24:00:00 value for Time column                                                                                                                                      |
 | 1.0.14  | 2022-10-03 | [17515](https://github.com/airbytehq/airbyte/pull/17515) | Fix an issue preventing connection using client certificate                                                                                                                |
 | 1.0.13  | 2022-10-01 | [17459](https://github.com/airbytehq/airbyte/pull/17459) | Upgrade debezium version to 1.9.6 from 1.9.2                                                                                                                               |
 | 1.0.12  | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres source                                                                                                                  |