From 2d26337bda32805a48c51d186fefb33384018a65 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Thu, 30 Nov 2023 11:05:45 -0800 Subject: [PATCH] Use parse base64Binary to parse binary related data (#32610) Co-authored-by: xiaohansong Co-authored-by: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Co-authored-by: Alexandre Girard --- airbyte-cdk/java/airbyte-cdk/README.md | 3 ++- .../cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java | 2 +- .../airbyte-cdk/core/src/main/resources/version.properties | 2 +- .../src/test/java/io/airbyte/cdk/db/jdbc/TestJdbcUtils.java | 6 +++++- airbyte-integrations/connectors/source-mysql/build.gradle | 4 ++-- airbyte-integrations/connectors/source-mysql/metadata.yaml | 3 ++- docs/integrations/sources/mysql.md | 3 ++- 7 files changed, 15 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 164dfa230a25..7733817f4ebd 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -155,7 +155,8 @@ MavenLocal debugging steps: ### Java CDK | Version | Date | Pull Request | Subject | -|:--------|:-----------| :--------------------------------------------------------- |:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.6.1 | 2023-11-30 | [\#32610](https://github.com/airbytehq/airbyte/pull/32610) | Support DB inital sync using binary as primary key. | | 0.6.0 | 2023-11-30 | [\#32888](https://github.com/airbytehq/airbyte/pull/32888) | JDBC destinations now use the async framework | | 0.5.3 | 2023-11-28 | [\#32686](https://github.com/airbytehq/airbyte/pull/32686) | Better attribution of debezium engine shutdown due to heartbeat. | | 0.5.1 | 2023-11-27 | [\#32662](https://github.com/airbytehq/airbyte/pull/32662) | Debezium initialization wait time will now read from initial setup time. | diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index 0015a537dcd7..e7b8514cb7f5 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -222,7 +222,7 @@ protected void setString(final PreparedStatement preparedStatement, final int pa } protected void setBinary(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseHexBinary(value)); + preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseBase64Binary(value)); } protected ObjectType getObject(final ResultSet resultSet, final int index, final Class clazz) throws SQLException { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index af0eb1ce35a2..3f83643ec920 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.6.0 +version=0.6.1 diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestJdbcUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestJdbcUtils.java index a41f310dea97..317566be30ec 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestJdbcUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestJdbcUtils.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BinaryNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; @@ -35,6 +36,7 @@ import java.util.Map; import java.util.stream.Collectors; import javax.sql.DataSource; +import org.bouncycastle.util.encoders.Base64; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -376,6 +378,8 @@ private ObjectNode jsonFieldExpectedValues() { arrayNode2.add("3"); expected.set("int_array", arrayNode2); + expected.set("binary1", new BinaryNode("aaaa".getBytes(Charsets.UTF_8))); + return expected; } @@ -396,7 +400,7 @@ private ObjectNode expectedValues() { expected.put("date", "2020-11-01"); expected.put("time", "05:00:00.000000"); expected.put("timestamp", "2001-09-29T03:00:00.000000"); - expected.put("binary1", "aaaa".getBytes(Charsets.UTF_8)); + expected.put("binary1", Base64.decode("61616161".getBytes(Charsets.UTF_8))); return expected; } diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 70966b334176..9a465b64d553 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -7,7 +7,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.5.1' + cdkVersionRequired = '0.5.4' features = ['db-sources'] useLocalCdk = false } @@ -19,6 +19,7 @@ configurations.all { } + application { mainClass = 'io.airbyte.integrations.source.mysql.MySqlSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -35,7 +36,6 @@ dependencies { testImplementation libs.junit.jupiter.system.stubs testImplementation libs.testcontainers.mysql testFixturesImplementation libs.testcontainers.mysql - performanceTestJavaImplementation project(':airbyte-integrations:connectors:source-mysql') } diff --git a/airbyte-integrations/connectors/source-mysql/metadata.yaml b/airbyte-integrations/connectors/source-mysql/metadata.yaml index 8a3792b505df..64335de6a25b 100644 --- a/airbyte-integrations/connectors/source-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.2.0 + dockerImageTag: 3.2.1 dockerRepository: airbyte/source-mysql documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql @@ -18,6 +18,7 @@ data: name: MySQL registries: cloud: + dockerRepository: airbyte/source-mysql-strict-encrypt enabled: true oss: enabled: true diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index e855e669eaaf..adc3204b238c 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -221,7 +221,8 @@ Any database or table encoding combination of charset and collation is supported ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------- | +|:--------| :--------- | :--------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.2.1 | 2023-11-28 | [32610](https://github.com/airbytehq/airbyte/pull/32610) | Support initial syncs using binary as primary key. | | 3.2.0 | 2023-11-29 | [31062](https://github.com/airbytehq/airbyte/pull/31062) | enforce SSL on Airbyte Cloud | | 3.1.9 | 2023-11-27 | [32662](https://github.com/airbytehq/airbyte/pull/32662) | Apply initial setup time to debezium engine warmup time. | | 3.1.8 | 2023-11-22 | [32656](https://github.com/airbytehq/airbyte/pull/32656) | Adopt java CDK version 0.5.0. |