From 92b8b8f17dca8666424a5f57db8702c81a848e1f Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Thu, 21 Jul 2022 17:30:37 +0300 Subject: [PATCH 01/24] updated mysql source specification and added field for root and clients SSL certificates --- .../source/mysql/MySqlSource.java | 149 +++++++++++++++++- .../source-mysql/src/main/resources/spec.json | 129 +++++++++++++++ 2 files changed, 273 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 60e6940054ba..2f37896812bc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -34,13 +34,21 @@ import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.SyncMode; + +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,8 +63,10 @@ public class MySqlSource extends AbstractJdbcSource implements Source public static final String CDC_LOG_POS = "_ab_cdc_log_pos"; public static final List SSL_PARAMETERS = List.of( "useSSL=true", - "requireSSL=true", - "verifyServerCertificate=false"); + "requireSSL=true"); + + public static final String SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION = "verifyServerCertificate=true"; + public static final String SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION = "verifyServerCertificate=false"; public static Source sshWrappedSource() { return new SshWrappedSource(new MySqlSource(), List.of("host"), List.of("port")); @@ -145,20 +155,35 @@ public JsonNode toDatabaseConfig(final JsonNode config) { if (config.get("jdbc_url_params") != null && !config.get("jdbc_url_params").asText().isEmpty()) { jdbcUrl.append("&").append(config.get("jdbc_url_params").asText()); } - + Map additionalParameters = new HashMap<>(); // assume ssl if not explicitly mentioned. if (!config.has("ssl") || config.get("ssl").asBoolean()) { - jdbcUrl.append("&").append(String.join("&", SSL_PARAMETERS)); + if (config.has("ssl_mode")) { + if ("disable".equals(config.get("ssl_mode").get("mode").asText())) { + jdbcUrl.append("&").append("sslMode=DISABLE"); + } else { + additionalParameters.putAll(obtainConnectionOptions(config.get("ssl_mode"))); + jdbcUrl.append("&").append(String.join("&", SSL_PARAMETERS)).append("&"); + if (additionalParameters.isEmpty()) { + jdbcUrl.append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); + } else { + jdbcUrl.append(SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION); + } + } + } else { + jdbcUrl.append("&").append(String.join("&", SSL_PARAMETERS)); + } } final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put("username", config.get("username").asText()) .put("jdbc_url", jdbcUrl.toString()); + configBuilder.putAll(additionalParameters); + if (config.has("password")) { configBuilder.put("password", config.get("password").asText()); } - return Jsons.jsonNode(configBuilder.build()); } @@ -215,4 +240,118 @@ public enum ReplicationMethod { CDC } + /* Helpers */ + + public static Map obtainConnectionOptions(final JsonNode encryption) { + Map additionalParameters = new HashMap<>(); + if (!encryption.isNull()) { + final var method = encryption.get("mode").asText().toUpperCase(); + String sslPassword = encryption.has("client_key_password") ? encryption.get("client_key_password").asText() : ""; + var keyStorePassword = RandomStringUtils.randomAlphanumeric(10); + if (!sslPassword.isEmpty()) { + keyStorePassword = sslPassword; + } + switch (method) { + case "VERIFY_CA" -> { + additionalParameters.putAll(obtainConnectionCaOptions(encryption, method, keyStorePassword)); + } + case "VERIFY_IDENTITY" -> { + additionalParameters.putAll(obtainConnectionFullOptions(encryption, method, keyStorePassword)); + } + } + } + return additionalParameters; + } + + private static Map obtainConnectionFullOptions(final JsonNode encryption, + final String method, + final String clientKeyPassword) { + Map additionalParameters = new HashMap<>(); + try { + convertAndImportFullCertificate(encryption.get("ca_certificate").asText(), + encryption.get("client_certificate").asText(), encryption.get("client_key").asText(), clientKeyPassword); + } catch (final IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + additionalParameters.put("trustCertificateKeyStoreUrl", "customtruststore"); + additionalParameters.put("trustCertificateKeyStorePassword", clientKeyPassword); + additionalParameters.put("clientCertificateKeyStoreUrl", "customkeystore"); + additionalParameters.put("clientCertificateKeyStorePassword", clientKeyPassword); + additionalParameters.put("sslMode", method.toUpperCase()); + return additionalParameters; + } + + private static Map obtainConnectionCaOptions(final JsonNode encryption, + final String method, + final String clientKeyPassword) { + Map additionalParameters = new HashMap<>(); + try { + convertAndImportCaCertificate(encryption.get("ca_certificate").asText(), clientKeyPassword); + } catch (final IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + additionalParameters.put("trustCertificateKeyStoreUrl", "customtruststore"); + additionalParameters.put("trustCertificateKeyStorePassword", clientKeyPassword); + additionalParameters.put("sslMode", method.toUpperCase()); + return additionalParameters; + } + + private static void convertAndImportFullCertificate(final String caCertificate, + final String clientCertificate, + final String clientKey, + final String clientKeyPassword) + throws IOException, InterruptedException { + final Runtime run = Runtime.getRuntime(); + createCaCertificate(caCertificate, clientKeyPassword, run); + createCertificateFile("client-cert.pem", clientCertificate); + createCertificateFile("client-key.pem", clientKey); + // add client certificate to the custom keystore + runProcess("keytool -alias client-certificate -keystore customkeystore" + + " -import -file client-cert.pem -storepass " + clientKeyPassword + " -noprompt", run); + // add client key to the custom keystore + runProcess("keytool -alias client-key -keystore customkeystore" + + " -import -file client-key.pem -storepass " + clientKeyPassword + " -noprompt", run); + runProcess("rm client-key.pem", run); + + updateTrustStoreSystemProperty(clientKeyPassword); + } + + private static void convertAndImportCaCertificate(final String caCertificate, + final String clientKeyPassword) + throws IOException, InterruptedException { + final Runtime run = Runtime.getRuntime(); + createCaCertificate(caCertificate, clientKeyPassword, run); + updateTrustStoreSystemProperty(clientKeyPassword); + } + + private static void createCaCertificate(final String caCertificate, + final String clientKeyPassword, + final Runtime run) + throws IOException, InterruptedException { + createCertificateFile("ca.pem", caCertificate); + // add CA certificate to the custom keystore + runProcess("keytool -import -alias root-certificate -keystore customtruststore" + + " -file ca.pem -storepass " + clientKeyPassword + " -noprompt", run); + } + + private static void updateTrustStoreSystemProperty(final String clientKeyPassword) { + String result = System.getProperty("user.dir") + "/customtruststore"; + System.setProperty("javax.net.ssl.trustStore", result); + System.setProperty("javax.net.ssl.trustStorePassword", clientKeyPassword); + } + + private static void createCertificateFile(String fileName, String fileValue) throws IOException { + try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { + out.print(fileValue); + } + } + + private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { + final Process pr = run.exec(cmd); + if (!pr.waitFor(30, TimeUnit.SECONDS)) { + pr.destroy(); + throw new RuntimeException("Timeout while executing: " + cmd); + } + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index c6a652f3e48e..e249328e9ee4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -55,6 +55,135 @@ "default": true, "order": 6 }, + "ssl_mode": { + "title": "SSL modes", + "description": "SSL modes", + "type": "object", + "order": 7, + "oneOf": [ + { + "title": "No", + "description": "Disable SSL.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "disable", + "enum": ["disable"], + "default": "disable", + "order": 0 + } + } + }, + { + "title": "If available", + "description": "Preferred SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "preferred", + "enum": ["preferred"], + "default": "preferred", + "order": 0 + } + } + }, + { + "title": "Require", + "description": "Require SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "required", + "enum": ["required"], + "default": "required", + "order": 0 + } + } + }, + { + "title": "Require and Verify CA", + "description": "Verify CA SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "verify_ca", + "enum": ["verify_ca"], + "default": "verify_ca", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_key_password": { + "type": "string", + "title": "Client key password (Optional)", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 2 + } + } + }, + { + "title": "Require and Verify Identity", + "description": "Verify-full SSL mode.", + "required": [ + "mode", + "ca_certificate", + "client_certificate", + "client_key" + ], + "properties": { + "mode": { + "type": "string", + "const": "verify_identity", + "enum": ["verify_identity"], + "default": "verify_identity", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate", + "airbyte_secret": true, + "multiline": true, + "order": 2 + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key", + "airbyte_secret": true, + "multiline": true, + "order": 3 + }, + "client_key_password": { + "type": "string", + "title": "Client key password (Optional)", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + } + ] + }, "replication_method": { "type": "string", "title": "Replication Method", From 7178243bb483cd67bf1199101635fe6574f100c9 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 26 Jul 2022 20:48:09 +0300 Subject: [PATCH 02/24] added SSL mode for mysql source --- airbyte-db/db-lib/build.gradle | 2 + .../main/java/io/airbyte/db/MySqlUtils.java | 75 +++++++++ .../java/io/airbyte/db/jdbc/JdbcUtils.java | 3 + .../util/MySqlSslConnectionUtils.java | 153 ++++++++++++++++++ .../source/mysql/MySqlSource.java | 139 ++-------------- ...slFullCertificateSourceAcceptanceTest.java | 78 +++++++++ .../mysql/MySqlSslSourceAcceptanceTest.java | 9 +- 7 files changed, 331 insertions(+), 128 deletions(-) create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java create mode 100644 airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java create mode 100644 airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java diff --git a/airbyte-db/db-lib/build.gradle b/airbyte-db/db-lib/build.gradle index 80b1fc5bca94..26828b8c8932 100644 --- a/airbyte-db/db-lib/build.gradle +++ b/airbyte-db/db-lib/build.gradle @@ -15,6 +15,7 @@ dependencies { // Mark as compile only to avoid leaking transitively to connectors compileOnly libs.platform.testcontainers.postgresql + compileOnly libs.connectors.testcontainers.mysql // These are required because gradle might be using lower version of Jna from other // library transitive dependency. Can be removed if we can figure out which library is the cause. @@ -25,6 +26,7 @@ dependencies { testImplementation project(':airbyte-test-utils') testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation libs.platform.testcontainers.postgresql + testImplementation libs.connectors.testcontainers.mysql // Big Query implementation('com.google.cloud:google-cloud-bigquery:1.133.1') diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java new file mode 100644 index 000000000000..dc78b48a491a --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db; + +import com.google.common.annotations.VisibleForTesting; +import org.testcontainers.containers.MySQLContainer; + +import java.io.IOException; + +public class MySqlUtils { + + @VisibleForTesting + public static Certificate getCertificate(final MySQLContainer container) throws IOException, InterruptedException { + container.execInContainer("sh", "-c", "mkdir -p /etc/mysql/newcerts"); + container.execInContainer("sh", "-c", "chown -R test:test /etc/mysql/newcerts"); + // create root certificates + container.execInContainer("sh", "-c", "openssl genrsa 2048 > /etc/mysql/newcerts/ca-key.pem"); + container.execInContainer("sh", "-c", + "openssl req -new -x509 -nodes -days 1000 -key /etc/mysql/newcerts/ca-key.pem > /etc/mysql/newcerts/ca-cert.pem -subj \"/CN=127.0.0.1\""); + // create server certificates + container.execInContainer("sh", "-c", + "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/server-key.pem > /etc/mysql/newcerts/server-req.pem -subj \"/CN=localhost\""); + container.execInContainer("sh", "-c", + "openssl x509 -req -in /etc/mysql/newcerts/server-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/server-cert.pem"); + // create client certificates + container.execInContainer("sh", "-c", + "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/client-key.pem > /etc/mysql/newcerts/client-req.pem -subj \"/CN=test\""); + container.execInContainer("sh", "-c", + "openssl x509 -req -in /etc/mysql/newcerts/client-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/client-cert.pem"); + // add root and server certificates to config file + container.execInContainer("sh", "-c", "sed -i '31 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '32 a ssl-cert=/etc/mysql/newcerts/server-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '33 a ssl-key=/etc/mysql/newcerts/server-key.pem' /etc/my.cnf"); + // add client certificates to config file + container.execInContainer("sh", "-c", "sed -i '38 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '39 a ssl-cert=/etc/mysql/newcerts/client-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '40 a ssl-key=/etc/mysql/newcerts/client-key.pem' /etc/my.cnf"); + // restart server + container.execInContainer("sh", "-c", "service mysqld restart"); + // copy root certificate and client certificates + var caCert = container.execInContainer("sh", "-c", "cat /etc/mysql/newcerts/ca-cert.pem").getStdout().trim(); + var clientKey = container.execInContainer("sh", "-c", "cat /etc/mysql/newcerts/client-key.pem").getStdout().trim(); + var clientCert = container.execInContainer("sh", "-c", "cat /etc/mysql/newcerts/client-cert.pem").getStdout().trim(); + return new Certificate(caCert, clientCert, clientKey); + } + + public static class Certificate { + + private final String caCertificate; + private final String clientCertificate; + private final String clientKey; + + public Certificate(final String caCertificate, final String clientCertificate, final String clientKey) { + this.caCertificate = caCertificate; + this.clientCertificate = clientCertificate; + this.clientKey = clientKey; + } + + public String getCaCertificate() { + return caCertificate; + } + + public String getClientCertificate() { + return clientCertificate; + } + + public String getClientKey() { + return clientKey; + } + + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 4288d672a64d..82eddf096abf 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -28,8 +28,11 @@ public class JdbcUtils { // NOTE: this is the plural version of SCHEMA_KEY public static final String SCHEMAS_KEY = "schemas"; public static final String SSL_KEY = "ssl"; + public static final String SSL_MODE_KEY = "ssl_mode"; public static final String TLS_KEY = "tls"; public static final String USERNAME_KEY = "username"; + public static final String MODE_KEY = "mode"; + public static final String AMPERSAND = "&"; private static final JdbcSourceOperations defaultSourceOperations = new JdbcSourceOperations(); private static final JSONFormat defaultJSONFormat = new JSONFormat().recordFormat(JSONFormat.RecordFormat.OBJECT); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java new file mode 100644 index 000000000000..ea1a3826454f --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.util; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class MySqlSslConnectionUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlSslConnectionUtils.class); + + public static final String PARAM_MODE = "mode"; + public static final String PARAM_CLIENT_KEY_PASSWORD = "client_key_password"; + public static final String PARAM_CA_CERTIFICATE = "ca_certificate"; + public static final String PARAM_CLIENT_CERTIFICATE = "client_certificate"; + public static final String PARAM_CLIENT_KEY = "client_key"; + public static final String TRUST_KEY_STORE_URL = "trustCertificateKeyStoreUrl"; + public static final String TRUST_KEY_STORE_PASS = "trustCertificateKeyStorePassword"; + public static final String CLIENT_KEY_STORE_URL = "clientCertificateKeyStoreUrl"; + public static final String CLIENT_KEY_STORE_PASS = "clientCertificateKeyStorePassword"; + public static final String CUSTOM_TRUST_STORE = "customtruststore"; + public static final String SSL_MODE = "sslMode"; + public static final String DISABLE = "disable"; + public static final String VERIFY_CA = "VERIFY_CA"; + public static final String VERIFY_IDENTITY = "VERIFY_IDENTITY"; + public static final String ROOT_CERTIFICARE_NAME = "ca.pem"; + public static final String CLIENT_CERTIFICARE_NAME = "client-cert.pem"; + public static final String CLIENT_KEY_NAME = "client-key.pem"; + + public static Map obtainConnectionOptions(final JsonNode encryption) { + Map additionalParameters = new HashMap<>(); + if (!encryption.isNull()) { + final var method = encryption.get(PARAM_MODE).asText().toUpperCase(); + String sslPassword = encryption.has(PARAM_CLIENT_KEY_PASSWORD) ? encryption.get(PARAM_CLIENT_KEY_PASSWORD).asText() : ""; + var keyStorePassword = RandomStringUtils.randomAlphanumeric(10); + if (!sslPassword.isEmpty()) { + keyStorePassword = sslPassword; + } + switch (method) { + case VERIFY_CA -> { + additionalParameters.putAll(obtainConnectionCaOptions(encryption, method, keyStorePassword)); + } + case VERIFY_IDENTITY -> { + additionalParameters.putAll(obtainConnectionFullOptions(encryption, method, keyStorePassword)); + } + } + } + return additionalParameters; + } + + private static Map obtainConnectionFullOptions(final JsonNode encryption, + final String method, + final String clientKeyPassword) { + Map additionalParameters = new HashMap<>(); + try { + convertAndImportFullCertificate(encryption.get(PARAM_CA_CERTIFICATE).asText(), + encryption.get(PARAM_CLIENT_CERTIFICATE).asText(), encryption.get(PARAM_CLIENT_KEY).asText(), clientKeyPassword); + } catch (final IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + additionalParameters.put(TRUST_KEY_STORE_URL, CUSTOM_TRUST_STORE); + additionalParameters.put(TRUST_KEY_STORE_PASS, clientKeyPassword); + additionalParameters.put(CLIENT_KEY_STORE_URL, CUSTOM_TRUST_STORE); + additionalParameters.put(CLIENT_KEY_STORE_PASS, clientKeyPassword); + additionalParameters.put(SSL_MODE, method.toUpperCase()); + return additionalParameters; + } + + private static Map obtainConnectionCaOptions(final JsonNode encryption, + final String method, + final String clientKeyPassword) { + Map additionalParameters = new HashMap<>(); + try { + convertAndImportCaCertificate(encryption.get(PARAM_CA_CERTIFICATE).asText(), clientKeyPassword); + } catch (final IOException | InterruptedException e) { + throw new RuntimeException("Failed to import certificate into Java Keystore"); + } + additionalParameters.put(TRUST_KEY_STORE_URL, CUSTOM_TRUST_STORE); + additionalParameters.put(TRUST_KEY_STORE_PASS, clientKeyPassword); + additionalParameters.put(SSL_MODE, method.toUpperCase()); + return additionalParameters; + } + + private static void convertAndImportFullCertificate(final String caCertificate, + final String clientCertificate, + final String clientKey, + final String clientKeyPassword) + throws IOException, InterruptedException { + final Runtime run = Runtime.getRuntime(); + createCaCertificate(caCertificate, clientKeyPassword, run); + createCertificateFile(CLIENT_CERTIFICARE_NAME, clientCertificate); + createCertificateFile(CLIENT_KEY_NAME, clientKey); + // add client certificate to the custom keystore + runProcess("keytool -alias client-certificate -keystore " + CUSTOM_TRUST_STORE + + " -import -file " + CLIENT_CERTIFICARE_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); + // add client key to the custom keystore + runProcess("keytool -alias client-key -keystore " + CUSTOM_TRUST_STORE + + " -import -file " + CLIENT_KEY_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); + runProcess("rm " + CLIENT_KEY_NAME, run); + + updateTrustStoreSystemProperty(clientKeyPassword); + } + + private static void convertAndImportCaCertificate(final String caCertificate, + final String clientKeyPassword) + throws IOException, InterruptedException { + final Runtime run = Runtime.getRuntime(); + createCaCertificate(caCertificate, clientKeyPassword, run); + updateTrustStoreSystemProperty(clientKeyPassword); + } + + private static void createCaCertificate(final String caCertificate, + final String clientKeyPassword, + final Runtime run) + throws IOException, InterruptedException { + createCertificateFile(ROOT_CERTIFICARE_NAME, caCertificate); + // add CA certificate to the custom keystore + runProcess("keytool -import -alias root-certificate -keystore " + CUSTOM_TRUST_STORE + + " -file " + ROOT_CERTIFICARE_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); + } + + private static void updateTrustStoreSystemProperty(final String clientKeyPassword) { + String result = System.getProperty("user.dir") + "/" + CUSTOM_TRUST_STORE; + System.setProperty("javax.net.ssl.trustStore", result); + System.setProperty("javax.net.ssl.trustStorePassword", clientKeyPassword); + } + + private static void createCertificateFile(String fileName, String fileValue) throws IOException { + try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { + out.print(fileValue); + } + } + + private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { + final Process pr = run.exec(cmd); + if (!pr.waitFor(120, TimeUnit.SECONDS)) { + pr.destroy(); + throw new RuntimeException("Timeout while executing: " + cmd); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 3bb39627ccef..6227e0d669d4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -7,6 +7,8 @@ import static io.airbyte.integrations.debezium.AirbyteDebeziumHandler.shouldUseCDC; import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.integrations.util.MySqlSslConnectionUtils.DISABLE; +import static io.airbyte.integrations.util.MySqlSslConnectionUtils.obtainConnectionOptions; import static java.util.stream.Collectors.toList; import com.fasterxml.jackson.databind.JsonNode; @@ -35,11 +37,9 @@ import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.SyncMode; + import java.time.Duration; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -48,9 +48,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -155,17 +152,18 @@ public JsonNode toDatabaseConfig(final JsonNode config) { // in the setJsonField method in MySqlSourceOperations.java jdbcUrl.append("&yearIsDateType=true"); if (config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty()) { - jdbcUrl.append("&").append(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); + jdbcUrl.append(JdbcUtils.AMPERSAND).append(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); } Map additionalParameters = new HashMap<>(); // assume ssl if not explicitly mentioned. if (!config.has(JdbcUtils.SSL_KEY) || config.get(JdbcUtils.SSL_KEY).asBoolean()) { - if (config.has("ssl_mode")) { - if ("disable".equals(config.get("ssl_mode").get("mode").asText())) { - jdbcUrl.append("&").append("sslMode=DISABLE"); + if (config.has(JdbcUtils.SSL_MODE_KEY)) { + if (DISABLE.equals(config.get(JdbcUtils.SSL_MODE_KEY).get(JdbcUtils.MODE_KEY).asText())) { + jdbcUrl.append(JdbcUtils.AMPERSAND).append("sslMode=DISABLE"); } else { - additionalParameters.putAll(obtainConnectionOptions(config.get("ssl_mode"))); - jdbcUrl.append("&").append(String.join("&", SSL_PARAMETERS)).append("&"); + additionalParameters.putAll(obtainConnectionOptions(config.get(JdbcUtils.SSL_MODE_KEY))); + jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) + .append(JdbcUtils.AMPERSAND); if (additionalParameters.isEmpty()) { jdbcUrl.append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); } else { @@ -173,7 +171,8 @@ public JsonNode toDatabaseConfig(final JsonNode config) { } } } else { - jdbcUrl.append("&").append(String.join("&", SSL_PARAMETERS)); + jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) + .append(JdbcUtils.AMPERSAND).append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); } } @@ -242,118 +241,4 @@ public enum ReplicationMethod { CDC } - /* Helpers */ - - public static Map obtainConnectionOptions(final JsonNode encryption) { - Map additionalParameters = new HashMap<>(); - if (!encryption.isNull()) { - final var method = encryption.get("mode").asText().toUpperCase(); - String sslPassword = encryption.has("client_key_password") ? encryption.get("client_key_password").asText() : ""; - var keyStorePassword = RandomStringUtils.randomAlphanumeric(10); - if (!sslPassword.isEmpty()) { - keyStorePassword = sslPassword; - } - switch (method) { - case "VERIFY_CA" -> { - additionalParameters.putAll(obtainConnectionCaOptions(encryption, method, keyStorePassword)); - } - case "VERIFY_IDENTITY" -> { - additionalParameters.putAll(obtainConnectionFullOptions(encryption, method, keyStorePassword)); - } - } - } - return additionalParameters; - } - - private static Map obtainConnectionFullOptions(final JsonNode encryption, - final String method, - final String clientKeyPassword) { - Map additionalParameters = new HashMap<>(); - try { - convertAndImportFullCertificate(encryption.get("ca_certificate").asText(), - encryption.get("client_certificate").asText(), encryption.get("client_key").asText(), clientKeyPassword); - } catch (final IOException | InterruptedException e) { - throw new RuntimeException("Failed to import certificate into Java Keystore"); - } - additionalParameters.put("trustCertificateKeyStoreUrl", "customtruststore"); - additionalParameters.put("trustCertificateKeyStorePassword", clientKeyPassword); - additionalParameters.put("clientCertificateKeyStoreUrl", "customkeystore"); - additionalParameters.put("clientCertificateKeyStorePassword", clientKeyPassword); - additionalParameters.put("sslMode", method.toUpperCase()); - return additionalParameters; - } - - private static Map obtainConnectionCaOptions(final JsonNode encryption, - final String method, - final String clientKeyPassword) { - Map additionalParameters = new HashMap<>(); - try { - convertAndImportCaCertificate(encryption.get("ca_certificate").asText(), clientKeyPassword); - } catch (final IOException | InterruptedException e) { - throw new RuntimeException("Failed to import certificate into Java Keystore"); - } - additionalParameters.put("trustCertificateKeyStoreUrl", "customtruststore"); - additionalParameters.put("trustCertificateKeyStorePassword", clientKeyPassword); - additionalParameters.put("sslMode", method.toUpperCase()); - return additionalParameters; - } - - private static void convertAndImportFullCertificate(final String caCertificate, - final String clientCertificate, - final String clientKey, - final String clientKeyPassword) - throws IOException, InterruptedException { - final Runtime run = Runtime.getRuntime(); - createCaCertificate(caCertificate, clientKeyPassword, run); - createCertificateFile("client-cert.pem", clientCertificate); - createCertificateFile("client-key.pem", clientKey); - // add client certificate to the custom keystore - runProcess("keytool -alias client-certificate -keystore customkeystore" - + " -import -file client-cert.pem -storepass " + clientKeyPassword + " -noprompt", run); - // add client key to the custom keystore - runProcess("keytool -alias client-key -keystore customkeystore" - + " -import -file client-key.pem -storepass " + clientKeyPassword + " -noprompt", run); - runProcess("rm client-key.pem", run); - - updateTrustStoreSystemProperty(clientKeyPassword); - } - - private static void convertAndImportCaCertificate(final String caCertificate, - final String clientKeyPassword) - throws IOException, InterruptedException { - final Runtime run = Runtime.getRuntime(); - createCaCertificate(caCertificate, clientKeyPassword, run); - updateTrustStoreSystemProperty(clientKeyPassword); - } - - private static void createCaCertificate(final String caCertificate, - final String clientKeyPassword, - final Runtime run) - throws IOException, InterruptedException { - createCertificateFile("ca.pem", caCertificate); - // add CA certificate to the custom keystore - runProcess("keytool -import -alias root-certificate -keystore customtruststore" - + " -file ca.pem -storepass " + clientKeyPassword + " -noprompt", run); - } - - private static void updateTrustStoreSystemProperty(final String clientKeyPassword) { - String result = System.getProperty("user.dir") + "/customtruststore"; - System.setProperty("javax.net.ssl.trustStore", result); - System.setProperty("javax.net.ssl.trustStorePassword", clientKeyPassword); - } - - private static void createCertificateFile(String fileName, String fileValue) throws IOException { - try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { - out.print(fileValue); - } - } - - private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { - final Process pr = run.exec(cmd); - if (!pr.waitFor(30, TimeUnit.SECONDS)) { - pr.destroy(); - throw new RuntimeException("Timeout while executing: " + cmd); - } - } - } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java new file mode 100644 index 000000000000..e36470ab32e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.MySqlUtils; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.testcontainers.containers.MySQLContainer; + +import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; +import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION; + +public class MySqlSslFullCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { + + private static MySqlUtils.Certificate certs; + private static final String PASSWORD = "Passw0rd"; + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + certs = MySqlUtils.getCertificate(container); + + var sslMode = ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_identity") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_certificate", certs.getClientCertificate()) + .put("client_key", certs.getClientKey()) + .put("client_key_password", PASSWORD) + .build(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) + .put("replication_method", ReplicationMethod.STANDARD) + .build()); + + try (final DSLContext dslContext = DSLContextFactory.create( + config.get(JdbcUtils.USERNAME_KEY).asText(), + config.get(JdbcUtils.PASSWORD_KEY).asText(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s/%s?%s", + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asText(), + config.get(JdbcUtils.DATABASE_KEY).asText(), + String.join("&", SSL_PARAMETERS), + String.join("&", SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION)), + SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); + + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java index 05104c470b11..ba68fb79e96c 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.source.mysql; import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; +import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; @@ -25,6 +26,10 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc container = new MySQLContainer<>("mysql:8.0"); container.start(); + var sslMode = ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "required") + .build(); + config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) @@ -32,6 +37,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) .put("replication_method", ReplicationMethod.STANDARD) .build()); @@ -43,7 +49,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get(JdbcUtils.HOST_KEY).asText(), config.get(JdbcUtils.PORT_KEY).asText(), config.get(JdbcUtils.DATABASE_KEY).asText(), - String.join("&", SSL_PARAMETERS)), + String.join("&", SSL_PARAMETERS), + String.join("&", SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION)), SQLDialect.MYSQL)) { final Database database = new Database(dslContext); From f5bd9464c0502fab00b1ea10a771a6ccd8b0c73f Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 26 Jul 2022 20:59:12 +0300 Subject: [PATCH 03/24] fixed code style --- .../main/java/io/airbyte/db/MySqlUtils.java | 13 ++++++------ .../util/MySqlSslConnectionUtils.java | 21 +++++++++---------- .../source/mysql/MySqlSource.java | 6 ++---- ...slFullCertificateSourceAcceptanceTest.java | 18 ++++++++-------- .../mysql/MySqlSslSourceAcceptanceTest.java | 4 ++-- 5 files changed, 29 insertions(+), 33 deletions(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java index dc78b48a491a..3b85f87c3b0b 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java @@ -5,9 +5,8 @@ package io.airbyte.db; import com.google.common.annotations.VisibleForTesting; -import org.testcontainers.containers.MySQLContainer; - import java.io.IOException; +import org.testcontainers.containers.MySQLContainer; public class MySqlUtils { @@ -18,17 +17,17 @@ public static Certificate getCertificate(final MySQLContainer container) thro // create root certificates container.execInContainer("sh", "-c", "openssl genrsa 2048 > /etc/mysql/newcerts/ca-key.pem"); container.execInContainer("sh", "-c", - "openssl req -new -x509 -nodes -days 1000 -key /etc/mysql/newcerts/ca-key.pem > /etc/mysql/newcerts/ca-cert.pem -subj \"/CN=127.0.0.1\""); + "openssl req -new -x509 -nodes -days 1000 -key /etc/mysql/newcerts/ca-key.pem > /etc/mysql/newcerts/ca-cert.pem -subj \"/CN=127.0.0.1\""); // create server certificates container.execInContainer("sh", "-c", - "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/server-key.pem > /etc/mysql/newcerts/server-req.pem -subj \"/CN=localhost\""); + "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/server-key.pem > /etc/mysql/newcerts/server-req.pem -subj \"/CN=localhost\""); container.execInContainer("sh", "-c", - "openssl x509 -req -in /etc/mysql/newcerts/server-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/server-cert.pem"); + "openssl x509 -req -in /etc/mysql/newcerts/server-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/server-cert.pem"); // create client certificates container.execInContainer("sh", "-c", - "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/client-key.pem > /etc/mysql/newcerts/client-req.pem -subj \"/CN=test\""); + "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/client-key.pem > /etc/mysql/newcerts/client-req.pem -subj \"/CN=test\""); container.execInContainer("sh", "-c", - "openssl x509 -req -in /etc/mysql/newcerts/client-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/client-cert.pem"); + "openssl x509 -req -in /etc/mysql/newcerts/client-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/client-cert.pem"); // add root and server certificates to config file container.execInContainer("sh", "-c", "sed -i '31 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); container.execInContainer("sh", "-c", "sed -i '32 a ssl-cert=/etc/mysql/newcerts/server-cert.pem' /etc/my.cnf"); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java index ea1a3826454f..77480e17ff0f 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -5,16 +5,15 @@ package io.airbyte.integrations.util; import com.fasterxml.jackson.databind.JsonNode; -import org.apache.commons.lang3.RandomStringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MySqlSslConnectionUtils { @@ -65,7 +64,7 @@ private static Map obtainConnectionFullOptions(final JsonNode en Map additionalParameters = new HashMap<>(); try { convertAndImportFullCertificate(encryption.get(PARAM_CA_CERTIFICATE).asText(), - encryption.get(PARAM_CLIENT_CERTIFICATE).asText(), encryption.get(PARAM_CLIENT_KEY).asText(), clientKeyPassword); + encryption.get(PARAM_CLIENT_CERTIFICATE).asText(), encryption.get(PARAM_CLIENT_KEY).asText(), clientKeyPassword); } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } @@ -96,17 +95,17 @@ private static void convertAndImportFullCertificate(final String caCertificate, final String clientCertificate, final String clientKey, final String clientKeyPassword) - throws IOException, InterruptedException { + throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); createCaCertificate(caCertificate, clientKeyPassword, run); createCertificateFile(CLIENT_CERTIFICARE_NAME, clientCertificate); createCertificateFile(CLIENT_KEY_NAME, clientKey); // add client certificate to the custom keystore runProcess("keytool -alias client-certificate -keystore " + CUSTOM_TRUST_STORE - + " -import -file " + CLIENT_CERTIFICARE_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); + + " -import -file " + CLIENT_CERTIFICARE_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); // add client key to the custom keystore runProcess("keytool -alias client-key -keystore " + CUSTOM_TRUST_STORE - + " -import -file " + CLIENT_KEY_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); + + " -import -file " + CLIENT_KEY_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); runProcess("rm " + CLIENT_KEY_NAME, run); updateTrustStoreSystemProperty(clientKeyPassword); @@ -114,7 +113,7 @@ private static void convertAndImportFullCertificate(final String caCertificate, private static void convertAndImportCaCertificate(final String caCertificate, final String clientKeyPassword) - throws IOException, InterruptedException { + throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); createCaCertificate(caCertificate, clientKeyPassword, run); updateTrustStoreSystemProperty(clientKeyPassword); @@ -123,11 +122,11 @@ private static void convertAndImportCaCertificate(final String caCertificate, private static void createCaCertificate(final String caCertificate, final String clientKeyPassword, final Runtime run) - throws IOException, InterruptedException { + throws IOException, InterruptedException { createCertificateFile(ROOT_CERTIFICARE_NAME, caCertificate); // add CA certificate to the custom keystore runProcess("keytool -import -alias root-certificate -keystore " + CUSTOM_TRUST_STORE - + " -file " + ROOT_CERTIFICARE_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); + + " -file " + ROOT_CERTIFICARE_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); } private static void updateTrustStoreSystemProperty(final String clientKeyPassword) { diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 6227e0d669d4..e0861dab28b9 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -37,9 +37,7 @@ import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.SyncMode; - import java.time.Duration; - import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -163,7 +161,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { } else { additionalParameters.putAll(obtainConnectionOptions(config.get(JdbcUtils.SSL_MODE_KEY))); jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) - .append(JdbcUtils.AMPERSAND); + .append(JdbcUtils.AMPERSAND); if (additionalParameters.isEmpty()) { jdbcUrl.append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); } else { @@ -172,7 +170,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { } } else { jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) - .append(JdbcUtils.AMPERSAND).append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); + .append(JdbcUtils.AMPERSAND).append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java index e36470ab32e4..9fdffd585376 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -4,6 +4,9 @@ package io.airbyte.integrations.source.mysql; +import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; +import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION; + import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; @@ -17,9 +20,6 @@ import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION; - public class MySqlSslFullCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { private static MySqlUtils.Certificate certs; @@ -32,12 +32,12 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc certs = MySqlUtils.getCertificate(container); var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_identity") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) - .build(); + .put(JdbcUtils.MODE_KEY, "verify_identity") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_certificate", certs.getClientCertificate()) + .put("client_key", certs.getClientKey()) + .put("client_key_password", PASSWORD) + .build(); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java index ba68fb79e96c..737953f3006f 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java @@ -27,8 +27,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc container.start(); var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "required") - .build(); + .put(JdbcUtils.MODE_KEY, "required") + .build(); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) From cfe583d53a97df196871bf9c5a88f1e5e338cf73 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Wed, 27 Jul 2022 10:30:04 +0300 Subject: [PATCH 04/24] updated run process timeout --- .../io/airbyte/integrations/util/MySqlSslConnectionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java index 77480e17ff0f..0c218d7f6a36 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -143,7 +143,7 @@ private static void createCertificateFile(String fileName, String fileValue) thr private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { final Process pr = run.exec(cmd); - if (!pr.waitFor(120, TimeUnit.SECONDS)) { + if (!pr.waitFor(30, TimeUnit.SECONDS)) { pr.destroy(); throw new RuntimeException("Timeout while executing: " + cmd); } From 6e2dbb3a005b8cd6fb9652424398b7f948fef263 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Thu, 28 Jul 2022 10:54:24 +0300 Subject: [PATCH 05/24] updated method for create keystore and updated tests --- .../util/MySqlSslConnectionUtils.java | 48 ++++++++++++------- ...slFullCertificateSourceAcceptanceTest.java | 9 +--- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java index 0c218d7f6a36..c34bf589926f 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -5,6 +5,8 @@ package io.airbyte.integrations.util; import com.fasterxml.jackson.databind.JsonNode; + +import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; @@ -29,6 +31,7 @@ public class MySqlSslConnectionUtils { public static final String CLIENT_KEY_STORE_URL = "clientCertificateKeyStoreUrl"; public static final String CLIENT_KEY_STORE_PASS = "clientCertificateKeyStorePassword"; public static final String CUSTOM_TRUST_STORE = "customtruststore"; + public static final String CUSTOM_KEY_STORE = "customkeystore"; public static final String SSL_MODE = "sslMode"; public static final String DISABLE = "disable"; public static final String VERIFY_CA = "VERIFY_CA"; @@ -64,13 +67,14 @@ private static Map obtainConnectionFullOptions(final JsonNode en Map additionalParameters = new HashMap<>(); try { convertAndImportFullCertificate(encryption.get(PARAM_CA_CERTIFICATE).asText(), - encryption.get(PARAM_CLIENT_CERTIFICATE).asText(), encryption.get(PARAM_CLIENT_KEY).asText(), clientKeyPassword); + encryption.get(PARAM_CLIENT_CERTIFICATE).asText(), + encryption.get(PARAM_CLIENT_KEY).asText(), clientKeyPassword); } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } additionalParameters.put(TRUST_KEY_STORE_URL, CUSTOM_TRUST_STORE); additionalParameters.put(TRUST_KEY_STORE_PASS, clientKeyPassword); - additionalParameters.put(CLIENT_KEY_STORE_URL, CUSTOM_TRUST_STORE); + additionalParameters.put(CLIENT_KEY_STORE_URL, CUSTOM_KEY_STORE); additionalParameters.put(CLIENT_KEY_STORE_PASS, clientKeyPassword); additionalParameters.put(SSL_MODE, method.toUpperCase()); return additionalParameters; @@ -97,26 +101,36 @@ private static void convertAndImportFullCertificate(final String caCertificate, final String clientKeyPassword) throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); - createCaCertificate(caCertificate, clientKeyPassword, run); - createCertificateFile(CLIENT_CERTIFICARE_NAME, clientCertificate); - createCertificateFile(CLIENT_KEY_NAME, clientKey); - // add client certificate to the custom keystore - runProcess("keytool -alias client-certificate -keystore " + CUSTOM_TRUST_STORE - + " -import -file " + CLIENT_CERTIFICARE_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); - // add client key to the custom keystore - runProcess("keytool -alias client-key -keystore " + CUSTOM_TRUST_STORE - + " -import -file " + CLIENT_KEY_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); - runProcess("rm " + CLIENT_KEY_NAME, run); - - updateTrustStoreSystemProperty(clientKeyPassword); + convertAndImportCaCertificate(caCertificate, clientKeyPassword); + LOGGER.error("==>> check if KEY store exist"); + File f = new File(System.getProperty("user.dir") + "/" + CUSTOM_KEY_STORE); + if (!f.exists()) { + LOGGER.error("==> need create KEY store!!"); + createCertificateFile(CLIENT_CERTIFICARE_NAME, clientCertificate); + createCertificateFile(CLIENT_KEY_NAME, clientKey); + // add client certificate to the custom keystore + runProcess("openssl pkcs12 -export -in client-cert.pem -inkey client-key.pem -out certificate.p12 -name \"certificate\" -passout pass:Passw0rd", run); + // add client key to the custom keystore + runProcess("keytool -importkeystore -srckeystore certificate.p12 -srcstoretype pkcs12 -destkeystore customkeystore -srcstorepass Passw0rd -deststoretype JKS -deststorepass Passw0rd", run); + runProcess("rm " + CLIENT_KEY_NAME, run); + + String result = System.getProperty("user.dir") + "/" + CUSTOM_KEY_STORE; + System.setProperty("javax.net.ssl.keyStore", result); + System.setProperty("javax.net.ssl.keyStorePassword", clientKeyPassword); + } } private static void convertAndImportCaCertificate(final String caCertificate, final String clientKeyPassword) throws IOException, InterruptedException { - final Runtime run = Runtime.getRuntime(); - createCaCertificate(caCertificate, clientKeyPassword, run); - updateTrustStoreSystemProperty(clientKeyPassword); + LOGGER.error("==>> check if TRUST store exist"); + File f = new File(System.getProperty("user.dir") + "/" + CUSTOM_TRUST_STORE); + if (!f.exists()) { + LOGGER.error("==> need create TRUST tore!!"); + final Runtime run = Runtime.getRuntime(); + createCaCertificate(caCertificate, clientKeyPassword, run); + updateTrustStoreSystemProperty(clientKeyPassword); + } } private static void createCaCertificate(final String caCertificate, diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java index 9fdffd585376..391a28b71e1b 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -4,9 +4,6 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION; - import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; @@ -54,12 +51,10 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get(JdbcUtils.USERNAME_KEY).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s?%s", + String.format("jdbc:mysql://%s:%s/%s", config.get(JdbcUtils.HOST_KEY).asText(), config.get(JdbcUtils.PORT_KEY).asText(), - config.get(JdbcUtils.DATABASE_KEY).asText(), - String.join("&", SSL_PARAMETERS), - String.join("&", SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION)), + config.get(JdbcUtils.DATABASE_KEY).asText()), SQLDialect.MYSQL)) { final Database database = new Database(dslContext); From ea08e2b0a424521ab245ea815d2cd39f73943da5 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 9 Aug 2022 12:37:03 +0300 Subject: [PATCH 06/24] updated normalization version for postgres destination --- .../main/java/io/airbyte/db/MySqlUtils.java | 19 +++-- .../util/MySqlSslConnectionUtils.java | 66 ++++++++--------- .../source/mysql/MySqlSource.java | 1 + ...slFullCertificateSourceAcceptanceTest.java | 73 ------------------- build.gradle | 2 +- 5 files changed, 45 insertions(+), 116 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java index 3b85f87c3b0b..b46e08ade22d 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java @@ -13,11 +13,10 @@ public class MySqlUtils { @VisibleForTesting public static Certificate getCertificate(final MySQLContainer container) throws IOException, InterruptedException { container.execInContainer("sh", "-c", "mkdir -p /etc/mysql/newcerts"); - container.execInContainer("sh", "-c", "chown -R test:test /etc/mysql/newcerts"); // create root certificates container.execInContainer("sh", "-c", "openssl genrsa 2048 > /etc/mysql/newcerts/ca-key.pem"); container.execInContainer("sh", "-c", - "openssl req -new -x509 -nodes -days 1000 -key /etc/mysql/newcerts/ca-key.pem > /etc/mysql/newcerts/ca-cert.pem -subj \"/CN=127.0.0.1\""); + "openssl req -new -x509 -nodes -days 1000 -key /etc/mysql/newcerts/ca-key.pem > /etc/mysql/newcerts/ca-cert.pem -subj \"/CN=localhost\""); // create server certificates container.execInContainer("sh", "-c", "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/server-key.pem > /etc/mysql/newcerts/server-req.pem -subj \"/CN=localhost\""); @@ -29,14 +28,18 @@ public static Certificate getCertificate(final MySQLContainer container) thro container.execInContainer("sh", "-c", "openssl x509 -req -in /etc/mysql/newcerts/client-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/client-cert.pem"); // add root and server certificates to config file - container.execInContainer("sh", "-c", "sed -i '31 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '32 a ssl-cert=/etc/mysql/newcerts/server-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '33 a ssl-key=/etc/mysql/newcerts/server-key.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '31 a ssl' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '32 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '33 a ssl-cert=/etc/mysql/newcerts/server-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '34 a ssl-key=/etc/mysql/newcerts/server-key.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '35 a require_secure_transport=ON' /etc/my.cnf"); // add client certificates to config file - container.execInContainer("sh", "-c", "sed -i '38 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '39 a ssl-cert=/etc/mysql/newcerts/client-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '40 a ssl-key=/etc/mysql/newcerts/client-key.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '39 a [client]' /etc/mysql/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '40 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '41 a ssl-cert=/etc/mysql/newcerts/client-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '42 a ssl-key=/etc/mysql/newcerts/client-key.pem' /etc/my.cnf"); // restart server + container.execInContainer("sh", "-c", "chown -R mysql:mysql /etc/mysql/newcerts"); container.execInContainer("sh", "-c", "service mysqld restart"); // copy root certificate and client certificates var caCert = container.execInContainer("sh", "-c", "cat /etc/mysql/newcerts/ca-cert.pem").getStdout().trim(); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java index c34bf589926f..1fc6db15d9fb 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; -import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; @@ -30,15 +29,17 @@ public class MySqlSslConnectionUtils { public static final String TRUST_KEY_STORE_PASS = "trustCertificateKeyStorePassword"; public static final String CLIENT_KEY_STORE_URL = "clientCertificateKeyStoreUrl"; public static final String CLIENT_KEY_STORE_PASS = "clientCertificateKeyStorePassword"; - public static final String CUSTOM_TRUST_STORE = "customtruststore"; - public static final String CUSTOM_KEY_STORE = "customkeystore"; + public static final String CUSTOM_TRUST_STORE = "customtruststore.jks"; + public static final String CUSTOM_KEY_STORE = "customkeystore.jks"; public static final String SSL_MODE = "sslMode"; public static final String DISABLE = "disable"; public static final String VERIFY_CA = "VERIFY_CA"; public static final String VERIFY_IDENTITY = "VERIFY_IDENTITY"; - public static final String ROOT_CERTIFICARE_NAME = "ca.pem"; + public static final String ROOT_CERTIFICARE_NAME = "ca-cert.pem"; + public static final String ROOT_CERTIFICARE_DER_NAME = "ca-cert.der"; public static final String CLIENT_CERTIFICARE_NAME = "client-cert.pem"; public static final String CLIENT_KEY_NAME = "client-key.pem"; + public static final String CLIENT_CERT_P12 = "certificate.p12"; public static Map obtainConnectionOptions(final JsonNode encryption) { Map additionalParameters = new HashMap<>(); @@ -72,11 +73,17 @@ private static Map obtainConnectionFullOptions(final JsonNode en } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } - additionalParameters.put(TRUST_KEY_STORE_URL, CUSTOM_TRUST_STORE); + additionalParameters.put(TRUST_KEY_STORE_URL, "file:" + CUSTOM_TRUST_STORE); additionalParameters.put(TRUST_KEY_STORE_PASS, clientKeyPassword); - additionalParameters.put(CLIENT_KEY_STORE_URL, CUSTOM_KEY_STORE); + additionalParameters.put(CLIENT_KEY_STORE_URL, "file:" + CUSTOM_KEY_STORE); additionalParameters.put(CLIENT_KEY_STORE_PASS, clientKeyPassword); - additionalParameters.put(SSL_MODE, method.toUpperCase()); + additionalParameters.put(SSL_MODE, VERIFY_IDENTITY); + + String result = CUSTOM_KEY_STORE; + updateTrustStoreSystemProperty(clientKeyPassword); + System.setProperty("javax.net.ssl.keyStore", result); + System.setProperty("javax.net.ssl.keyStorePassword", clientKeyPassword); + return additionalParameters; } @@ -89,9 +96,12 @@ private static Map obtainConnectionCaOptions(final JsonNode encr } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } - additionalParameters.put(TRUST_KEY_STORE_URL, CUSTOM_TRUST_STORE); + additionalParameters.put(TRUST_KEY_STORE_URL, "file:" + CUSTOM_TRUST_STORE); additionalParameters.put(TRUST_KEY_STORE_PASS, clientKeyPassword); - additionalParameters.put(SSL_MODE, method.toUpperCase()); + additionalParameters.put(SSL_MODE, VERIFY_CA); + + updateTrustStoreSystemProperty(clientKeyPassword); + return additionalParameters; } @@ -102,35 +112,22 @@ private static void convertAndImportFullCertificate(final String caCertificate, throws IOException, InterruptedException { final Runtime run = Runtime.getRuntime(); convertAndImportCaCertificate(caCertificate, clientKeyPassword); - LOGGER.error("==>> check if KEY store exist"); - File f = new File(System.getProperty("user.dir") + "/" + CUSTOM_KEY_STORE); - if (!f.exists()) { - LOGGER.error("==> need create KEY store!!"); - createCertificateFile(CLIENT_CERTIFICARE_NAME, clientCertificate); - createCertificateFile(CLIENT_KEY_NAME, clientKey); - // add client certificate to the custom keystore - runProcess("openssl pkcs12 -export -in client-cert.pem -inkey client-key.pem -out certificate.p12 -name \"certificate\" -passout pass:Passw0rd", run); - // add client key to the custom keystore - runProcess("keytool -importkeystore -srckeystore certificate.p12 -srcstoretype pkcs12 -destkeystore customkeystore -srcstorepass Passw0rd -deststoretype JKS -deststorepass Passw0rd", run); - runProcess("rm " + CLIENT_KEY_NAME, run); - - String result = System.getProperty("user.dir") + "/" + CUSTOM_KEY_STORE; - System.setProperty("javax.net.ssl.keyStore", result); - System.setProperty("javax.net.ssl.keyStorePassword", clientKeyPassword); - } + createCertificateFile(CLIENT_CERTIFICARE_NAME, clientCertificate); + createCertificateFile(CLIENT_KEY_NAME, clientKey); + // add client certificate to the custom keystore + runProcess("openssl pkcs12 -export -in " + CLIENT_CERTIFICARE_NAME + " -inkey " + CLIENT_KEY_NAME + + " -out " + CLIENT_CERT_P12 + " -name \"certificate\" -passout pass:" + clientKeyPassword, run); + // add client key to the custom keystore + runProcess("keytool -importkeystore -srckeystore " + CLIENT_CERT_P12 + + " -srcstoretype pkcs12 -destkeystore " + CUSTOM_KEY_STORE + " -srcstorepass " + clientKeyPassword + + " -deststoretype JKS -deststorepass " + clientKeyPassword + " -noprompt", run); } private static void convertAndImportCaCertificate(final String caCertificate, final String clientKeyPassword) throws IOException, InterruptedException { - LOGGER.error("==>> check if TRUST store exist"); - File f = new File(System.getProperty("user.dir") + "/" + CUSTOM_TRUST_STORE); - if (!f.exists()) { - LOGGER.error("==> need create TRUST tore!!"); final Runtime run = Runtime.getRuntime(); createCaCertificate(caCertificate, clientKeyPassword, run); - updateTrustStoreSystemProperty(clientKeyPassword); - } } private static void createCaCertificate(final String caCertificate, @@ -139,12 +136,13 @@ private static void createCaCertificate(final String caCertificate, throws IOException, InterruptedException { createCertificateFile(ROOT_CERTIFICARE_NAME, caCertificate); // add CA certificate to the custom keystore - runProcess("keytool -import -alias root-certificate -keystore " + CUSTOM_TRUST_STORE - + " -file " + ROOT_CERTIFICARE_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); + runProcess("openssl x509 -outform der -in " + ROOT_CERTIFICARE_NAME + " -out " + ROOT_CERTIFICARE_DER_NAME, run); + runProcess("keytool -importcert -alias root-certificate -keystore " + CUSTOM_TRUST_STORE + + " -file " + ROOT_CERTIFICARE_DER_NAME + " -storepass " + clientKeyPassword + " -noprompt", run); } private static void updateTrustStoreSystemProperty(final String clientKeyPassword) { - String result = System.getProperty("user.dir") + "/" + CUSTOM_TRUST_STORE; + String result = CUSTOM_TRUST_STORE; System.setProperty("javax.net.ssl.trustStore", result); System.setProperty("javax.net.ssl.trustStorePassword", clientKeyPassword); } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index e0861dab28b9..ea5533498d6d 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -160,6 +160,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { jdbcUrl.append(JdbcUtils.AMPERSAND).append("sslMode=DISABLE"); } else { additionalParameters.putAll(obtainConnectionOptions(config.get(JdbcUtils.SSL_MODE_KEY))); + additionalParameters.put("fallbackToSystemTrustStore", "false"); jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) .append(JdbcUtils.AMPERSAND); if (additionalParameters.isEmpty()) { diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java deleted file mode 100644 index 391a28b71e1b..000000000000 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; - -public class MySqlSslFullCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { - - private static MySqlUtils.Certificate certs; - private static final String PASSWORD = "Passw0rd"; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - certs = MySqlUtils.getCertificate(container); - - var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_identity") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) - .build(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.STANDARD) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } - } - -} diff --git a/build.gradle b/build.gradle index 0615fa7671c6..72463569096a 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ buildscript { plugins { id 'base' id 'pmd' - id 'com.diffplug.spotless' version '6.7.1' + id 'com.diffplug.spotless' version '6.0.0' id 'com.github.hierynomus.license' version '0.16.1' id 'com.github.spotbugs' version '5.0.6' id 'version-catalog' From ac0f65190e96e1db41db73f11a3cae2d219a140f Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 9 Aug 2022 12:37:56 +0300 Subject: [PATCH 07/24] updated normalization version for postgres destination --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 72463569096a..0615fa7671c6 100644 --- a/build.gradle +++ b/build.gradle @@ -15,7 +15,7 @@ buildscript { plugins { id 'base' id 'pmd' - id 'com.diffplug.spotless' version '6.0.0' + id 'com.diffplug.spotless' version '6.7.1' id 'com.github.hierynomus.license' version '0.16.1' id 'com.github.spotbugs' version '5.0.6' id 'version-catalog' From a10374dff7efa2b03e4308d5aacd588c7ad35b59 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 9 Aug 2022 15:09:33 +0300 Subject: [PATCH 08/24] added tests for connection with certificates --- ...slFullCertificateSourceAcceptanceTest.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java new file mode 100644 index 000000000000..f4b3b9fe09c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.MySqlUtils; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.testcontainers.containers.MySQLContainer; + +public class MySqlSslFullCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { + + private static MySqlUtils.Certificate certs; + private static final String PASSWORD = "Passw0rd"; + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + // + container = new MySQLContainer<>("mysql:8.0"); +// container = new MySQLContainer<>("mysql:8.0") +// .withClasspathResourceMapping("my1.cnf", "/etc/my1.cnf", BindMode.READ_ONLY) +// .withCommand("mysql --defaults-file=/etc/my1.cnf"); + container.start(); + addTestData(container); + certs = MySqlUtils.getCertificate(container); + + var sslMode = ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_identity") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_certificate", certs.getClientCertificate()) + .put("client_key", certs.getClientKey()) + .put("client_key_password", PASSWORD) + .build(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) + .put("replication_method", ReplicationMethod.STANDARD) + .build()); + } + + private void addTestData(MySQLContainer container) throws Exception { + try (final DSLContext dslContext = DSLContextFactory.create( + container.getUsername(), + container.getPassword(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s/%s", + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); + + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } + } + +} From 9cc861deab1ad5a8142bb05c4592c7b339df4b2a Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 9 Aug 2022 18:18:17 +0300 Subject: [PATCH 09/24] updated tests for connection with full certificates and added tests for CA certificate --- .../main/java/io/airbyte/db/MySqlUtils.java | 57 +++++++------- ...lSslCaCertificateSourceAcceptanceTest.java | 75 +++++++++++++++++++ ...slFullCertificateSourceAcceptanceTest.java | 7 +- 3 files changed, 104 insertions(+), 35 deletions(-) create mode 100644 airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java index b46e08ade22d..eb6306a9f852 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java @@ -11,41 +11,32 @@ public class MySqlUtils { @VisibleForTesting - public static Certificate getCertificate(final MySQLContainer container) throws IOException, InterruptedException { - container.execInContainer("sh", "-c", "mkdir -p /etc/mysql/newcerts"); - // create root certificates - container.execInContainer("sh", "-c", "openssl genrsa 2048 > /etc/mysql/newcerts/ca-key.pem"); - container.execInContainer("sh", "-c", - "openssl req -new -x509 -nodes -days 1000 -key /etc/mysql/newcerts/ca-key.pem > /etc/mysql/newcerts/ca-cert.pem -subj \"/CN=localhost\""); - // create server certificates - container.execInContainer("sh", "-c", - "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/server-key.pem > /etc/mysql/newcerts/server-req.pem -subj \"/CN=localhost\""); - container.execInContainer("sh", "-c", - "openssl x509 -req -in /etc/mysql/newcerts/server-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/server-cert.pem"); - // create client certificates - container.execInContainer("sh", "-c", - "openssl req -newkey rsa:2048 -days 1000 -nodes -keyout /etc/mysql/newcerts/client-key.pem > /etc/mysql/newcerts/client-req.pem -subj \"/CN=test\""); - container.execInContainer("sh", "-c", - "openssl x509 -req -in /etc/mysql/newcerts/client-req.pem -days 1000 -CA /etc/mysql/newcerts/ca-cert.pem -CAkey /etc/mysql/newcerts/ca-key.pem -set_serial 01 > /etc/mysql/newcerts/client-cert.pem"); + public static Certificate getCertificate(final MySQLContainer container, + final boolean useAllCertificates) + throws IOException, InterruptedException { // add root and server certificates to config file container.execInContainer("sh", "-c", "sed -i '31 a ssl' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '32 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '33 a ssl-cert=/etc/mysql/newcerts/server-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '34 a ssl-key=/etc/mysql/newcerts/server-key.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '32 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '33 a ssl-cert=/var/lib/mysql/server-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '34 a ssl-key=/var/lib/mysql/server-key.pem' /etc/my.cnf"); container.execInContainer("sh", "-c", "sed -i '35 a require_secure_transport=ON' /etc/my.cnf"); // add client certificates to config file - container.execInContainer("sh", "-c", "sed -i '39 a [client]' /etc/mysql/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '40 a ssl-ca=/etc/mysql/newcerts/ca-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '41 a ssl-cert=/etc/mysql/newcerts/client-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '42 a ssl-key=/etc/mysql/newcerts/client-key.pem' /etc/my.cnf"); - // restart server - container.execInContainer("sh", "-c", "chown -R mysql:mysql /etc/mysql/newcerts"); - container.execInContainer("sh", "-c", "service mysqld restart"); + if (useAllCertificates) { + container.execInContainer("sh", "-c", "sed -i '39 a [client]' /etc/mysql/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '40 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '41 a ssl-cert=/var/lib/mysql/client-cert.pem' /etc/my.cnf"); + container.execInContainer("sh", "-c", "sed -i '42 a ssl-key=/var/lib/mysql/client-key.pem' /etc/my.cnf"); + } // copy root certificate and client certificates - var caCert = container.execInContainer("sh", "-c", "cat /etc/mysql/newcerts/ca-cert.pem").getStdout().trim(); - var clientKey = container.execInContainer("sh", "-c", "cat /etc/mysql/newcerts/client-key.pem").getStdout().trim(); - var clientCert = container.execInContainer("sh", "-c", "cat /etc/mysql/newcerts/client-cert.pem").getStdout().trim(); - return new Certificate(caCert, clientCert, clientKey); + var caCert = container.execInContainer("sh", "-c", "cat /var/lib/mysql/ca.pem").getStdout().trim(); + + if (useAllCertificates) { + var clientKey = container.execInContainer("sh", "-c", "cat /var/lib/mysql/client-key.pem").getStdout().trim(); + var clientCert = container.execInContainer("sh", "-c", "cat /var/lib/mysql/client-cert.pem").getStdout().trim(); + return new Certificate(caCert, clientCert, clientKey); + } else { + return new Certificate(caCert); + } } public static class Certificate { @@ -54,6 +45,12 @@ public static class Certificate { private final String clientCertificate; private final String clientKey; + public Certificate(final String caCertificate) { + this.caCertificate = caCertificate; + this.clientCertificate = null; + this.clientKey = null; + } + public Certificate(final String caCertificate, final String clientCertificate, final String clientKey) { this.caCertificate = caCertificate; this.clientCertificate = clientCertificate; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java new file mode 100644 index 000000000000..817a01a8b9ac --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.MySqlUtils; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.testcontainers.containers.MySQLContainer; + +public class MySqlSslCaCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { + + private static MySqlUtils.Certificate certs; + private static final String PASSWORD = "Passw0rd"; + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + addTestData(container); + certs = MySqlUtils.getCertificate(container, false); + + var sslMode = ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_key_password", PASSWORD) + .build(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) + .put("replication_method", ReplicationMethod.STANDARD) + .build()); + } + + private void addTestData(MySQLContainer container) throws Exception { + try (final DSLContext dslContext = DSLContextFactory.create( + container.getUsername(), + container.getPassword(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s/%s", + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); + + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java index f4b3b9fe09c6..37bfeffd289e 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -24,14 +24,11 @@ public class MySqlSslFullCertificateSourceAcceptanceTest extends MySqlSourceAcce @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - // + container = new MySQLContainer<>("mysql:8.0"); -// container = new MySQLContainer<>("mysql:8.0") -// .withClasspathResourceMapping("my1.cnf", "/etc/my1.cnf", BindMode.READ_ONLY) -// .withCommand("mysql --defaults-file=/etc/my1.cnf"); container.start(); addTestData(container); - certs = MySqlUtils.getCertificate(container); + certs = MySqlUtils.getCertificate(container, true); var sslMode = ImmutableMap.builder() .put(JdbcUtils.MODE_KEY, "verify_identity") From 2575317c45d6335e1213c637d0feb686a6d5cd00 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Wed, 10 Aug 2022 10:53:25 +0300 Subject: [PATCH 10/24] updated tests --- .../source/mysql/MySqlSource.java | 1 - ...SqlSslCertificateSourceAcceptanceTest.java | 73 +++++++++++++++++++ ...lSslCaCertificateSourceAcceptanceTest.java | 60 +-------------- ...slFullCertificateSourceAcceptanceTest.java | 60 +-------------- .../mysql/MySqlSslSourceAcceptanceTest.java | 9 +-- 5 files changed, 81 insertions(+), 122 deletions(-) create mode 100644 airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index ea5533498d6d..e0861dab28b9 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -160,7 +160,6 @@ public JsonNode toDatabaseConfig(final JsonNode config) { jdbcUrl.append(JdbcUtils.AMPERSAND).append("sslMode=DISABLE"); } else { additionalParameters.putAll(obtainConnectionOptions(config.get(JdbcUtils.SSL_MODE_KEY))); - additionalParameters.put("fallbackToSystemTrustStore", "false"); jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) .append(JdbcUtils.AMPERSAND); if (additionalParameters.isEmpty()) { diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java new file mode 100644 index 000000000000..d3d58fa8098e --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.MySqlUtils; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.testcontainers.containers.MySQLContainer; + +public abstract class AbstractMySqlSslCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { + + protected static MySqlUtils.Certificate certs; + protected static final String PASSWORD = "Passw0rd"; + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + addTestData(container); + certs = MySqlUtils.getCertificate(container, true); + + var sslMode = getSslConfig(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) + .put("replication_method", ReplicationMethod.STANDARD) + .build()); + } + + public abstract ImmutableMap getSslConfig(); + + private void addTestData(MySQLContainer container) throws Exception { + try (final DSLContext dslContext = DSLContextFactory.create( + container.getUsername(), + container.getPassword(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s/%s", + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); + + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java index 817a01a8b9ac..9130876847c3 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java @@ -5,71 +5,17 @@ package io.airbyte.integrations.source.mysql; import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; -public class MySqlSslCaCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { - - private static MySqlUtils.Certificate certs; - private static final String PASSWORD = "Passw0rd"; +public class MySqlSslCaCertificateSourceAcceptanceTest extends AbstractMySqlSslCertificateSourceAcceptanceTest { @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - addTestData(container); - certs = MySqlUtils.getCertificate(container, false); - - var sslMode = ImmutableMap.builder() + public ImmutableMap getSslConfig() { + return ImmutableMap.builder() .put(JdbcUtils.MODE_KEY, "verify_ca") .put("ca_certificate", certs.getCaCertificate()) .put("client_key_password", PASSWORD) .build(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.STANDARD) - .build()); - } - - private void addTestData(MySQLContainer container) throws Exception { - try (final DSLContext dslContext = DSLContextFactory.create( - container.getUsername(), - container.getPassword(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s", - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java index 37bfeffd289e..39e6029cd1b3 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -5,73 +5,19 @@ package io.airbyte.integrations.source.mysql; import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; -public class MySqlSslFullCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { - - private static MySqlUtils.Certificate certs; - private static final String PASSWORD = "Passw0rd"; +public class MySqlSslFullCertificateSourceAcceptanceTest extends AbstractMySqlSslCertificateSourceAcceptanceTest { @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - addTestData(container); - certs = MySqlUtils.getCertificate(container, true); - - var sslMode = ImmutableMap.builder() + public ImmutableMap getSslConfig() { + return ImmutableMap.builder() .put(JdbcUtils.MODE_KEY, "verify_identity") .put("ca_certificate", certs.getCaCertificate()) .put("client_certificate", certs.getClientCertificate()) .put("client_key", certs.getClientKey()) .put("client_key_password", PASSWORD) .build(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.STANDARD) - .build()); - } - - private void addTestData(MySQLContainer container) throws Exception { - try (final DSLContext dslContext = DSLContextFactory.create( - container.getUsername(), - container.getPassword(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s", - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java index 737953f3006f..9d8dfdb6bd86 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java @@ -4,9 +4,6 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION; - import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; @@ -45,12 +42,10 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get(JdbcUtils.USERNAME_KEY).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s?%s", + String.format("jdbc:mysql://%s:%s/%s", config.get(JdbcUtils.HOST_KEY).asText(), config.get(JdbcUtils.PORT_KEY).asText(), - config.get(JdbcUtils.DATABASE_KEY).asText(), - String.join("&", SSL_PARAMETERS), - String.join("&", SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION)), + config.get(JdbcUtils.DATABASE_KEY).asText()), SQLDialect.MYSQL)) { final Database database = new Database(dslContext); From bd1fcb5d2bd2bd34a8b5c9d6e2cf3cfaa24eec5d Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Wed, 10 Aug 2022 11:52:10 +0300 Subject: [PATCH 11/24] updated source-mysql-strict-encrypt and updated versions --- .../source-mysql-strict-encrypt/Dockerfile | 2 +- .../MySqlStrictEncryptSource.java | 7 ++ ...cateStrictEncryptSourceAcceptanceTest.java | 73 +++++++++++++ ...cateStrictEncryptSourceAcceptanceTest.java | 21 ++++ ...cateStrictEncryptSourceAcceptanceTest.java | 23 ++++ ...ySqlStrictEncryptSourceAcceptanceTest.java | 5 + .../src/test/resources/expected_spec.json | 103 +++++++++++++++++- .../connectors/source-mysql/Dockerfile | 2 +- .../source-mysql/src/main/resources/spec.json | 2 +- docs/integrations/sources/mysql.md | 99 ++++++++--------- 10 files changed, 284 insertions(+), 53 deletions(-) create mode 100644 airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index cc998b59af85..370187835e0b 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.1 +LABEL io.airbyte.version=0.6.2 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java index 5c5320f33b0d..bc0e4b77b209 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.source.mysql_strict_encrypt; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcUtils; @@ -34,6 +35,12 @@ public ConnectorSpecification modifySpec(final ConnectorSpecification originalSp // SSL property should be enabled by default for secure versions of connectors // that can be used in the Airbyte cloud. User should not be able to change this property. ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.SSL_KEY); + final ArrayNode modifiedSslModes = spec.getConnectionSpecification().get("properties").get("ssl_mode").get("oneOf").deepCopy(); + // Assume that the first items is the "disable" and "preferred" options; remove these options + modifiedSslModes.remove(1); + modifiedSslModes.remove(0); + ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).remove("oneOf"); + ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).put("oneOf", modifiedSslModes); return spec; } diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java new file mode 100644 index 000000000000..e7c26ace903d --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql_strict_encrypt; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.MySqlUtils; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.testcontainers.containers.MySQLContainer; + +public abstract class AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest extends MySqlStrictEncryptSourceAcceptanceTest { + + protected static MySqlUtils.Certificate certs; + protected static final String PASSWORD = "Passw0rd"; + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + addTestData(container); + certs = MySqlUtils.getCertificate(container, true); + + var sslMode = getSslConfig(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) + .put("replication_method", ReplicationMethod.STANDARD) + .build()); + } + + public abstract ImmutableMap getSslConfig(); + + private void addTestData(MySQLContainer container) throws Exception { + try (final DSLContext dslContext = DSLContextFactory.create( + container.getUsername(), + container.getPassword(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s/%s", + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); + + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java new file mode 100644 index 000000000000..2c13ade52f10 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql_strict_encrypt; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.db.jdbc.JdbcUtils; + +public class MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest extends AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest { + + @Override + public ImmutableMap getSslConfig() { + return ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_key_password", PASSWORD) + .build(); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java new file mode 100644 index 000000000000..0d7029caef60 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql_strict_encrypt; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.db.jdbc.JdbcUtils; + +public class MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest extends AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest { + + @Override + public ImmutableMap getSslConfig() { + return ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_identity") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_certificate", certs.getClientCertificate()) + .put("client_key", certs.getClientKey()) + .put("client_key_password", PASSWORD) + .build(); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java index ef8e449fd6af..481b23879119 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java @@ -45,12 +45,17 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc container = new MySQLContainer<>("mysql:8.0"); container.start(); + var sslMode = ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "required") + .build(); + config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) .put("replication_method", MySqlSource.ReplicationMethod.STANDARD) .build()); diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index 1bd91e3f0f91..eee88512d3c6 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -47,11 +47,112 @@ "type": "string", "order": 5 }, + "ssl_mode": { + "title": "SSL modes", + "description": "SSL modes", + "type": "object", + "order": 7, + "oneOf": [ + { + "title": "Require", + "description": "Require SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "required", + "enum": ["required"], + "default": "required", + "order": 0 + } + } + }, + { + "title": "Require and Verify CA", + "description": "Verify CA SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "verify_ca", + "enum": ["verify_ca"], + "default": "verify_ca", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_key_password": { + "type": "string", + "title": "Client key password (Optional)", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 2 + } + } + }, + { + "title": "Require and Verify Identity", + "description": "Verify-full SSL mode.", + "required": [ + "mode", + "ca_certificate", + "client_certificate", + "client_key" + ], + "properties": { + "mode": { + "type": "string", + "const": "verify_identity", + "enum": ["verify_identity"], + "default": "verify_identity", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate", + "airbyte_secret": true, + "multiline": true, + "order": 2 + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key", + "airbyte_secret": true, + "multiline": true, + "order": 3 + }, + "client_key_password": { + "type": "string", + "title": "Client key password (Optional)", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + } + ] + }, "replication_method": { "type": "string", "title": "Replication Method", "description": "Replication method which is used for data extraction from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "order": 7, + "order": 8, "default": "STANDARD", "enum": ["STANDARD", "CDC"] } diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index b9465054f44a..54ce574ace98 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.1 +LABEL io.airbyte.version=0.6.2 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index a47f058defbe..ea303529fa34 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -187,7 +187,7 @@ "type": "string", "title": "Replication Method", "description": "Replication method which is used for data extraction from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "order": 7, + "order": 8, "default": "STANDARD", "enum": ["STANDARD", "CDC"] } diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 537e0338005d..22e0b6643b74 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -183,52 +183,53 @@ If you do not see a type in this list, assume that it is coerced into a string. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| -| 0.6.1 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | -| 0.6.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | -| 0.5.17 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.5.16 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.5.15 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | -| 0.5.13 | 2022-06-21 | [13945](https://github.com/airbytehq/airbyte/pull/13945) | Aligned datatype test | -| 0.5.12 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.5.11 | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544) | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads. | -| 0.5.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.5.9 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.5.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.5.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.5.4 | 2022-02-11 | [10251](https://github.com/airbytehq/airbyte/issues/10251) | bug Source MySQL CDC: sync failed when has Zero-date value in mandatory column | -| 0.5.2 | 2021-12-14 | [6425](https://github.com/airbytehq/airbyte/issues/6425) | MySQL CDC sync fails because starting binlog position not found in DB | -| 0.5.1 | 2021-12-13 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | -| 0.5.0 | 2021-12-11 | [7970](https://github.com/airbytehq/airbyte/pull/7970) | Support all MySQL types | -| 0.4.13 | 2021-12-03 | [8335](https://github.com/airbytehq/airbyte/pull/8335) | Source-MySql: do not check cdc required param binlog_row_image for standard replication | -| 0.4.12 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.4.11 | 2021-11-19 | [8047](https://github.com/airbytehq/airbyte/pull/8047) | Source MySQL: transform binary data base64 format | -| 0.4.10 | 2021-11-15 | [7820](https://github.com/airbytehq/airbyte/pull/7820) | Added basic performance test | -| 0.4.9 | 2021-11-02 | [7559](https://github.com/airbytehq/airbyte/pull/7559) | Correctly process large unsigned short integer values which may fall outside java's `Short` data type capability | -| 0.4.8 | 2021-09-16 | [6093](https://github.com/airbytehq/airbyte/pull/6093) | Improve reliability of processing various data types like decimals, dates, datetime, binary, and text | -| 0.4.7 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | -| 0.4.6 | 2021-09-29 | [6510](https://github.com/airbytehq/airbyte/pull/6510) | Support SSL connection | -| 0.4.5 | 2021-09-17 | [6146](https://github.com/airbytehq/airbyte/pull/6146) | Added option to connect to DB via SSH | -| 0.4.1 | 2021-07-23 | [4956](https://github.com/airbytehq/airbyte/pull/4956) | Fix log link | -| 0.3.7 | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE\_ENTRYPOINT for Kubernetes support | -| 0.3.6 | 2021-06-09 | [3966](https://github.com/airbytehq/airbyte/pull/3966) | Fix excessive logging for CDC method | -| 0.3.5 | 2021-06-07 | [3890](https://github.com/airbytehq/airbyte/pull/3890) | Fix CDC handle tinyint\(1\) and boolean types | -| 0.3.4 | 2021-06-04 | [3846](https://github.com/airbytehq/airbyte/pull/3846) | Fix max integer value failure | -| 0.3.3 | 2021-06-02 | [3789](https://github.com/airbytehq/airbyte/pull/3789) | MySQL CDC poll wait 5 minutes when not received a single record | -| 0.3.2 | 2021-06-01 | [3757](https://github.com/airbytehq/airbyte/pull/3757) | MySQL CDC poll 5s to 5 min | -| 0.3.1 | 2021-06-01 | [3505](https://github.com/airbytehq/airbyte/pull/3505) | Implemented MySQL CDC | -| 0.3.0 | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990) | Support namespaces | -| 0.2.5 | 2021-04-15 | [2899](https://github.com/airbytehq/airbyte/pull/2899) | Fix bug in tests | -| 0.2.4 | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600) | Add NCHAR and NVCHAR support to DB and cursor type casting | -| 0.2.3 | 2021-03-26 | [2611](https://github.com/airbytehq/airbyte/pull/2611) | Add an optional `jdbc_url_params` in parameters | -| 0.2.2 | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460) | Destination supports destination sync mode | -| 0.2.1 | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488) | Sources support primary keys | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.1.10 | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887) | Migrate AbstractJdbcSource to use iterators | -| 0.1.9 | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746) | Fix NPE in State Decorator | -| 0.1.8 | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724) | Fix JdbcSource handling of tables with same names in different schemas | -| 0.1.7 | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655) | Fix JdbcSource OOM | -| 0.1.6 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySQL to use new JdbcSource | -| 0.1.5 | 2020-12-11 | [1267](https://github.com/airbytehq/airbyte/pull/1267) | Support incremental sync | -| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.6.2 | 2022-08-10 | [15044](https://github.com/airbytehq/airbyte/pull/15044) | Added the ability to connect using different SSL modes and SSL certificates | +| 0.6.1 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 0.6.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | +| 0.5.17 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.5.16 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.5.15 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | +| 0.5.13 | 2022-06-21 | [13945](https://github.com/airbytehq/airbyte/pull/13945) | Aligned datatype test | +| 0.5.12 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.5.11 | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544) | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads. | +| 0.5.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.5.9 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.5.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.5.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.5.4 | 2022-02-11 | [10251](https://github.com/airbytehq/airbyte/issues/10251) | bug Source MySQL CDC: sync failed when has Zero-date value in mandatory column | +| 0.5.2 | 2021-12-14 | [6425](https://github.com/airbytehq/airbyte/issues/6425) | MySQL CDC sync fails because starting binlog position not found in DB | +| 0.5.1 | 2021-12-13 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | +| 0.5.0 | 2021-12-11 | [7970](https://github.com/airbytehq/airbyte/pull/7970) | Support all MySQL types | +| 0.4.13 | 2021-12-03 | [8335](https://github.com/airbytehq/airbyte/pull/8335) | Source-MySql: do not check cdc required param binlog_row_image for standard replication | +| 0.4.12 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.4.11 | 2021-11-19 | [8047](https://github.com/airbytehq/airbyte/pull/8047) | Source MySQL: transform binary data base64 format | +| 0.4.10 | 2021-11-15 | [7820](https://github.com/airbytehq/airbyte/pull/7820) | Added basic performance test | +| 0.4.9 | 2021-11-02 | [7559](https://github.com/airbytehq/airbyte/pull/7559) | Correctly process large unsigned short integer values which may fall outside java's `Short` data type capability | +| 0.4.8 | 2021-09-16 | [6093](https://github.com/airbytehq/airbyte/pull/6093) | Improve reliability of processing various data types like decimals, dates, datetime, binary, and text | +| 0.4.7 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | +| 0.4.6 | 2021-09-29 | [6510](https://github.com/airbytehq/airbyte/pull/6510) | Support SSL connection | +| 0.4.5 | 2021-09-17 | [6146](https://github.com/airbytehq/airbyte/pull/6146) | Added option to connect to DB via SSH | +| 0.4.1 | 2021-07-23 | [4956](https://github.com/airbytehq/airbyte/pull/4956) | Fix log link | +| 0.3.7 | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE\_ENTRYPOINT for Kubernetes support | +| 0.3.6 | 2021-06-09 | [3966](https://github.com/airbytehq/airbyte/pull/3966) | Fix excessive logging for CDC method | +| 0.3.5 | 2021-06-07 | [3890](https://github.com/airbytehq/airbyte/pull/3890) | Fix CDC handle tinyint\(1\) and boolean types | +| 0.3.4 | 2021-06-04 | [3846](https://github.com/airbytehq/airbyte/pull/3846) | Fix max integer value failure | +| 0.3.3 | 2021-06-02 | [3789](https://github.com/airbytehq/airbyte/pull/3789) | MySQL CDC poll wait 5 minutes when not received a single record | +| 0.3.2 | 2021-06-01 | [3757](https://github.com/airbytehq/airbyte/pull/3757) | MySQL CDC poll 5s to 5 min | +| 0.3.1 | 2021-06-01 | [3505](https://github.com/airbytehq/airbyte/pull/3505) | Implemented MySQL CDC | +| 0.3.0 | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990) | Support namespaces | +| 0.2.5 | 2021-04-15 | [2899](https://github.com/airbytehq/airbyte/pull/2899) | Fix bug in tests | +| 0.2.4 | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600) | Add NCHAR and NVCHAR support to DB and cursor type casting | +| 0.2.3 | 2021-03-26 | [2611](https://github.com/airbytehq/airbyte/pull/2611) | Add an optional `jdbc_url_params` in parameters | +| 0.2.2 | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460) | Destination supports destination sync mode | +| 0.2.1 | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488) | Sources support primary keys | +| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | +| 0.1.10 | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887) | Migrate AbstractJdbcSource to use iterators | +| 0.1.9 | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746) | Fix NPE in State Decorator | +| 0.1.8 | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724) | Fix JdbcSource handling of tables with same names in different schemas | +| 0.1.7 | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655) | Fix JdbcSource OOM | +| 0.1.6 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySQL to use new JdbcSource | +| 0.1.5 | 2020-12-11 | [1267](https://github.com/airbytehq/airbyte/pull/1267) | Support incremental sync | +| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | From 95d65454af5c8606e0d2fe70d65718efb67b8cea Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Wed, 10 Aug 2022 12:01:22 +0300 Subject: [PATCH 12/24] updated code style --- .../main/java/io/airbyte/db/MySqlUtils.java | 2 +- .../util/MySqlSslConnectionUtils.java | 13 +++---- ...cateStrictEncryptSourceAcceptanceTest.java | 38 +++++++++---------- ...cateStrictEncryptSourceAcceptanceTest.java | 8 ++-- ...cateStrictEncryptSourceAcceptanceTest.java | 12 +++--- ...ySqlStrictEncryptSourceAcceptanceTest.java | 4 +- ...SqlSslCertificateSourceAcceptanceTest.java | 38 +++++++++---------- ...lSslCaCertificateSourceAcceptanceTest.java | 8 ++-- ...slFullCertificateSourceAcceptanceTest.java | 12 +++--- 9 files changed, 67 insertions(+), 68 deletions(-) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java index eb6306a9f852..5f637c2f72fa 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java @@ -13,7 +13,7 @@ public class MySqlUtils { @VisibleForTesting public static Certificate getCertificate(final MySQLContainer container, final boolean useAllCertificates) - throws IOException, InterruptedException { + throws IOException, InterruptedException { // add root and server certificates to config file container.execInContainer("sh", "-c", "sed -i '31 a ssl' /etc/my.cnf"); container.execInContainer("sh", "-c", "sed -i '32 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf"); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java index 1fc6db15d9fb..97ecff9ef55a 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -5,7 +5,6 @@ package io.airbyte.integrations.util; import com.fasterxml.jackson.databind.JsonNode; - import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; @@ -69,7 +68,7 @@ private static Map obtainConnectionFullOptions(final JsonNode en try { convertAndImportFullCertificate(encryption.get(PARAM_CA_CERTIFICATE).asText(), encryption.get(PARAM_CLIENT_CERTIFICATE).asText(), - encryption.get(PARAM_CLIENT_KEY).asText(), clientKeyPassword); + encryption.get(PARAM_CLIENT_KEY).asText(), clientKeyPassword); } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } @@ -116,18 +115,18 @@ private static void convertAndImportFullCertificate(final String caCertificate, createCertificateFile(CLIENT_KEY_NAME, clientKey); // add client certificate to the custom keystore runProcess("openssl pkcs12 -export -in " + CLIENT_CERTIFICARE_NAME + " -inkey " + CLIENT_KEY_NAME + - " -out " + CLIENT_CERT_P12 + " -name \"certificate\" -passout pass:" + clientKeyPassword, run); + " -out " + CLIENT_CERT_P12 + " -name \"certificate\" -passout pass:" + clientKeyPassword, run); // add client key to the custom keystore runProcess("keytool -importkeystore -srckeystore " + CLIENT_CERT_P12 + - " -srcstoretype pkcs12 -destkeystore " + CUSTOM_KEY_STORE + " -srcstorepass " + clientKeyPassword + - " -deststoretype JKS -deststorepass " + clientKeyPassword + " -noprompt", run); + " -srcstoretype pkcs12 -destkeystore " + CUSTOM_KEY_STORE + " -srcstorepass " + clientKeyPassword + + " -deststoretype JKS -deststorepass " + clientKeyPassword + " -noprompt", run); } private static void convertAndImportCaCertificate(final String caCertificate, final String clientKeyPassword) throws IOException, InterruptedException { - final Runtime run = Runtime.getRuntime(); - createCaCertificate(caCertificate, clientKeyPassword, run); + final Runtime run = Runtime.getRuntime(); + createCaCertificate(caCertificate, clientKeyPassword, run); } private static void createCaCertificate(final String caCertificate, diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java index e7c26ace903d..ecfa9e92953a 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java @@ -33,38 +33,38 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc var sslMode = getSslConfig(); config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.STANDARD) - .build()); + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) + .put("replication_method", ReplicationMethod.STANDARD) + .build()); } public abstract ImmutableMap getSslConfig(); private void addTestData(MySQLContainer container) throws Exception { try (final DSLContext dslContext = DSLContextFactory.create( - container.getUsername(), - container.getPassword(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s", - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { + container.getUsername(), + container.getPassword(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s/%s", + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + SQLDialect.MYSQL)) { final Database database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); return null; }); } diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java index 2c13ade52f10..22925d4a06bc 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java @@ -12,10 +12,10 @@ public class MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest extends Abst @Override public ImmutableMap getSslConfig() { return ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_key_password", PASSWORD) - .build(); + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_key_password", PASSWORD) + .build(); } } diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java index 0d7029caef60..f86ff0d88c69 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java @@ -12,12 +12,12 @@ public class MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest extends Ab @Override public ImmutableMap getSslConfig() { return ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_identity") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) - .build(); + .put(JdbcUtils.MODE_KEY, "verify_identity") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_certificate", certs.getClientCertificate()) + .put("client_key", certs.getClientKey()) + .put("client_key_password", PASSWORD) + .build(); } } diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java index 481b23879119..43493837f006 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java @@ -46,8 +46,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc container.start(); var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "required") - .build(); + .put(JdbcUtils.MODE_KEY, "required") + .build(); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java index d3d58fa8098e..e708eeb7909c 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java @@ -33,38 +33,38 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc var sslMode = getSslConfig(); config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.STANDARD) - .build()); + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) + .put("replication_method", ReplicationMethod.STANDARD) + .build()); } public abstract ImmutableMap getSslConfig(); private void addTestData(MySQLContainer container) throws Exception { try (final DSLContext dslContext = DSLContextFactory.create( - container.getUsername(), - container.getPassword(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s", - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { + container.getUsername(), + container.getPassword(), + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s/%s", + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + SQLDialect.MYSQL)) { final Database database = new Database(dslContext); database.query(ctx -> { ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); return null; }); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java index 9130876847c3..bcb0622b8fd9 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java @@ -12,10 +12,10 @@ public class MySqlSslCaCertificateSourceAcceptanceTest extends AbstractMySqlSslC @Override public ImmutableMap getSslConfig() { return ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_key_password", PASSWORD) - .build(); + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_key_password", PASSWORD) + .build(); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java index 39e6029cd1b3..94e905298e8d 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -12,12 +12,12 @@ public class MySqlSslFullCertificateSourceAcceptanceTest extends AbstractMySqlSs @Override public ImmutableMap getSslConfig() { return ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_identity") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) - .build(); + .put(JdbcUtils.MODE_KEY, "verify_identity") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_certificate", certs.getClientCertificate()) + .put("client_key", certs.getClientKey()) + .put("client_key_password", PASSWORD) + .build(); } } From 5ccdc64481f32758a55094061b17dfc70eaf71cd Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Wed, 10 Aug 2022 16:51:43 +0300 Subject: [PATCH 13/24] updated doc --- docs/integrations/sources/mysql.md | 100 ++++++++++++++--------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 22e0b6643b74..57cb2a30b9cd 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -183,53 +183,53 @@ If you do not see a type in this list, assume that it is coerced into a string. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.6.2 | 2022-08-10 | [15044](https://github.com/airbytehq/airbyte/pull/15044) | Added the ability to connect using different SSL modes and SSL certificates | -| 0.6.1 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | -| 0.6.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | -| 0.5.17 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.5.16 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.5.15 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | -| 0.5.13 | 2022-06-21 | [13945](https://github.com/airbytehq/airbyte/pull/13945) | Aligned datatype test | -| 0.5.12 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.5.11 | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544) | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads. | -| 0.5.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.5.9 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.5.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.5.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.5.4 | 2022-02-11 | [10251](https://github.com/airbytehq/airbyte/issues/10251) | bug Source MySQL CDC: sync failed when has Zero-date value in mandatory column | -| 0.5.2 | 2021-12-14 | [6425](https://github.com/airbytehq/airbyte/issues/6425) | MySQL CDC sync fails because starting binlog position not found in DB | -| 0.5.1 | 2021-12-13 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | -| 0.5.0 | 2021-12-11 | [7970](https://github.com/airbytehq/airbyte/pull/7970) | Support all MySQL types | -| 0.4.13 | 2021-12-03 | [8335](https://github.com/airbytehq/airbyte/pull/8335) | Source-MySql: do not check cdc required param binlog_row_image for standard replication | -| 0.4.12 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.4.11 | 2021-11-19 | [8047](https://github.com/airbytehq/airbyte/pull/8047) | Source MySQL: transform binary data base64 format | -| 0.4.10 | 2021-11-15 | [7820](https://github.com/airbytehq/airbyte/pull/7820) | Added basic performance test | -| 0.4.9 | 2021-11-02 | [7559](https://github.com/airbytehq/airbyte/pull/7559) | Correctly process large unsigned short integer values which may fall outside java's `Short` data type capability | -| 0.4.8 | 2021-09-16 | [6093](https://github.com/airbytehq/airbyte/pull/6093) | Improve reliability of processing various data types like decimals, dates, datetime, binary, and text | -| 0.4.7 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | -| 0.4.6 | 2021-09-29 | [6510](https://github.com/airbytehq/airbyte/pull/6510) | Support SSL connection | -| 0.4.5 | 2021-09-17 | [6146](https://github.com/airbytehq/airbyte/pull/6146) | Added option to connect to DB via SSH | -| 0.4.1 | 2021-07-23 | [4956](https://github.com/airbytehq/airbyte/pull/4956) | Fix log link | -| 0.3.7 | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE\_ENTRYPOINT for Kubernetes support | -| 0.3.6 | 2021-06-09 | [3966](https://github.com/airbytehq/airbyte/pull/3966) | Fix excessive logging for CDC method | -| 0.3.5 | 2021-06-07 | [3890](https://github.com/airbytehq/airbyte/pull/3890) | Fix CDC handle tinyint\(1\) and boolean types | -| 0.3.4 | 2021-06-04 | [3846](https://github.com/airbytehq/airbyte/pull/3846) | Fix max integer value failure | -| 0.3.3 | 2021-06-02 | [3789](https://github.com/airbytehq/airbyte/pull/3789) | MySQL CDC poll wait 5 minutes when not received a single record | -| 0.3.2 | 2021-06-01 | [3757](https://github.com/airbytehq/airbyte/pull/3757) | MySQL CDC poll 5s to 5 min | -| 0.3.1 | 2021-06-01 | [3505](https://github.com/airbytehq/airbyte/pull/3505) | Implemented MySQL CDC | -| 0.3.0 | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990) | Support namespaces | -| 0.2.5 | 2021-04-15 | [2899](https://github.com/airbytehq/airbyte/pull/2899) | Fix bug in tests | -| 0.2.4 | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600) | Add NCHAR and NVCHAR support to DB and cursor type casting | -| 0.2.3 | 2021-03-26 | [2611](https://github.com/airbytehq/airbyte/pull/2611) | Add an optional `jdbc_url_params` in parameters | -| 0.2.2 | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460) | Destination supports destination sync mode | -| 0.2.1 | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488) | Sources support primary keys | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.1.10 | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887) | Migrate AbstractJdbcSource to use iterators | -| 0.1.9 | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746) | Fix NPE in State Decorator | -| 0.1.8 | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724) | Fix JdbcSource handling of tables with same names in different schemas | -| 0.1.7 | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655) | Fix JdbcSource OOM | -| 0.1.6 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySQL to use new JdbcSource | -| 0.1.5 | 2020-12-11 | [1267](https://github.com/airbytehq/airbyte/pull/1267) | Support incremental sync | -| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| +| 0.6.2 | 2022-08-10 | [15044](https://github.com/airbytehq/airbyte/pull/15044) | Added the ability to connect using different SSL modes and SSL certificates | +| 0.6.1 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | +| 0.6.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | +| 0.5.17 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.5.16 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.5.15 | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077) | Use the new state management | +| 0.5.13 | 2022-06-21 | [13945](https://github.com/airbytehq/airbyte/pull/13945) | Aligned datatype test | +| 0.5.12 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.5.11 | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544) | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads. | +| 0.5.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.5.9 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.5.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.5.5 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.5.4 | 2022-02-11 | [10251](https://github.com/airbytehq/airbyte/issues/10251) | bug Source MySQL CDC: sync failed when has Zero-date value in mandatory column | +| 0.5.2 | 2021-12-14 | [6425](https://github.com/airbytehq/airbyte/issues/6425) | MySQL CDC sync fails because starting binlog position not found in DB | +| 0.5.1 | 2021-12-13 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | +| 0.5.0 | 2021-12-11 | [7970](https://github.com/airbytehq/airbyte/pull/7970) | Support all MySQL types | +| 0.4.13 | 2021-12-03 | [8335](https://github.com/airbytehq/airbyte/pull/8335) | Source-MySql: do not check cdc required param binlog_row_image for standard replication | +| 0.4.12 | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.4.11 | 2021-11-19 | [8047](https://github.com/airbytehq/airbyte/pull/8047) | Source MySQL: transform binary data base64 format | +| 0.4.10 | 2021-11-15 | [7820](https://github.com/airbytehq/airbyte/pull/7820) | Added basic performance test | +| 0.4.9 | 2021-11-02 | [7559](https://github.com/airbytehq/airbyte/pull/7559) | Correctly process large unsigned short integer values which may fall outside java's `Short` data type capability | +| 0.4.8 | 2021-09-16 | [6093](https://github.com/airbytehq/airbyte/pull/6093) | Improve reliability of processing various data types like decimals, dates, datetime, binary, and text | +| 0.4.7 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | +| 0.4.6 | 2021-09-29 | [6510](https://github.com/airbytehq/airbyte/pull/6510) | Support SSL connection | +| 0.4.5 | 2021-09-17 | [6146](https://github.com/airbytehq/airbyte/pull/6146) | Added option to connect to DB via SSH | +| 0.4.1 | 2021-07-23 | [4956](https://github.com/airbytehq/airbyte/pull/4956) | Fix log link | +| 0.3.7 | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE\_ENTRYPOINT for Kubernetes support | +| 0.3.6 | 2021-06-09 | [3966](https://github.com/airbytehq/airbyte/pull/3966) | Fix excessive logging for CDC method | +| 0.3.5 | 2021-06-07 | [3890](https://github.com/airbytehq/airbyte/pull/3890) | Fix CDC handle tinyint\(1\) and boolean types | +| 0.3.4 | 2021-06-04 | [3846](https://github.com/airbytehq/airbyte/pull/3846) | Fix max integer value failure | +| 0.3.3 | 2021-06-02 | [3789](https://github.com/airbytehq/airbyte/pull/3789) | MySQL CDC poll wait 5 minutes when not received a single record | +| 0.3.2 | 2021-06-01 | [3757](https://github.com/airbytehq/airbyte/pull/3757) | MySQL CDC poll 5s to 5 min | +| 0.3.1 | 2021-06-01 | [3505](https://github.com/airbytehq/airbyte/pull/3505) | Implemented MySQL CDC | +| 0.3.0 | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990) | Support namespaces | +| 0.2.5 | 2021-04-15 | [2899](https://github.com/airbytehq/airbyte/pull/2899) | Fix bug in tests | +| 0.2.4 | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600) | Add NCHAR and NVCHAR support to DB and cursor type casting | +| 0.2.3 | 2021-03-26 | [2611](https://github.com/airbytehq/airbyte/pull/2611) | Add an optional `jdbc_url_params` in parameters | +| 0.2.2 | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460) | Destination supports destination sync mode | +| 0.2.1 | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488) | Sources support primary keys | +| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | +| 0.1.10 | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887) | Migrate AbstractJdbcSource to use iterators | +| 0.1.9 | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746) | Fix NPE in State Decorator | +| 0.1.8 | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724) | Fix JdbcSource handling of tables with same names in different schemas | +| 0.1.7 | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655) | Fix JdbcSource OOM | +| 0.1.6 | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307) | Migrate Postgres and MySQL to use new JdbcSource | +| 0.1.5 | 2020-12-11 | [1267](https://github.com/airbytehq/airbyte/pull/1267) | Support incremental sync | +| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | \ No newline at end of file From 6c5a9a92c182dac080bd697d404beb6f447c30ef Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Wed, 10 Aug 2022 20:53:32 +0300 Subject: [PATCH 14/24] updated specs --- .../src/test/resources/expected_spec.json | 8 ++++---- .../source-mysql/src/main/resources/spec.json | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index eee88512d3c6..2bc37fdeec14 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -49,12 +49,12 @@ }, "ssl_mode": { "title": "SSL modes", - "description": "SSL modes", + "description": "SSL connection modes. \n
  • disable - Use to make a general connection without SSL.
    • \n
  • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
    • \n
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
    • \n
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
    • \n
  • Verify Identity -Always connect with SSL. Verify both CA and Hostname.
\n Read more in the docs.", "type": "object", "order": 7, "oneOf": [ { - "title": "Require", + "title": "required", "description": "Require SSL mode.", "required": ["mode"], "properties": { @@ -68,7 +68,7 @@ } }, { - "title": "Require and Verify CA", + "title": "Verify CA", "description": "Verify CA SSL mode.", "required": ["mode"], "properties": { @@ -97,7 +97,7 @@ } }, { - "title": "Require and Verify Identity", + "title": "Verify Identity", "description": "Verify-full SSL mode.", "required": [ "mode", diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index ea303529fa34..28c39a522ced 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -56,12 +56,12 @@ }, "ssl_mode": { "title": "SSL modes", - "description": "SSL modes", + "description": "SSL connection modes. \n
  • disable - Use to make a general connection without SSL.
    • \n
  • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
    • \n
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
    • \n
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
    • \n
  • Verify Identity -Always connect with SSL. Verify both CA and Hostname.
\n Read more in the docs.", "type": "object", "order": 7, "oneOf": [ { - "title": "No", + "title": "disable", "description": "Disable SSL.", "required": ["mode"], "properties": { @@ -75,7 +75,7 @@ } }, { - "title": "If available", + "title": "preferred", "description": "Preferred SSL mode.", "required": ["mode"], "properties": { @@ -89,7 +89,7 @@ } }, { - "title": "Require", + "title": "required", "description": "Require SSL mode.", "required": ["mode"], "properties": { @@ -103,7 +103,7 @@ } }, { - "title": "Require and Verify CA", + "title": "Verify CA", "description": "Verify CA SSL mode.", "required": ["mode"], "properties": { @@ -132,7 +132,7 @@ } }, { - "title": "Require and Verify Identity", + "title": "Verify Identity", "description": "Verify-full SSL mode.", "required": [ "mode", From 9109ec10ed600dd59a9e3981cd02a31ee7089594 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Thu, 11 Aug 2022 11:38:20 +0300 Subject: [PATCH 15/24] fixed minor remarks --- .../util/MySqlSslConnectionUtils.java | 18 ++++------ .../src/test/resources/expected_spec.json | 7 ++-- .../source/mysql/MySqlSource.java | 36 +++++++++---------- .../source-mysql/src/main/resources/spec.json | 21 +++-------- 4 files changed, 33 insertions(+), 49 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java index 97ecff9ef55a..26886753d015 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -31,7 +31,6 @@ public class MySqlSslConnectionUtils { public static final String CUSTOM_TRUST_STORE = "customtruststore.jks"; public static final String CUSTOM_KEY_STORE = "customkeystore.jks"; public static final String SSL_MODE = "sslMode"; - public static final String DISABLE = "disable"; public static final String VERIFY_CA = "VERIFY_CA"; public static final String VERIFY_IDENTITY = "VERIFY_IDENTITY"; public static final String ROOT_CERTIFICARE_NAME = "ca-cert.pem"; @@ -51,10 +50,10 @@ public static Map obtainConnectionOptions(final JsonNode encrypt } switch (method) { case VERIFY_CA -> { - additionalParameters.putAll(obtainConnectionCaOptions(encryption, method, keyStorePassword)); + additionalParameters.putAll(obtainConnectionCaOptions(encryption, keyStorePassword)); } case VERIFY_IDENTITY -> { - additionalParameters.putAll(obtainConnectionFullOptions(encryption, method, keyStorePassword)); + additionalParameters.putAll(obtainConnectionFullOptions(encryption, keyStorePassword)); } } } @@ -62,7 +61,6 @@ public static Map obtainConnectionOptions(final JsonNode encrypt } private static Map obtainConnectionFullOptions(final JsonNode encryption, - final String method, final String clientKeyPassword) { Map additionalParameters = new HashMap<>(); try { @@ -72,22 +70,21 @@ private static Map obtainConnectionFullOptions(final JsonNode en } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } + var trustStorePassword = RandomStringUtils.randomAlphanumeric(10); additionalParameters.put(TRUST_KEY_STORE_URL, "file:" + CUSTOM_TRUST_STORE); - additionalParameters.put(TRUST_KEY_STORE_PASS, clientKeyPassword); + additionalParameters.put(TRUST_KEY_STORE_PASS, trustStorePassword); additionalParameters.put(CLIENT_KEY_STORE_URL, "file:" + CUSTOM_KEY_STORE); additionalParameters.put(CLIENT_KEY_STORE_PASS, clientKeyPassword); additionalParameters.put(SSL_MODE, VERIFY_IDENTITY); - String result = CUSTOM_KEY_STORE; - updateTrustStoreSystemProperty(clientKeyPassword); - System.setProperty("javax.net.ssl.keyStore", result); + updateTrustStoreSystemProperty(trustStorePassword); + System.setProperty("javax.net.ssl.keyStore", CUSTOM_KEY_STORE); System.setProperty("javax.net.ssl.keyStorePassword", clientKeyPassword); return additionalParameters; } private static Map obtainConnectionCaOptions(final JsonNode encryption, - final String method, final String clientKeyPassword) { Map additionalParameters = new HashMap<>(); try { @@ -141,8 +138,7 @@ private static void createCaCertificate(final String caCertificate, } private static void updateTrustStoreSystemProperty(final String clientKeyPassword) { - String result = CUSTOM_TRUST_STORE; - System.setProperty("javax.net.ssl.trustStore", result); + System.setProperty("javax.net.ssl.trustStore", CUSTOM_TRUST_STORE); System.setProperty("javax.net.ssl.trustStorePassword", clientKeyPassword); } diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index 2bc37fdeec14..cfad25399687 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -49,7 +49,7 @@ }, "ssl_mode": { "title": "SSL modes", - "description": "SSL connection modes. \n
  • disable - Use to make a general connection without SSL.
    • \n
  • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
    • \n
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
    • \n
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
    • \n
  • Verify Identity -Always connect with SSL. Verify both CA and Hostname.
\n Read more in the docs.", + "description": "SSL connection modes.
  • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
  • Verify Identity -Always connect with SSL. Verify both CA and Hostname.
    • Read more in the docs.", "type": "object", "order": 7, "oneOf": [ @@ -70,7 +70,10 @@ { "title": "Verify CA", "description": "Verify CA SSL mode.", - "required": ["mode"], + "required": [ + "mode", + "ca_certificate" + ], "properties": { "mode": { "type": "string", diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index e0861dab28b9..2e40195b8fbc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -4,13 +4,6 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.debezium.AirbyteDebeziumHandler.shouldUseCDC; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; -import static io.airbyte.integrations.util.MySqlSslConnectionUtils.DISABLE; -import static io.airbyte.integrations.util.MySqlSslConnectionUtils.obtainConnectionOptions; -import static java.util.stream.Collectors.toList; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; @@ -37,6 +30,9 @@ import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.SyncMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -46,8 +42,12 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import static io.airbyte.integrations.debezium.AirbyteDebeziumHandler.shouldUseCDC; +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.integrations.util.MySqlSslConnectionUtils.obtainConnectionOptions; +import static java.util.stream.Collectors.toList; public class MySqlSource extends AbstractJdbcSource implements Source { @@ -156,21 +156,17 @@ public JsonNode toDatabaseConfig(final JsonNode config) { // assume ssl if not explicitly mentioned. if (!config.has(JdbcUtils.SSL_KEY) || config.get(JdbcUtils.SSL_KEY).asBoolean()) { if (config.has(JdbcUtils.SSL_MODE_KEY)) { - if (DISABLE.equals(config.get(JdbcUtils.SSL_MODE_KEY).get(JdbcUtils.MODE_KEY).asText())) { - jdbcUrl.append(JdbcUtils.AMPERSAND).append("sslMode=DISABLE"); + additionalParameters.putAll(obtainConnectionOptions(config.get(JdbcUtils.SSL_MODE_KEY))); + jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) + .append(JdbcUtils.AMPERSAND); + if (additionalParameters.isEmpty()) { + jdbcUrl.append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); } else { - additionalParameters.putAll(obtainConnectionOptions(config.get(JdbcUtils.SSL_MODE_KEY))); - jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) - .append(JdbcUtils.AMPERSAND); - if (additionalParameters.isEmpty()) { - jdbcUrl.append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); - } else { - jdbcUrl.append(SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION); - } + jdbcUrl.append(SSL_PARAMETERS_WITH_CERTIFICATE_VALIDATION); } } else { jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) - .append(JdbcUtils.AMPERSAND).append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); + .append(JdbcUtils.AMPERSAND).append(SSL_PARAMETERS_WITHOUT_CERTIFICATE_VALIDATION); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index 28c39a522ced..2b0829c477fd 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -56,24 +56,10 @@ }, "ssl_mode": { "title": "SSL modes", - "description": "SSL connection modes. \n
    • disable - Use to make a general connection without SSL.
      • \n
    • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
      • \n
    • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
      • \n
    • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
      • \n
    • Verify Identity -Always connect with SSL. Verify both CA and Hostname.
    \n Read more in the docs.", + "description": "SSL connection modes.
  • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
  • Verify Identity -Always connect with SSL. Verify both CA and Hostname.
    • Read more in the docs.", "type": "object", "order": 7, "oneOf": [ - { - "title": "disable", - "description": "Disable SSL.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "disable", - "enum": ["disable"], - "default": "disable", - "order": 0 - } - } - }, { "title": "preferred", "description": "Preferred SSL mode.", @@ -105,7 +91,10 @@ { "title": "Verify CA", "description": "Verify CA SSL mode.", - "required": ["mode"], + "required": [ + "mode", + "ca_certificate" + ], "properties": { "mode": { "type": "string", From 3c908c9ce874d31a665ac6c545c961b46ca2f207 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Thu, 11 Aug 2022 12:04:48 +0300 Subject: [PATCH 16/24] fixed minor remarks --- .../MySqlStrictEncryptSource.java | 14 ++++++++++++-- .../src/test/resources/expected_spec.json | 2 +- .../source-mysql/src/main/resources/spec.json | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java index bc0e4b77b209..6bac4cccf876 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java @@ -24,6 +24,10 @@ public class MySqlStrictEncryptSource extends SpecModifyingSource implements Source { private static final Logger LOGGER = LoggerFactory.getLogger(MySqlStrictEncryptSource.class); + private static final String SSL_MODE_DESCRIPTION = "SSL connection modes. " + + "
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
    • " + + "
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
    • " + + "
  • Verify Identity - Always connect with SSL. Verify both CA and Hostname.
    • Read more in the docs."; MySqlStrictEncryptSource() { super(MySqlSource.sshWrappedSource()); @@ -36,8 +40,14 @@ public ConnectorSpecification modifySpec(final ConnectorSpecification originalSp // that can be used in the Airbyte cloud. User should not be able to change this property. ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.SSL_KEY); final ArrayNode modifiedSslModes = spec.getConnectionSpecification().get("properties").get("ssl_mode").get("oneOf").deepCopy(); - // Assume that the first items is the "disable" and "preferred" options; remove these options - modifiedSslModes.remove(1); + // update description for ssl_mode property + ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).remove("description"); + ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).remove("type"); + ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).remove("order"); + ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).put("description", SSL_MODE_DESCRIPTION); + ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).put("type", "object"); + ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).put("order", 7); + // Assume that the first items is the "preferred" option; remove this option modifiedSslModes.remove(0); ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).remove("oneOf"); ((ObjectNode) spec.getConnectionSpecification().get("properties").get("ssl_mode")).put("oneOf", modifiedSslModes); diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index cfad25399687..fd695ce5e62c 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -49,7 +49,7 @@ }, "ssl_mode": { "title": "SSL modes", - "description": "SSL connection modes.
  • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
  • Verify Identity -Always connect with SSL. Verify both CA and Hostname.
    • Read more in the docs.", + "description": "SSL connection modes.
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
  • Verify Identity - Always connect with SSL. Verify both CA and Hostname.
    • Read more in the docs.", "type": "object", "order": 7, "oneOf": [ diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index 2b0829c477fd..4c108e5d6ac4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -56,7 +56,7 @@ }, "ssl_mode": { "title": "SSL modes", - "description": "SSL connection modes.
  • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
  • Verify Identity -Always connect with SSL. Verify both CA and Hostname.
    • Read more in the docs.", + "description": "SSL connection modes.
  • preferred - Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.
  • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
  • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
  • Verify Identity - Always connect with SSL. Verify both CA and Hostname.
    • Read more in the docs.", "type": "object", "order": 7, "oneOf": [ From 0e1c0cb0a856da65a85296b697c69a71dd26893a Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Thu, 11 Aug 2022 12:41:48 +0300 Subject: [PATCH 17/24] updated tests --- .../AbstractMySqlSslCertificateSourceAcceptanceTest.java | 6 +++++- .../mysql/MySqlSslCaCertificateSourceAcceptanceTest.java | 8 ++++++++ .../MySqlSslFullCertificateSourceAcceptanceTest.java | 8 ++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java index e708eeb7909c..4f4ad709d2ad 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java @@ -17,6 +17,8 @@ import org.jooq.SQLDialect; import org.testcontainers.containers.MySQLContainer; +import java.io.IOException; + public abstract class AbstractMySqlSslCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { protected static MySqlUtils.Certificate certs; @@ -28,7 +30,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc container = new MySQLContainer<>("mysql:8.0"); container.start(); addTestData(container); - certs = MySqlUtils.getCertificate(container, true); + certs = getCertificates(); var sslMode = getSslConfig(); @@ -44,6 +46,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .build()); } + public abstract MySqlUtils.Certificate getCertificates() throws IOException, InterruptedException; + public abstract ImmutableMap getSslConfig(); private void addTestData(MySQLContainer container) throws Exception { diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java index bcb0622b8fd9..6872a9a67051 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslCaCertificateSourceAcceptanceTest.java @@ -5,10 +5,18 @@ package io.airbyte.integrations.source.mysql; import com.google.common.collect.ImmutableMap; +import io.airbyte.db.MySqlUtils; import io.airbyte.db.jdbc.JdbcUtils; +import java.io.IOException; + public class MySqlSslCaCertificateSourceAcceptanceTest extends AbstractMySqlSslCertificateSourceAcceptanceTest { + @Override + public MySqlUtils.Certificate getCertificates() throws IOException, InterruptedException { + return MySqlUtils.getCertificate(container, false); + } + @Override public ImmutableMap getSslConfig() { return ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java index 94e905298e8d..4f3d29108198 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -5,10 +5,18 @@ package io.airbyte.integrations.source.mysql; import com.google.common.collect.ImmutableMap; +import io.airbyte.db.MySqlUtils; import io.airbyte.db.jdbc.JdbcUtils; +import java.io.IOException; + public class MySqlSslFullCertificateSourceAcceptanceTest extends AbstractMySqlSslCertificateSourceAcceptanceTest { + @Override + public MySqlUtils.Certificate getCertificates() throws IOException, InterruptedException { + return MySqlUtils.getCertificate(container, true); + } + @Override public ImmutableMap getSslConfig() { return ImmutableMap.builder() From 2c47c571a6a0e5cdebb66b0526e747070e24a5a5 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Sun, 14 Aug 2022 21:55:32 +0300 Subject: [PATCH 18/24] fixed remarks and updated specification --- .../util/MySqlSslConnectionUtils.java | 88 ++++++++++++++----- .../src/test/resources/expected_spec.json | 26 ++++-- .../source/mysql/MySqlSource.java | 4 +- .../source-mysql/src/main/resources/spec.json | 26 ++++-- 4 files changed, 109 insertions(+), 35 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java index 26886753d015..015b478f9506 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -5,11 +5,16 @@ package io.airbyte.integrations.util; import com.fasterxml.jackson.databind.JsonNode; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; @@ -38,30 +43,71 @@ public class MySqlSslConnectionUtils { public static final String CLIENT_CERTIFICARE_NAME = "client-cert.pem"; public static final String CLIENT_KEY_NAME = "client-key.pem"; public static final String CLIENT_CERT_P12 = "certificate.p12"; + public static final String ENCRYPT_FILE_NAME = "encrypt"; - public static Map obtainConnectionOptions(final JsonNode encryption) { + public static Map obtainConnection(final JsonNode encryption) { Map additionalParameters = new HashMap<>(); if (!encryption.isNull()) { final var method = encryption.get(PARAM_MODE).asText().toUpperCase(); - String sslPassword = encryption.has(PARAM_CLIENT_KEY_PASSWORD) ? encryption.get(PARAM_CLIENT_KEY_PASSWORD).asText() : ""; - var keyStorePassword = RandomStringUtils.randomAlphanumeric(10); - if (!sslPassword.isEmpty()) { - keyStorePassword = sslPassword; + var keyStorePassword = checkOrCreatePassword(encryption); + if (method.equals(VERIFY_CA) || method.equals(VERIFY_IDENTITY)) { + additionalParameters.putAll(checkCertificatesAndObtainConnection(encryption, method, keyStorePassword)); } - switch (method) { - case VERIFY_CA -> { - additionalParameters.putAll(obtainConnectionCaOptions(encryption, keyStorePassword)); - } - case VERIFY_IDENTITY -> { - additionalParameters.putAll(obtainConnectionFullOptions(encryption, keyStorePassword)); + } + return additionalParameters; + } + + private static String checkOrCreatePassword(final JsonNode encryption) { + String sslPassword = encryption.has(PARAM_CLIENT_KEY_PASSWORD) ? encryption.get(PARAM_CLIENT_KEY_PASSWORD).asText() : ""; + var keyStorePassword = RandomStringUtils.randomAlphanumeric(10); + if (sslPassword.isEmpty()) { + var file = new File(ENCRYPT_FILE_NAME); + if (file.exists()) { + keyStorePassword = readFile(file); + } else { + try { + createCertificateFile(ENCRYPT_FILE_NAME, keyStorePassword); + } catch (final IOException e) { + throw new RuntimeException("Failed to create encryption file "); } } + + } else { + keyStorePassword = sslPassword; } - return additionalParameters; + return keyStorePassword; } - private static Map obtainConnectionFullOptions(final JsonNode encryption, - final String clientKeyPassword) { + private static String readFile(final File file) { + try { + BufferedReader reader = new BufferedReader(new FileReader(file)); + String currentLine = reader.readLine(); + reader.close(); + return currentLine; + } catch (final IOException e) { + throw new RuntimeException("Failed to read file with encryption"); + } + } + + private static Map checkCertificatesAndObtainConnection(final JsonNode encryption, + final String mode, + final String clientKeyPassword) { + var clientCert = encryption.has(PARAM_CLIENT_CERTIFICATE) && + !encryption.get(PARAM_CLIENT_CERTIFICATE).asText().isEmpty() ? encryption.get(PARAM_CLIENT_CERTIFICATE).asText() : null; + var clientKey = encryption.has(PARAM_CLIENT_KEY) && + !encryption.get(PARAM_CLIENT_KEY).asText().isEmpty() ? encryption.get(PARAM_CLIENT_KEY).asText() : null; + if (Objects.nonNull(clientCert) && Objects.nonNull(clientKey)) { + return obtainConnectionWithFullCertificatesOptions(encryption, mode, clientKeyPassword); + } else if (Objects.isNull(clientCert) && Objects.isNull(clientKey)) { + return obtainConnectionWithCaCertificateOptions(encryption, mode, clientKeyPassword); + } else { + throw new RuntimeException("Both fields \"Client certificate\" and \"Client key\" must be added to connect with client certificates."); + } + } + + private static Map obtainConnectionWithFullCertificatesOptions(final JsonNode encryption, + final String mode, + final String clientKeyPassword) { Map additionalParameters = new HashMap<>(); try { convertAndImportFullCertificate(encryption.get(PARAM_CA_CERTIFICATE).asText(), @@ -70,22 +116,22 @@ private static Map obtainConnectionFullOptions(final JsonNode en } catch (final IOException | InterruptedException e) { throw new RuntimeException("Failed to import certificate into Java Keystore"); } - var trustStorePassword = RandomStringUtils.randomAlphanumeric(10); additionalParameters.put(TRUST_KEY_STORE_URL, "file:" + CUSTOM_TRUST_STORE); - additionalParameters.put(TRUST_KEY_STORE_PASS, trustStorePassword); + additionalParameters.put(TRUST_KEY_STORE_PASS, clientKeyPassword); additionalParameters.put(CLIENT_KEY_STORE_URL, "file:" + CUSTOM_KEY_STORE); additionalParameters.put(CLIENT_KEY_STORE_PASS, clientKeyPassword); - additionalParameters.put(SSL_MODE, VERIFY_IDENTITY); + additionalParameters.put(SSL_MODE, mode); - updateTrustStoreSystemProperty(trustStorePassword); + updateTrustStoreSystemProperty(clientKeyPassword); System.setProperty("javax.net.ssl.keyStore", CUSTOM_KEY_STORE); System.setProperty("javax.net.ssl.keyStorePassword", clientKeyPassword); return additionalParameters; } - private static Map obtainConnectionCaOptions(final JsonNode encryption, - final String clientKeyPassword) { + private static Map obtainConnectionWithCaCertificateOptions(final JsonNode encryption, + final String mode, + final String clientKeyPassword) { Map additionalParameters = new HashMap<>(); try { convertAndImportCaCertificate(encryption.get(PARAM_CA_CERTIFICATE).asText(), clientKeyPassword); @@ -94,7 +140,7 @@ private static Map obtainConnectionCaOptions(final JsonNode encr } additionalParameters.put(TRUST_KEY_STORE_URL, "file:" + CUSTOM_TRUST_STORE); additionalParameters.put(TRUST_KEY_STORE_PASS, clientKeyPassword); - additionalParameters.put(SSL_MODE, VERIFY_CA); + additionalParameters.put(SSL_MODE, mode); updateTrustStoreSystemProperty(clientKeyPassword); diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index fd695ce5e62c..51bdd7793f73 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -90,12 +90,28 @@ "multiline": true, "order": 1 }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2 + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3 + }, "client_key_password": { "type": "string", "title": "Client key password (Optional)", "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, - "order": 2 + "order": 4 } } }, @@ -104,9 +120,7 @@ "description": "Verify-full SSL mode.", "required": [ "mode", - "ca_certificate", - "client_certificate", - "client_key" + "ca_certificate" ], "properties": { "mode": { @@ -127,7 +141,7 @@ "client_certificate": { "type": "string", "title": "Client certificate", - "description": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", "airbyte_secret": true, "multiline": true, "order": 2 @@ -135,7 +149,7 @@ "client_key": { "type": "string", "title": "Client key", - "description": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", "airbyte_secret": true, "multiline": true, "order": 3 diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 2e40195b8fbc..03a4cab465dc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -46,7 +46,7 @@ import static io.airbyte.integrations.debezium.AirbyteDebeziumHandler.shouldUseCDC; import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; -import static io.airbyte.integrations.util.MySqlSslConnectionUtils.obtainConnectionOptions; +import static io.airbyte.integrations.util.MySqlSslConnectionUtils.obtainConnection; import static java.util.stream.Collectors.toList; public class MySqlSource extends AbstractJdbcSource implements Source { @@ -156,7 +156,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { // assume ssl if not explicitly mentioned. if (!config.has(JdbcUtils.SSL_KEY) || config.get(JdbcUtils.SSL_KEY).asBoolean()) { if (config.has(JdbcUtils.SSL_MODE_KEY)) { - additionalParameters.putAll(obtainConnectionOptions(config.get(JdbcUtils.SSL_MODE_KEY))); + additionalParameters.putAll(obtainConnection(config.get(JdbcUtils.SSL_MODE_KEY))); jdbcUrl.append(JdbcUtils.AMPERSAND).append(String.join(JdbcUtils.AMPERSAND, SSL_PARAMETERS)) .append(JdbcUtils.AMPERSAND); if (additionalParameters.isEmpty()) { diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index 4c108e5d6ac4..00cb937b6806 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -111,12 +111,28 @@ "multiline": true, "order": 1 }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2 + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3 + }, "client_key_password": { "type": "string", "title": "Client key password (Optional)", "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", "airbyte_secret": true, - "order": 2 + "order": 4 } } }, @@ -125,9 +141,7 @@ "description": "Verify-full SSL mode.", "required": [ "mode", - "ca_certificate", - "client_certificate", - "client_key" + "ca_certificate" ], "properties": { "mode": { @@ -148,7 +162,7 @@ "client_certificate": { "type": "string", "title": "Client certificate", - "description": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", "airbyte_secret": true, "multiline": true, "order": 2 @@ -156,7 +170,7 @@ "client_key": { "type": "string", "title": "Client key", - "description": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", "airbyte_secret": true, "multiline": true, "order": 3 From 2418e98af7e4a03dfec6baa54feade7a6facd0f6 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Sun, 14 Aug 2022 22:02:24 +0300 Subject: [PATCH 19/24] fixed mysql sources connectors version --- .../connectors/source-mysql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-mysql/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index 370187835e0b..e1b0f49ecf89 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.2 +LABEL io.airbyte.version=0.6.3 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 54ce574ace98..f686153111b4 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.2 +LABEL io.airbyte.version=0.6.3 LABEL io.airbyte.name=airbyte/source-mysql From 132c7ce1a252339e5caba07c0d7a37f7ba0943d3 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Sun, 14 Aug 2022 23:38:34 +0300 Subject: [PATCH 20/24] added CDC + SSL Certificates tests --- ...lFullCertificatesSourceAcceptanceTest.java | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslFullCertificatesSourceAcceptanceTest.java diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslFullCertificatesSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslFullCertificatesSourceAcceptanceTest.java new file mode 100644 index 000000000000..5423eedc9902 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslFullCertificatesSourceAcceptanceTest.java @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.MySqlUtils; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.ssh.SshHelpers; +import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.SyncMode; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MySQLContainer; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static io.airbyte.protocol.models.SyncMode.INCREMENTAL; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +public class CdcMySqlSslFullCertificatesSourceAcceptanceTest extends SourceAcceptanceTest { + + private static final String STREAM_NAME = "id_and_name"; + private static final String STREAM_NAME2 = "starships"; + private MySQLContainer container; + private JsonNode config; + private static MySqlUtils.Certificate certs; + + @Override + protected String getImageName() { + return "airbyte/source-mysql:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.getSpecAndInjectSsh(); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s", STREAM_NAME), + String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))), + new ConfiguredAirbyteStream() + .withSyncMode(INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s", STREAM_NAME2), + String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))))); + } + + @Override + protected JsonNode getState() { + return null; + } + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + container = new MySQLContainer<>("mysql:8.0"); + container.start(); + certs = MySqlUtils.getCertificate(container, true); + + var sslMode = ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_identity") + .put("ca_certificate", certs.getCaCertificate()) + .put("client_certificate", certs.getClientCertificate()) + .put("client_key", certs.getClientKey()) + .put("client_key_password", "Passw0rd") + .build(); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, container.getHost()) + .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.USERNAME_KEY, container.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, sslMode) + .put("replication_method", ReplicationMethod.CDC) + .build()); + + revokeAllPermissions(); + grantCorrectPermissions(); + createAndPopulateTables(); + } + + private void createAndPopulateTables() { + executeQuery("CREATE TABLE id_and_name(id INTEGER PRIMARY KEY, name VARCHAR(200));"); + executeQuery( + "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + executeQuery("CREATE TABLE starships(id INTEGER PRIMARY KEY, name VARCHAR(200));"); + executeQuery( + "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + } + + private void revokeAllPermissions() { + executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); + } + + private void grantCorrectPermissions() { + executeQuery( + "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " + + container.getUsername() + "@'%';"); + } + + private void executeQuery(final String query) { + try (final DSLContext dslContext = DSLContextFactory.create( + "root", + "test", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), + container.getHost(), + container.getFirstMappedPort(), + container.getDatabaseName()), + SQLDialect.MYSQL)) { + final Database database = new Database(dslContext); + database.query( + ctx -> ctx + .execute(query)); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + container.close(); + } + + @Test + public void testIncrementalSyncShouldNotFailIfBinlogIsDeleted() throws Exception { + final ConfiguredAirbyteCatalog configuredCatalog = withSourceDefinedCursors(getConfiguredCatalog()); + // only sync incremental streams + configuredCatalog.setStreams( + configuredCatalog.getStreams().stream().filter(s -> s.getSyncMode() == INCREMENTAL).collect(Collectors.toList())); + + final List airbyteMessages = runRead(configuredCatalog, getState()); + final List recordMessages = filterRecords(airbyteMessages); + final List stateMessages = airbyteMessages + .stream() + .filter(m -> m.getType() == AirbyteMessage.Type.STATE) + .map(AirbyteMessage::getState) + .collect(Collectors.toList()); + assertFalse(recordMessages.isEmpty(), "Expected the first incremental sync to produce records"); + assertFalse(stateMessages.isEmpty(), "Expected incremental sync to produce STATE messages"); + + // when we run incremental sync again there should be no new records. Run a sync with the latest + // state message and assert no records were emitted. + final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); + // RESET MASTER removes all binary log files that are listed in the index file, + // leaving only a single, empty binary log file with a numeric suffix of .000001 + executeQuery("RESET MASTER;"); + + assertEquals(6, filterRecords(runRead(configuredCatalog, latestState)).size()); + } + +} From 5503174a2bc268b3b7a90752e7dc4785cf31fb8b Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Mon, 15 Aug 2022 14:47:53 +0300 Subject: [PATCH 21/24] added property for CDC and added tests for test SSL with CDC together --- .../source/mysql/MySqlCdcProperties.java | 13 ++++++++++++- ...cMySqlSslCaCertificateSourceAcceptanceTest.java} | 5 ++--- 2 files changed, 14 insertions(+), 4 deletions(-) rename airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/{CdcMySqlSslFullCertificatesSourceAcceptanceTest.java => CdcMySqlSslCaCertificateSourceAcceptanceTest.java} (97%) diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java index cf30b04329a3..5db5d2b71566 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java @@ -5,6 +5,10 @@ package io.airbyte.integrations.source.mysql; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.db.jdbc.JdbcUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Properties; public class MySqlCdcProperties { @@ -44,7 +48,14 @@ static Properties getDebeziumProperties(final JsonNode config) { // https://debezium.io/documentation/reference/1.9/connectors/mysql.html#mysql-property-binary-handling-mode props.setProperty("binary.handling.mode", "base64"); props.setProperty("database.include.list", config.get("database").asText()); - + // https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-property-database-ssl-mode + if (!config.has(JdbcUtils.SSL_KEY) || config.get(JdbcUtils.SSL_KEY).asBoolean()) { + if (config.has(JdbcUtils.SSL_MODE_KEY) && config.get(JdbcUtils.SSL_MODE_KEY).has(JdbcUtils.MODE_KEY)) { + props.setProperty("database.ssl.mode", config.get(JdbcUtils.SSL_MODE_KEY).get(JdbcUtils.MODE_KEY).asText()); + } else { + props.setProperty("database.ssl.mode", "required"); + } + } return props; } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslFullCertificatesSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslCaCertificateSourceAcceptanceTest.java similarity index 97% rename from airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslFullCertificatesSourceAcceptanceTest.java rename to airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslCaCertificateSourceAcceptanceTest.java index 5423eedc9902..2b7b9fde1ff2 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslFullCertificatesSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslCaCertificateSourceAcceptanceTest.java @@ -34,7 +34,6 @@ import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; -import java.io.IOException; import java.util.List; import java.util.stream.Collectors; @@ -42,7 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -public class CdcMySqlSslFullCertificatesSourceAcceptanceTest extends SourceAcceptanceTest { +public class CdcMySqlSslCaCertificateSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; @@ -106,7 +105,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc certs = MySqlUtils.getCertificate(container, true); var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_identity") + .put(JdbcUtils.MODE_KEY, "verify_ca") .put("ca_certificate", certs.getCaCertificate()) .put("client_certificate", certs.getClientCertificate()) .put("client_key", certs.getClientKey()) From 9e732d7dec035d6abaf9038de2a58dfde4a8bde1 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Mon, 15 Aug 2022 17:10:22 +0300 Subject: [PATCH 22/24] fixed MySqlStrictEncryptJdbcSourceAcceptanceTest for work with datetime format --- ...StrictEncryptJdbcSourceAcceptanceTest.java | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java index 96b656995ea9..fe59d601fd92 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java @@ -20,10 +20,23 @@ import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.mysql.MySqlSource; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteCatalog; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConnectorSpecification; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.SyncMode; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; @@ -128,4 +141,91 @@ void testSpec() throws Exception { assertEquals(expected, actual); } + @Override + protected AirbyteCatalog getCatalog(final String defaultNamespace) { + return new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + TABLE_NAME, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), + CatalogHelpers.createAirbyteStream( + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(Collections.emptyList()), + CatalogHelpers.createAirbyteStream( + TABLE_NAME_COMPOSITE_PK, + defaultNamespace, + Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), + Field.of(COL_LAST_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey( + List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); + } + + @Override + protected void incrementalDateCheck() throws Exception { + incrementalCursorCheck( + COL_UPDATED_AT, + "2005-10-18", + "2006-10-19", + List.of(getTestMessages().get(1), getTestMessages().get(2))); + } + + @Override + protected List getTestMessages() { + return List.of( + new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19")))), + new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19")))), + new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19"))))); + } + + @Override + protected List getExpectedAirbyteMessagesSecondSync(String namespace) { + final List expectedMessages = new ArrayList<>(); + expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19"))))); + expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19"))))); + final DbStreamState state = new DbStreamState() + .withStreamName(streamName) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("5"); + expectedMessages.addAll(createExpectedTestMessages(List.of(state))); + return expectedMessages; + } + } From 5c2d5aa28454f1ab8cce96d8b38bacaaa7f45478 Mon Sep 17 00:00:00 2001 From: Andrii Korotkov Date: Tue, 16 Aug 2022 12:16:10 +0300 Subject: [PATCH 23/24] added property for CDC and added tests for test SSL with CDC together --- .../util/MySqlSslConnectionUtils.java | 2 +- .../source/mysql/MySqlCdcProperties.java | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java index 015b478f9506..f66da5436dc4 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/MySqlSslConnectionUtils.java @@ -57,7 +57,7 @@ public static Map obtainConnection(final JsonNode encryption) { return additionalParameters; } - private static String checkOrCreatePassword(final JsonNode encryption) { + public static String checkOrCreatePassword(final JsonNode encryption) { String sslPassword = encryption.has(PARAM_CLIENT_KEY_PASSWORD) ? encryption.get(PARAM_CLIENT_KEY_PASSWORD).asText() : ""; var keyStorePassword = RandomStringUtils.randomAlphanumeric(10); if (sslPassword.isEmpty()) { diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java index 5db5d2b71566..a19100946707 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java @@ -11,6 +11,8 @@ import java.util.Properties; +import static io.airbyte.integrations.util.MySqlSslConnectionUtils.checkOrCreatePassword; + public class MySqlCdcProperties { static Properties getDebeziumProperties(final JsonNode config) { @@ -48,10 +50,31 @@ static Properties getDebeziumProperties(final JsonNode config) { // https://debezium.io/documentation/reference/1.9/connectors/mysql.html#mysql-property-binary-handling-mode props.setProperty("binary.handling.mode", "base64"); props.setProperty("database.include.list", config.get("database").asText()); + // Check params for SSL connection in config and add properties for CDC SSL connection // https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-property-database-ssl-mode if (!config.has(JdbcUtils.SSL_KEY) || config.get(JdbcUtils.SSL_KEY).asBoolean()) { if (config.has(JdbcUtils.SSL_MODE_KEY) && config.get(JdbcUtils.SSL_MODE_KEY).has(JdbcUtils.MODE_KEY)) { props.setProperty("database.ssl.mode", config.get(JdbcUtils.SSL_MODE_KEY).get(JdbcUtils.MODE_KEY).asText()); + final var method = config.get(JdbcUtils.SSL_MODE_KEY).get(JdbcUtils.MODE_KEY).asText(); + if (method.equals("verify_ca") || method.equals("verify_identity")) { + var sslPassword = checkOrCreatePassword(config.get(JdbcUtils.SSL_MODE_KEY)); + props.setProperty("database.history.producer.security.protocol", "SSL"); + props.setProperty("database.history.producer.ssl.truststore.location", "customtruststore.jks"); + props.setProperty("database.history.producer.ssl.truststore.password", sslPassword); + props.setProperty("database.history.producer.ssl.key.password", sslPassword); + + props.setProperty("database.history.consumer.security.protocol", "SSL"); + props.setProperty("database.history.consumer.ssl.truststore.location", "customtruststore.jks"); + props.setProperty("database.history.consumer.ssl.truststore.password", sslPassword); + props.setProperty("database.history.consumer.ssl.key.password", sslPassword); + if (method.equals("verify_identity")) { + props.setProperty("database.history.producer.ssl.keystore.location", "customkeystore.jks"); + props.setProperty("database.history.producer.ssl.keystore.password", sslPassword); + + props.setProperty("database.history.consumer.ssl.keystore.location", "customkeystore.jks"); + props.setProperty("database.history.consumer.ssl.keystore.password", sslPassword); + } + } } else { props.setProperty("database.ssl.mode", "required"); } From 099f89da7547b1c142849806a9b2dd9347f726ff Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 16 Aug 2022 19:55:51 +0000 Subject: [PATCH 24/24] auto-bump connector version [ci skip] --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 133 +++++++++++++++++- 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f8f8bd47cfbc..78bcaa2017bd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -639,7 +639,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 0.6.2 + dockerImageTag: 0.6.3 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 6bce86b8145b..ac7d180aed53 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5942,7 +5942,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:0.6.2" +- dockerImage: "airbyte/source-mysql:0.6.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql" connectionSpecification: @@ -6000,6 +6000,135 @@ type: "boolean" default: true order: 6 + ssl_mode: + title: "SSL modes" + description: "SSL connection modes.
  • preferred - Automatically\ + \ attempt SSL connection. If the MySQL server does not support SSL, continue\ + \ with a regular connection.
  • required - Always connect\ + \ with SSL. If the MySQL server doesn’t support SSL, the connection will\ + \ not be established. Certificate Authority (CA) and Hostname are not\ + \ verified.
  • verify-ca - Always connect with SSL. Verifies\ + \ CA, but allows connection even if Hostname does not match.
  • Verify\ + \ Identity - Always connect with SSL. Verify both CA and Hostname.
    • Read\ + \ more in the docs." + type: "object" + order: 7 + oneOf: + - title: "preferred" + description: "Preferred SSL mode." + required: + - "mode" + properties: + mode: + type: "string" + const: "preferred" + enum: + - "preferred" + default: "preferred" + order: 0 + - title: "required" + description: "Require SSL mode." + required: + - "mode" + properties: + mode: + type: "string" + const: "required" + enum: + - "required" + default: "required" + order: 0 + - title: "Verify CA" + description: "Verify CA SSL mode." + required: + - "mode" + - "ca_certificate" + properties: + mode: + type: "string" + const: "verify_ca" + enum: + - "verify_ca" + default: "verify_ca" + order: 0 + ca_certificate: + type: "string" + title: "CA certificate" + description: "CA certificate" + airbyte_secret: true + multiline: true + order: 1 + client_certificate: + type: "string" + title: "Client certificate" + description: "Client certificate (this is not a required field, but\ + \ if you want to use it, you will need to add the Client key\ + \ as well)" + airbyte_secret: true + multiline: true + order: 2 + client_key: + type: "string" + title: "Client key" + description: "Client key (this is not a required field, but if you\ + \ want to use it, you will need to add the Client certificate\ + \ as well)" + airbyte_secret: true + multiline: true + order: 3 + client_key_password: + type: "string" + title: "Client key password (Optional)" + description: "Password for keystorage. This field is optional. If\ + \ you do not add it - the password will be generated automatically." + airbyte_secret: true + order: 4 + - title: "Verify Identity" + description: "Verify-full SSL mode." + required: + - "mode" + - "ca_certificate" + properties: + mode: + type: "string" + const: "verify_identity" + enum: + - "verify_identity" + default: "verify_identity" + order: 0 + ca_certificate: + type: "string" + title: "CA certificate" + description: "CA certificate" + airbyte_secret: true + multiline: true + order: 1 + client_certificate: + type: "string" + title: "Client certificate" + description: "Client certificate (this is not a required field, but\ + \ if you want to use it, you will need to add the Client key\ + \ as well)" + airbyte_secret: true + multiline: true + order: 2 + client_key: + type: "string" + title: "Client key" + description: "Client key (this is not a required field, but if you\ + \ want to use it, you will need to add the Client certificate\ + \ as well)" + airbyte_secret: true + multiline: true + order: 3 + client_key_password: + type: "string" + title: "Client key password (Optional)" + description: "Password for keystorage. This field is optional. If\ + \ you do not add it - the password will be generated automatically." + airbyte_secret: true + order: 4 replication_method: type: "string" title: "Replication Method" @@ -6008,7 +6137,7 @@ \ but will not be able to represent deletions incrementally. CDC uses\ \ the Binlog to detect inserts, updates, and deletes. This needs to be\ \ configured on the source database itself." - order: 7 + order: 8 default: "STANDARD" enum: - "STANDARD"