14895 Destination MongoDB Strict Encrypt Fixed Failing Integration Test

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java

@@ -14,58 +14,75 @@
 import io.airbyte.db.mongodb.MongoDatabase;
 import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType;
 import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
+import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator;
+import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import org.bson.Document;
-import org.junit.jupiter.api.BeforeAll;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.MongoDBContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
 
 public class MongodbDestinationStrictEncryptAcceptanceTest extends DestinationAcceptanceTest {
 
-  private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json");
-
   private static final String AUTH_TYPE = "auth_type";
   private static final String INSTANCE_TYPE = "instance_type";
   private static final String AIRBYTE_DATA = "_airbyte_data";
+  private static final String DATABASE_NAME = "test";
+  private static final String DATABASE_USER_NAME = "po";
+  private static final String DATABASE_PASSWORD = "password";
 
   private static JsonNode config;
-  private static JsonNode failCheckConfig;
-
-  private MongoDatabase mongoDatabase;
+  private static MongoDBContainer container;
   private final MongodbNameTransformer namingResolver = new MongodbNameTransformer();
 
-  @BeforeAll
-  static void setupConfig() throws IOException {
-    if (!Files.exists(CREDENTIALS_PATH)) {
-      throw new IllegalStateException(
-          "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH
-              + ". Override by setting setting path with the CREDENTIALS_PATH constant.");
-    }
-    final String credentialsJsonString = Files.readString(CREDENTIALS_PATH);
-    final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString);
+  @Override
+  protected String getImageName() {
+    return "airbyte/destination-mongodb-strict-encrypt:dev";
+  }
 
-    final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder()
-        .put("instance", MongoInstanceType.STANDALONE.getType())
-        .put(JdbcUtils.HOST_KEY, credentialsJson.get(JdbcUtils.HOST_KEY).asText())
-        .put(JdbcUtils.PORT_KEY, credentialsJson.get(JdbcUtils.PORT_KEY).asInt())
-        .build());
+  @Override
+  protected void setup(final TestDestinationEnv testEnv) {
+    List<String> list = new ArrayList<>();
+    list.add("MONGO_INITDB_ROOT_USERNAME=" + DATABASE_USER_NAME);
+    list.add("MONGO_INITDB_ROOT_PASSWORD=" + DATABASE_PASSWORD);
+    list.add("MONGO_INITDB_DATABASE=" + DATABASE_NAME);
+    container = new MongoDBContainer("mongo:4.0.10")
+        .withClasspathResourceMapping("mongod.conf", "/etc/mongod.conf", BindMode.READ_ONLY)
+        .withClasspathResourceMapping("mongodb.pem", "/etc/ssl/mongodb.pem", BindMode.READ_ONLY)
+        .withClasspathResourceMapping("mongodb-cert.crt", "/etc/ssl/mongodb-cert.crt", BindMode.READ_ONLY)
+        .withExposedPorts(27017)
+        .withCommand("mongod --auth --config /etc/mongod.conf")
+        .waitingFor(Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(120)));
+    container.setEnv(list);
+
+    container.start();
+  }
+
+  @Override
+  protected JsonNode getConfig() {
+    final JsonNode instanceConfig = getInstanceConfig();
 
     final JsonNode authConfig = Jsons.jsonNode(ImmutableMap.builder()
         .put("authorization", "login/password")
-        .put(JdbcUtils.USERNAME_KEY, credentialsJson.get("user").asText())
-        .put(JdbcUtils.PASSWORD_KEY, credentialsJson.get(JdbcUtils.PASSWORD_KEY).asText())
+        .put(JdbcUtils.USERNAME_KEY, DATABASE_USER_NAME)
+        .put(JdbcUtils.PASSWORD_KEY, DATABASE_PASSWORD)
         .build());
 
     config = Jsons.jsonNode(ImmutableMap.builder()
-        .put(JdbcUtils.DATABASE_KEY, credentialsJson.get(JdbcUtils.DATABASE_KEY).asText())
+        .put(JdbcUtils.DATABASE_KEY, DATABASE_NAME)
         .put(AUTH_TYPE, authConfig)
         .put(INSTANCE_TYPE, instanceConfig)
         .build());
+    return Jsons.clone(config);
+  }
 
-    failCheckConfig = Jsons.jsonNode(ImmutableMap.builder()
-        .put(JdbcUtils.DATABASE_KEY, credentialsJson.get(JdbcUtils.DATABASE_KEY).asText())
+  @Override
+  protected JsonNode getFailCheckConfig() {
+    final JsonNode instanceConfig = getInstanceConfig();
+    return Jsons.jsonNode(ImmutableMap.builder()
+        .put(JdbcUtils.DATABASE_KEY, DATABASE_NAME)
         .put(AUTH_TYPE, Jsons.jsonNode(ImmutableMap.builder()
             .put("authorization", "none")
             .build()))
@@ -74,25 +91,31 @@ static void setupConfig() throws IOException {
   }
 
   @Override
-  protected String getImageName() {
-    return "airbyte/destination-mongodb-strict-encrypt:dev";
+  protected TestDataComparator getTestDataComparator() {
+    return new AdvancedTestDataComparator();
   }
 
   @Override
-  protected JsonNode getConfig() {
-    return Jsons.clone(config);
+  protected boolean supportBasicDataTypeTest() {
+    return true;
   }
 
   @Override
-  protected JsonNode getFailCheckConfig() {
-    return Jsons.clone(failCheckConfig);
+  protected boolean supportArrayDataTypeTest() {
+    return true;
+  }
+
+  @Override
+  protected boolean supportObjectDataTypeTest() {
+    return true;
   }
 
   @Override
   protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
-                                           final String streamName,
-                                           final String namespace,
-                                           final JsonNode streamSchema) {
+      final String streamName,
+      final String namespace,
+      final JsonNode streamSchema) {
+    var mongoDatabase = getMongoDatabase();
     final var collection = mongoDatabase.getOrCreateNewCollection(namingResolver.getRawTableName(streamName));
     final List<JsonNode> result = new ArrayList<>();
     try (final MongoCursor<Document> cursor = collection.find().projection(excludeId()).iterator()) {
@@ -104,23 +127,27 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv testEnv,
   }
 
   @Override
-  protected void setup(final TestDestinationEnv testEnv) {
-    final String connectionString = String.format("mongodb://%s:%s@%s:%s/%s?authSource=admin&ssl=true",
+  protected void tearDown(final TestDestinationEnv testEnv) throws Exception {
+    container.close();
+  }
+
+  private MongoDatabase getMongoDatabase() {
+    final String connectionString = String.format("mongodb://%s:%s@%s:%s/%s?authSource=admin&tls=true",
         config.get(AUTH_TYPE).get(JdbcUtils.USERNAME_KEY).asText(),
         config.get(AUTH_TYPE).get(JdbcUtils.PASSWORD_KEY).asText(),
         config.get(INSTANCE_TYPE).get(JdbcUtils.HOST_KEY).asText(),
         config.get(INSTANCE_TYPE).get(JdbcUtils.PORT_KEY).asText(),
         config.get(JdbcUtils.DATABASE_KEY).asText());
 
-    mongoDatabase = new MongoDatabase(connectionString, config.get(JdbcUtils.DATABASE_KEY).asText());
+    return new MongoDatabase(connectionString, config.get(JdbcUtils.DATABASE_KEY).asText());
   }
 
-  @Override
-  protected void tearDown(final TestDestinationEnv testEnv) throws Exception {
-    for (final String collectionName : mongoDatabase.getCollectionNames()) {
-      mongoDatabase.getDatabase().getCollection(collectionName).drop();
-    }
-    mongoDatabase.close();
+  private JsonNode getInstanceConfig() {
+    return Jsons.jsonNode(ImmutableMap.builder()
+        .put("instance", MongoInstanceType.STANDALONE.getType())
+        .put(JdbcUtils.HOST_KEY, container.getHost())
+        .put(JdbcUtils.PORT_KEY, container.getMappedPort(27017))
+        .build());
   }
 
 }

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/resources/mongod.conf

@@ -0,0 +1,11 @@
+net:
+  bindIp: 0.0.0.0
+  port: 27017
+  ssl:
+    PEMKeyFile: /etc/ssl/mongodb.pem
+    mode: allowSSL
+    allowConnectionsWithoutCertificates: true
+    allowInvalidCertificates: true
+    allowInvalidHostnames: true
+    disabledProtocols: TLS1_0,TLS1_1
+

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/resources/mongodb-cert.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDOTCCAiECFGAxiWb4qfyQIgo4eGN2xML8CBapMA0GCSqGSIb3DQEBCwUAMFkx
+CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0yMjA3
+MjkxMDEwMDRaFw0yMzEyMTExMDEwMDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQI
+DApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQx
+EjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBALFqVajw2b6vPWsIcEh3+SxBsf4zEbm6f0L83AS1RH3+zSLQFcd5Bk886Uk5
+F98b2q6bfEKE1KT9PaUiRukq7carLmEyVqlGwWTHFoT756R3626u3zFgaLrqi+ss
+YVS3cy5BgACOdo27G5VjERwukj+9UKZcyBhobzZ0BQgiosMrHOeR2zja6HKf+qju
+9q6DmKKkVjqM1fbV9WK45tKzAZ3Quy1JXCT1BfLhkVXK3B0LsuX2JnmPVp7O5cAF
+aj/7hyAMmvI9ZnZWIB5LleiSTLI3QTAyv+j1rYZtPo7M4gceGRKLHIDKu8wL+kEt
+j8sZJTxifKeFSjPwBYwbgZNnHSUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAFzDl
+pxrcHJFUgf4rNXFvND+iq4K1AtqEnd4Ze7N3pSHEIUj4YNdiMZP06nP5FAJklRkP
+GsMMeR8s+jfZtPgTlwrW2IKcdYZAkA91VoEDs6IK2VzC7GiEsUPaGh4/2/DjXkSZ
+9Ie4a4zfRVHCvkRMO5q9/KKpdTo1vwc83mmxMSKQKEv8NKbEhVc3VlCZV1MvpMdc
+7He0Jqfg53+yLZv33w6NA4X8QIWdoBuK02ZP7ZhEBcMJOByetsuaHXjadsOpP4LM
+7KaP6zUq9kteKD6TgiCTmV6U/tpDvhS3YdVBEdd0aPlBeWGJ+GzubbOUUJOYSkt0
+jQ/Z56qy7eSveCa5ZQ==
+-----END CERTIFICATE-----

airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/resources/mongodb.pem

@@ -0,0 +1,47 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAsWpVqPDZvq89awhwSHf5LEGx/jMRubp/QvzcBLVEff7NItAV
+x3kGTzzpSTkX3xvarpt8QoTUpP09pSJG6SrtxqsuYTJWqUbBZMcWhPvnpHfrbq7f
+MWBouuqL6yxhVLdzLkGAAI52jbsblWMRHC6SP71QplzIGGhvNnQFCCKiwysc55Hb
+ONrocp/6qO72roOYoqRWOozV9tX1Yrjm0rMBndC7LUlcJPUF8uGRVcrcHQuy5fYm
+eY9Wns7lwAVqP/uHIAya8j1mdlYgHkuV6JJMsjdBMDK/6PWthm0+jsziBx4ZEosc
+gMq7zAv6QS2PyxklPGJ8p4VKM/AFjBuBk2cdJQIDAQABAoIBABH7uFEWkMu+UZhu
+04rU9mHhCty8aniMCehNXzPu99FZqYt00jBU+SnA/9iXO+82kJUQV6kac7+Bl+UM
+M2nhGxb/NQVPJ++ip2nCsGqsag+GxEuE2ETWA+embKZa6ka/DaLLquDjj+rP43zh
+/NMBeXuSggFK+FFZDKTVX8kyUZM8uU3GD7/qDm4VWtpdfVtwOVDP7Hexx6OQAcI/
+TXCssV48w+dWw/DrcAVeUo/s3TMy4QNtbHGmy6F3xLzmKi+il+uf9Fgpbm/x4+Yr
+6hEuEH4hi+rCoNQ876ts0c/hHGLhbc2VuFdC3GesRFEXOWb5p0CDYrP+fDfV56ee
+weac9WUCgYEA1lO0r2+jbSh89w1L99A71mxgVVnV+0pLEKWgk+T0lZfEQI1FDH7C
+KBFRw+G7Ck0rpsxf0674aVGI6iOjuYIxeSZOarIYt02en08/JzGfJYeUnxq9H6iO
+zefnqY6LjnnknGpvTNOA34NhQ0I/yr7dVL1G+1hyRE9Wh9NbxAyR++sCgYEA0+lT
+EtCpa/JR1rkuZNbluwETW/R5YAlkMK9Vw37eSk5+KDq3YYD+/5u4RNSyF96X3BI/
+lP6kwDQhyfMht0ziiudJ4rXEKjspzzC1QMZvwU5lW5aTRtATe8GmBVjubGiBkB4C
+153AGrFiTfggHkqVhPU8mZKLhPmY7sS30fm3Vy8CgYA+42TbLWjIuN6iJzapSmKk
+cjx6Dtgo2g3YnqDVhJL36gZd4pXmKrORyjMxxOR7C1XJpaJgnLCrRo9kFH0QC9G0
+eLb9u7Bg6Neoazpep8LI4lWLse7YgT8Vz/w71mXAsqeP9fMemdjwvL4nXCMXvJV5
+LejMC5HL1XPQewYsKnwTbQKBgQC1fAsAfyHJ6BK8wYWBKSZAXLB2vU5BuGlM5Kyf
+rAS2QLj8vSSWZcJ9qKBMmpgXpzQJbrtzYA466arSaN5EPJ179o8pkR/6RwcMgZuJ
+aSkGtZxKrw9cwxEcmQEVbU9sGqhzg7BNdrWyZp5ZRhpDtv6MivamU4IXlServV/s
+fV/OUwKBgE+5Bs3LZf5CKaSHnUagESXX2gt0917fdkaeaoHPkJWH9LcwrNYs+jC0
+eZnHv+exz2au62Abaq6xhEDQDj+U0aI1Um+i5o+TPklXrLf9WG8HhOdO/sm0+VIa
+1Bg4b/U8cyyxudhpg9mgEpGmujp1244y+dXMk2W/3seqLyLOIg8z
+-----END RSA PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIDOTCCAiECFGAxiWb4qfyQIgo4eGN2xML8CBapMA0GCSqGSIb3DQEBCwUAMFkx
+CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0yMjA3
+MjkxMDEwMDRaFw0yMzEyMTExMDEwMDRaMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQI
+DApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQx
+EjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBALFqVajw2b6vPWsIcEh3+SxBsf4zEbm6f0L83AS1RH3+zSLQFcd5Bk886Uk5
+F98b2q6bfEKE1KT9PaUiRukq7carLmEyVqlGwWTHFoT756R3626u3zFgaLrqi+ss
+YVS3cy5BgACOdo27G5VjERwukj+9UKZcyBhobzZ0BQgiosMrHOeR2zja6HKf+qju
+9q6DmKKkVjqM1fbV9WK45tKzAZ3Quy1JXCT1BfLhkVXK3B0LsuX2JnmPVp7O5cAF
+aj/7hyAMmvI9ZnZWIB5LleiSTLI3QTAyv+j1rYZtPo7M4gceGRKLHIDKu8wL+kEt
+j8sZJTxifKeFSjPwBYwbgZNnHSUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAFzDl
+pxrcHJFUgf4rNXFvND+iq4K1AtqEnd4Ze7N3pSHEIUj4YNdiMZP06nP5FAJklRkP
+GsMMeR8s+jfZtPgTlwrW2IKcdYZAkA91VoEDs6IK2VzC7GiEsUPaGh4/2/DjXkSZ
+9Ie4a4zfRVHCvkRMO5q9/KKpdTo1vwc83mmxMSKQKEv8NKbEhVc3VlCZV1MvpMdc
+7He0Jqfg53+yLZv33w6NA4X8QIWdoBuK02ZP7ZhEBcMJOByetsuaHXjadsOpP4LM
+7KaP6zUq9kteKD6TgiCTmV6U/tpDvhS3YdVBEdd0aPlBeWGJ+GzubbOUUJOYSkt0
+jQ/Z56qy7eSveCa5ZQ==
+-----END CERTIFICATE-----

airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java

@@ -40,9 +40,9 @@ public class MongodbDestination extends BaseConnector implements Destination {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(MongodbDestination.class);
 
-  private static final String MONGODB_SERVER_URL = "mongodb://%s%s:%s/%s?authSource=admin&ssl=%s";
+  private static final String MONGODB_SERVER_URL = "mongodb://%s%s:%s/%s?authSource=admin&tls=%s";
   private static final String MONGODB_CLUSTER_URL = "mongodb+srv://%s%s/%s?retryWrites=true&w=majority&tls=true";
-  private static final String MONGODB_REPLICA_URL = "mongodb://%s%s/%s?authSource=admin&directConnection=false&ssl=true";
+  private static final String MONGODB_REPLICA_URL = "mongodb://%s%s/%s?authSource=admin&directConnection=false&tls=true";
   private static final String INSTANCE_TYPE = "instance_type";
   private static final String INSTANCE = "instance";
   private static final String CLUSTER_URL = "cluster_url";

airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationTest.java

@@ -74,22 +74,22 @@ private static Stream<Arguments> configAndDataProvider() {
             .put("instance_type", standaloneConfig)
             .put(JdbcUtils.DATABASE_KEY, "dbName")
             .put("auth_type", authConfig).build()),
-            "mongodb://user:pass@localhost:27017/dbName?authSource=admin&ssl=false"),
+            "mongodb://user:pass@localhost:27017/dbName?authSource=admin&tls=false"),
         arguments(Jsons.jsonNode(ImmutableMap.builder()
             .put("instance_type", standaloneTlsConfig)
             .put(JdbcUtils.DATABASE_KEY, "dbName")
             .put("auth_type", noneAuthConfig).build()),
-            "mongodb://localhost:27017/dbName?authSource=admin&ssl=true"),
+            "mongodb://localhost:27017/dbName?authSource=admin&tls=true"),
         arguments(Jsons.jsonNode(ImmutableMap.builder()
             .put("instance_type", replicaWithNameConfig)
             .put(JdbcUtils.DATABASE_KEY, "dbName")
             .put("auth_type", authConfig).build()),
-            "mongodb://user:pass@localhost1:27017,localhost2:27017/dbName?authSource=admin&directConnection=false&ssl=true&replicaSet=replicaName"),
+            "mongodb://user:pass@localhost1:27017,localhost2:27017/dbName?authSource=admin&directConnection=false&tls=true&replicaSet=replicaName"),
         arguments(Jsons.jsonNode(ImmutableMap.builder()
             .put("instance_type", replicaWithoutNameConfig)
             .put(JdbcUtils.DATABASE_KEY, "dbName")
             .put("auth_type", noneAuthConfig).build()),
-            "mongodb://localhost1:27017,localhost2:27017/dbName?authSource=admin&directConnection=false&ssl=true"),
+            "mongodb://localhost1:27017,localhost2:27017/dbName?authSource=admin&directConnection=false&tls=true"),
         arguments(Jsons.jsonNode(ImmutableMap.builder()
             .put("instance_type", atlasConfig)
             .put(JdbcUtils.DATABASE_KEY, "dbName")
@@ -106,13 +106,13 @@ private static Stream<Arguments> configAndDataProvider() {
             .put(JdbcUtils.PORT_KEY, "27017")
             .put(JdbcUtils.DATABASE_KEY, "dbName")
             .put("auth_type", authConfig).build()),
-            "mongodb://user:pass@localhost:27017/dbName?authSource=admin&ssl=false"),
+            "mongodb://user:pass@localhost:27017/dbName?authSource=admin&tls=false"),
         arguments(Jsons.jsonNode(ImmutableMap.builder()
             .put(JdbcUtils.HOST_KEY, "localhost")
             .put(JdbcUtils.PORT_KEY, "27017")
             .put(JdbcUtils.DATABASE_KEY, "dbName")
             .put("auth_type", noneAuthConfig).build()),
-            "mongodb://localhost:27017/dbName?authSource=admin&ssl=false"));
+            "mongodb://localhost:27017/dbName?authSource=admin&tls=false"));
   }
 
 }