From 93a1a267e74ce8197a608c5a2cf17fe422d3d098 Mon Sep 17 00:00:00 2001 From: irynakruk Date: Fri, 17 Sep 2021 20:19:19 +0100 Subject: [PATCH 1/8] updated ssl configs; fixed replica connection; fixed incremental read; --- .../io/airbyte/db/mongodb/MongoUtils.java | 25 +++++++++++++ .../MongoDbSource.java | 37 +++++++++++-------- .../src/main/resources/spec.json | 4 +- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java index b3fa7aadd0d6..fb960f350bbb 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.api.client.util.DateTime; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import io.airbyte.commons.json.Jsons; @@ -36,13 +37,19 @@ import java.util.List; import java.util.Map; import org.bson.BsonBinary; +import org.bson.BsonDateTime; import org.bson.BsonDocument; import org.bson.BsonDocumentReader; +import org.bson.BsonDouble; +import org.bson.BsonInt32; +import org.bson.BsonInt64; import org.bson.BsonReader; +import org.bson.BsonTimestamp; import org.bson.BsonType; import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.Decimal128; +import org.bson.types.ObjectId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,6 +77,24 @@ public static JsonNode toJsonNode(final Document document, final List co return objectNode; } + public static Object getBsonValue(BsonType type, String value) { + try { + return switch (type) { + case INT32 -> new BsonInt32(Integer.parseInt(value)); + case INT64 -> new BsonInt64(Long.parseLong(value)); + case DOUBLE -> new BsonDouble(Double.parseDouble(value)); + case DECIMAL128 -> Decimal128.parse(value); + case TIMESTAMP -> new BsonTimestamp(Long.parseLong(value)); + case DATE_TIME -> new BsonDateTime(new DateTime(value).getValue()); + case OBJECT_ID -> new ObjectId(value); + default -> null; + }; + } catch (Exception e) { + LOGGER.error("Failed to get BsonValue for field type " + type, e.getMessage()); + return null; + } + } + private static void readBson(final Document document, final ObjectNode o, final List columnNames) { final BsonDocument bsonDocument = toBsonDocument(document); try (BsonReader reader = new BsonDocumentReader(bsonDocument)) { diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java index c2a5aaaf7ed9..5e133069b995 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java @@ -61,9 +61,9 @@ public class MongoDbSource extends AbstractDbSource { private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbSource.class); - private static final String MONGODB_SERVER_URL = "mongodb://%s%s:%s/?"; - private static final String MONGODB_CLUSTER_URL = "mongodb+srv://%s%s/%s?retryWrites=true&w=majority&"; - private static final String MONGODB_REPLICA_URL = "mongodb://%s%s/?replicaSet=%s&"; + private static final String MONGODB_SERVER_URL = "mongodb://%s%s:%s/%s?authSource=%s"; + private static final String MONGODB_CLUSTER_URL = "mongodb+srv://%s%s/%s?authSource=%s&retryWrites=true&w=majority"; + private static final String MONGODB_REPLICA_URL = "mongodb://%s%s/%s?authSource=%s&directConnection=false"; private static final String USER = "user"; private static final String PASSWORD = "password"; private static final String INSTANCE_TYPE = "instance_type"; @@ -90,26 +90,31 @@ public JsonNode toDatabaseConfig(JsonNode config) { ? String.format("%s:%s@", config.get(USER).asText(), config.get(PASSWORD).asText()) : StringUtils.EMPTY; + StringBuilder connectionStrBuilder = new StringBuilder(); JsonNode instanceConfig = config.get(INSTANCE_TYPE); - String instanceConnectUrl; if (instanceConfig.has(HOST) && instanceConfig.has(PORT)) { - instanceConnectUrl = String.format(MONGODB_SERVER_URL, - credentials, instanceConfig.get(HOST).asText(), instanceConfig.get(PORT).asText()); + // Standalone MongoDb Instance + connectionStrBuilder.append(String.format(MONGODB_SERVER_URL, credentials, instanceConfig.get(HOST).asText(), instanceConfig.get(PORT).asText(), + config.get(DATABASE).asText(), config.get(AUTH_SOURCE).asText())); } else if (instanceConfig.has(CLUSTER_URL)) { - instanceConnectUrl = String.format(MONGODB_CLUSTER_URL, - credentials, instanceConfig.get(CLUSTER_URL).asText(), config.get(DATABASE).asText()); + // MongoDB Atlas + connectionStrBuilder.append( + String.format(MONGODB_CLUSTER_URL, credentials, instanceConfig.get(CLUSTER_URL).asText(), config.get(DATABASE).asText(), + config.get(AUTH_SOURCE).asText())); } else { - instanceConnectUrl = String.format(MONGODB_REPLICA_URL, - credentials, instanceConfig.get(SERVER_ADDRESSES).asText(), config.get(REPLICA_SET).asText()); + // Replica Set & Shard + connectionStrBuilder.append( + String.format(MONGODB_REPLICA_URL, credentials, instanceConfig.get(SERVER_ADDRESSES).asText(), config.get(DATABASE).asText(), + config.get(AUTH_SOURCE).asText())); + if (instanceConfig.has(REPLICA_SET)) { + connectionStrBuilder.append(String.format("&replicaSet=%s", instanceConfig.get(REPLICA_SET).asText())); + } } - - String options = "authSource=".concat(config.get(AUTH_SOURCE).asText()); if (config.get(TLS).asBoolean()) { - options.concat("&tls=true"); + connectionStrBuilder.append("&ssl=true"); } - return Jsons.jsonNode(ImmutableMap.builder() - .put("connectionString", instanceConnectUrl + options) + .put("connectionString", connectionStrBuilder.toString()) .put("database", config.get(DATABASE).asText()) .build()); } @@ -207,7 +212,7 @@ public AutoCloseableIterator queryTableIncremental(MongoDatabase datab String cursorField, BsonType cursorFieldType, String cursor) { - Bson greaterComparison = gt(cursorField, cursor); + Bson greaterComparison = gt(cursorField, MongoUtils.getBsonValue(cursorFieldType, cursor)); return queryTable(database, columnNames, tableName, greaterComparison); } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json index 4b363cc363a0..7b1b8e38cf66 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json @@ -40,7 +40,7 @@ { "title": "Replica Set", "additionalProperties": false, - "required": ["server_addresses", "replica_set"], + "required": ["server_addresses"], "properties": { "server_addresses": { "title": "Server addresses", @@ -103,7 +103,7 @@ "title": "TLS connection", "type": "boolean", "description": "If this switch is enabled, TLS connections will be used to connect to MongoDB.", - "default": false, + "default": true, "order": 5 } } From dd04c26dea49f97e5c543857111944ba58e524af Mon Sep 17 00:00:00 2001 From: irynakruk Date: Mon, 20 Sep 2021 23:48:15 +0100 Subject: [PATCH 2/8] added MongoDbSourceAtlasAcceptanceTest with ssl enabled --- ... MongoDbSourceAbstractAcceptanceTest.java} | 55 ++---------- .../MongoDbSourceAtlasAcceptanceTest.java | 88 +++++++++++++++++++ ...MongoDbSourceStandaloneAcceptanceTest.java | 79 +++++++++++++++++ 3 files changed, 174 insertions(+), 48 deletions(-) rename airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/{MongoDbSourceAcceptanceTest.java => MongoDbSourceAbstractAcceptanceTest.java} (64%) create mode 100644 airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java similarity index 64% rename from airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAcceptanceTest.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java index c9bd7adee725..e935e9e83ee2 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java @@ -25,14 +25,11 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.mongodb.client.MongoCollection; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.mongodb.MongoDatabase; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; @@ -44,15 +41,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import org.bson.Document; -import org.testcontainers.containers.MongoDBContainer; -import org.testcontainers.utility.DockerImageName; -public class MongoDbSourceAcceptanceTest extends SourceAcceptanceTest { +public abstract class MongoDbSourceAbstractAcceptanceTest extends SourceAcceptanceTest { - private MongoDBContainer mongoDBContainer; - private JsonNode config; - private MongoDatabase database; + protected static final String DATABASE_NAME = "test"; + protected static final String COLLECTION_NAME = "acceptance_test"; + + protected JsonNode config; + protected MongoDatabase database; @Override protected String getImageName() { @@ -64,43 +60,6 @@ protected JsonNode getConfig() throws Exception { return config; } - @Override - protected void setupEnvironment(TestDestinationEnv environment) throws Exception { - mongoDBContainer = new MongoDBContainer(DockerImageName.parse("mongo:4.0.10")); - mongoDBContainer.start(); - - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("host", mongoDBContainer.getHost()) - .put("port", mongoDBContainer.getFirstMappedPort()) - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("instance_type", instanceConfig) - .put("database", "test") - .put("auth_source", "admin") - .put("tls", false) - .build()); - - String connectionString = String.format("mongodb://%s:%s/", - mongoDBContainer.getHost(), - mongoDBContainer.getFirstMappedPort()); - - database = new MongoDatabase(connectionString, "test"); - - MongoCollection collection = database.createCollection("acceptance_test"); - var doc1 = new Document("id", "0001").append("name", "Test"); - var doc2 = new Document("id", "0002").append("name", "Mongo"); - var doc3 = new Document("id", "0003").append("name", "Source"); - - collection.insertMany(List.of(doc1, doc2, doc3)); - } - - @Override - protected void tearDown(TestDestinationEnv testEnv) throws Exception { - database.close(); - mongoDBContainer.close(); - } - @Override protected ConnectorSpecification getSpec() throws Exception { return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); @@ -115,7 +74,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { .withDestinationSyncMode(DestinationSyncMode.APPEND) .withCursorField(List.of("_id")) .withStream(CatalogHelpers.createAirbyteStream( - "test.acceptance_test", + DATABASE_NAME + "." + COLLECTION_NAME, Field.of("_id", JsonSchemaPrimitive.STRING), Field.of("id", JsonSchemaPrimitive.STRING), Field.of("name", JsonSchemaPrimitive.STRING)) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java new file mode 100644 index 000000000000..5bceaf356ed6 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java @@ -0,0 +1,88 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.mongodb.client.MongoCollection; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.mongodb.MongoDatabase; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.bson.Document; + +public class MongoDbSourceAtlasAcceptanceTest extends MongoDbSourceAbstractAcceptanceTest { + + private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); + + @Override + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { + if (!Files.exists(CREDENTIALS_PATH)) { + throw new IllegalStateException( + "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH + + ". Override by setting setting path with the CREDENTIALS_PATH constant."); + } + + final String credentialsJsonString = new String(Files.readAllBytes(CREDENTIALS_PATH)); + final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString); + + final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() + .put("cluster_url", credentialsJson.get("cluster_url").asText()) + .build()); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("user", credentialsJson.get("user").asText()) + .put("password", credentialsJson.get("password").asText()) + .put("instance_type", instanceConfig) + .put("database", DATABASE_NAME) + .put("auth_source", "admin") + .put("tls", true) + .build()); + + String connectionString = String.format("mongodb+srv://%s:%s@%s/%s?authSource=admin&retryWrites=true&w=majority&ssl=true", + config.get("user").asText(), + config.get("password").asText(), + config.get("instance_type").get("cluster_url").asText(), + config.get("database").asText()); + + database = new MongoDatabase(connectionString, DATABASE_NAME); + + MongoCollection collection = database.createCollection(COLLECTION_NAME); + var doc1 = new Document("id", "0001").append("name", "Test"); + var doc2 = new Document("id", "0002").append("name", "Mongo"); + var doc3 = new Document("id", "0003").append("name", "Source"); + + collection.insertMany(List.of(doc1, doc2, doc3)); + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) throws Exception { + database.getDatabase().getCollection(COLLECTION_NAME).drop(); + database.close(); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java new file mode 100644 index 000000000000..89a97bb8ff49 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java @@ -0,0 +1,79 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.mongodb.client.MongoCollection; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.mongodb.MongoDatabase; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import java.util.List; +import org.bson.Document; +import org.testcontainers.containers.MongoDBContainer; +import org.testcontainers.utility.DockerImageName; + +public class MongoDbSourceStandaloneAcceptanceTest extends MongoDbSourceAbstractAcceptanceTest { + + private MongoDBContainer mongoDBContainer; + + @Override + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { + mongoDBContainer = new MongoDBContainer(DockerImageName.parse("mongo:4.0.10")); + mongoDBContainer.start(); + + final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() + .put("host", mongoDBContainer.getHost()) + .put("port", mongoDBContainer.getFirstMappedPort()) + .build()); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put("instance_type", instanceConfig) + .put("database", DATABASE_NAME) + .put("auth_source", "admin") + .put("tls", false) + .build()); + + String connectionString = String.format("mongodb://%s:%s/", + mongoDBContainer.getHost(), + mongoDBContainer.getFirstMappedPort()); + + database = new MongoDatabase(connectionString, DATABASE_NAME); + + MongoCollection collection = database.createCollection(COLLECTION_NAME); + var doc1 = new Document("id", "0001").append("name", "Test"); + var doc2 = new Document("id", "0002").append("name", "Mongo"); + var doc3 = new Document("id", "0003").append("name", "Source"); + + collection.insertMany(List.of(doc1, doc2, doc3)); + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) throws Exception { + database.close(); + mongoDBContainer.close(); + } + +} From b3de9aae07d001d9ea1cfc8f6001cfc092a53a40 Mon Sep 17 00:00:00 2001 From: irynakruk Date: Tue, 21 Sep 2021 23:58:45 -0400 Subject: [PATCH 3/8] updated docs, moved TLS option for standalone instance type, enabled it by default for other types --- .github/workflows/publish-command.yml | 1 + .github/workflows/test-command.yml | 1 + .../b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e.json | 2 +- .../main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-mongodb-v2/Dockerfile | 2 +- .../MongoDbSource.java | 12 +++++------- .../source-mongodb-v2/src/main/resources/spec.json | 14 +++++++------- .../sources/MongoDbSourceAtlasAcceptanceTest.java | 3 +-- .../MongoDbSourceStandaloneAcceptanceTest.java | 2 +- docs/integrations/sources/mongodb-v2.md | 13 ++++++++++--- tools/bin/ci_credentials.sh | 1 + 11 files changed, 30 insertions(+), 23 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 2c2363315da7..0b4643d5a7e5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -184,6 +184,7 @@ jobs: DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }} MSSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MSSQL_SSH_KEY_TEST_CREDS }} MSSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MSSQL_SSH_PWD_TEST_CREDS }} + MONGODB_TEST_CREDS: ${{ secrets.MONGODB_TEST_CREDS }} - run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD} ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 03346480a019..f178214df23e 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -179,6 +179,7 @@ jobs: DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }} MSSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MSSQL_SSH_KEY_TEST_CREDS }} MSSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MSSQL_SSH_PWD_TEST_CREDS }} + MONGODB_TEST_CREDS: ${{ secrets.MONGODB_TEST_CREDS }} - run: | ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} name: test ${{ github.event.inputs.connector }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e.json index e1cc4f3b4bc6..7f51ac905f9e 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e", "name": "MongoDb", "dockerRepository": "airbyte/source-mongodb-v2", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mongodb-v2", "icon": "mongodb.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index cb21cd049e9c..2b0a101a50ea 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -552,7 +552,7 @@ - sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e name: MongoDb dockerRepository: airbyte/source-mongodb-v2 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/mongodb-v2 icon: mongodb.svg sourceType: database diff --git a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile index f8d4e718992f..55efa989aad3 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile +++ b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java index 5e133069b995..41e069f51b30 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java @@ -61,9 +61,9 @@ public class MongoDbSource extends AbstractDbSource { private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbSource.class); - private static final String MONGODB_SERVER_URL = "mongodb://%s%s:%s/%s?authSource=%s"; - private static final String MONGODB_CLUSTER_URL = "mongodb+srv://%s%s/%s?authSource=%s&retryWrites=true&w=majority"; - private static final String MONGODB_REPLICA_URL = "mongodb://%s%s/%s?authSource=%s&directConnection=false"; + private static final String MONGODB_SERVER_URL = "mongodb://%s%s:%s/%s?authSource=%s&ssl=%s"; + private static final String MONGODB_CLUSTER_URL = "mongodb+srv://%s%s/%s?authSource=%s&retryWrites=true&w=majority&tls=true"; + private static final String MONGODB_REPLICA_URL = "mongodb://%s%s/%s?authSource=%s&directConnection=false&ssl=true"; private static final String USER = "user"; private static final String PASSWORD = "password"; private static final String INSTANCE_TYPE = "instance_type"; @@ -95,7 +95,7 @@ public JsonNode toDatabaseConfig(JsonNode config) { if (instanceConfig.has(HOST) && instanceConfig.has(PORT)) { // Standalone MongoDb Instance connectionStrBuilder.append(String.format(MONGODB_SERVER_URL, credentials, instanceConfig.get(HOST).asText(), instanceConfig.get(PORT).asText(), - config.get(DATABASE).asText(), config.get(AUTH_SOURCE).asText())); + config.get(DATABASE).asText(), config.get(AUTH_SOURCE).asText(), instanceConfig.get(TLS).asBoolean())); } else if (instanceConfig.has(CLUSTER_URL)) { // MongoDB Atlas connectionStrBuilder.append( @@ -110,9 +110,7 @@ public JsonNode toDatabaseConfig(JsonNode config) { connectionStrBuilder.append(String.format("&replicaSet=%s", instanceConfig.get(REPLICA_SET).asText())); } } - if (config.get(TLS).asBoolean()) { - connectionStrBuilder.append("&ssl=true"); - } + return Jsons.jsonNode(ImmutableMap.builder() .put("connectionString", connectionStrBuilder.toString()) .put("database", config.get(DATABASE).asText()) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json index 7b1b8e38cf66..785ae5541fd0 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json @@ -34,6 +34,13 @@ "default": 27017, "examples": ["27017"], "order": 1 + }, + "tls": { + "title": "TLS connection", + "type": "boolean", + "description": "Indicates whether TLS encryption protocol will be used to connect to MongoDB. It is recommended to use TLS connection if possible.", + "default": false, + "order": 2 } } }, @@ -98,13 +105,6 @@ "default": "admin", "examples": ["admin"], "order": 4 - }, - "tls": { - "title": "TLS connection", - "type": "boolean", - "description": "If this switch is enabled, TLS connections will be used to connect to MongoDB.", - "default": true, - "order": 5 } } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java index 5bceaf356ed6..f8bd4e98070a 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java @@ -60,10 +60,9 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception .put("instance_type", instanceConfig) .put("database", DATABASE_NAME) .put("auth_source", "admin") - .put("tls", true) .build()); - String connectionString = String.format("mongodb+srv://%s:%s@%s/%s?authSource=admin&retryWrites=true&w=majority&ssl=true", + String connectionString = String.format("mongodb+srv://%s:%s@%s/%s?authSource=admin&retryWrites=true&w=majority&tls=true", config.get("user").asText(), config.get("password").asText(), config.get("instance_type").get("cluster_url").asText(), diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java index 89a97bb8ff49..53b98822a156 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java @@ -47,13 +47,13 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() .put("host", mongoDBContainer.getHost()) .put("port", mongoDBContainer.getFirstMappedPort()) + .put("tls", false) .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put("instance_type", instanceConfig) .put("database", DATABASE_NAME) .put("auth_source", "admin") - .put("tls", false) .build()); String connectionString = String.format("mongodb://%s:%s/", diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index 683af0adb43a..15875183ecf8 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -33,10 +33,10 @@ Cursor field can not be nested. Currently only top level document properties are Cursor should **never** be blank. In case cursor is blank - the incremental sync results might be unpredictable and will totally rely on MongoDB comparison algorithm. -Only `datetime` and `integer` cursor types are supported. Cursor type is determined based on the cursor field name: +Only `datetime` and `number` cursor types are supported. Cursor type is determined based on the cursor field name: * `datetime` - if cursor field name contains a string from: `time`, `date`, `_at`, `timestamp`, `ts` -* `integer` - otherwise +* `number` - otherwise ## Getting started @@ -76,15 +76,21 @@ Make sure that MongoDB is accessible from external servers. Specific commands wi Your `READ_ONLY_USER` should now be ready for use with Airbyte. +### TLS/SSL on a Connection + +It is recommended to use encrypted connection. +Connection with TLS/SSL security protocol for MongoDb Atlas Cluster and Replica Set instances is enabled by default. Encryption is based on the underlying TLS/SSL support in the JDK. +To enable TSL/SSL connection with Standalone MongoDb instance, please refer to [MongoDb Documentation](https://docs.mongodb.com/manual/tutorial/configure-ssl/). + ### Сonfiguration Parameters * Database: database name * Authentication Source: specifies the database that the supplied credentials should be validated against. Defaults to `admin`. * User: username to use when connecting * Password: used to authenticate the user -* TSL: whether to use TSL connection * **Standalone MongoDb instance** * Host: URL of the database * Port: Port to use for connecting to the database + * TLS: indicates whether to create encrypted connection * **Replica Set** * Server addresses: the members of a replica set * Replica Set: A replica set name @@ -96,4 +102,5 @@ For more information regarding configuration parameters, please see [MongoDb Doc ## Changelog | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.1 | 2021-09-21 | [](https://github.com/airbytehq/airbyte/pull/) | Added support for MongoDB source via TLS/SSL | | 0.1.0 | 2021-08-30 | [5530](https://github.com/airbytehq/airbyte/pull/5530) | New source: MongoDb ported to java | diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index b7239471dbf8..15e0742f7ff7 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -98,6 +98,7 @@ write_standard_creds source-mailchimp "$MAILCHIMP_TEST_CREDS" write_standard_creds source-marketo-singer "$SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG" write_standard_creds source-microsoft-teams "$MICROSOFT_TEAMS_TEST_CREDS" write_standard_creds source-mixpanel "$MIXPANEL_INTEGRATION_TEST_CREDS" +write_standard_creds source-mongodb-v2 "$MONGODB_TEST_CREDS" write_standard_creds source-mssql "$MSSQL_RDS_TEST_CREDS" write_standard_creds source-okta "$SOURCE_OKTA_TEST_CREDS" write_standard_creds source-plaid "$PLAID_INTEGRATION_TEST_CREDS" From d78e3f5a913b7ca73339ed88bfcba8bfce691023 Mon Sep 17 00:00:00 2001 From: irynakruk Date: Wed, 22 Sep 2021 00:17:19 -0400 Subject: [PATCH 4/8] Update mongodb-v2.md --- docs/integrations/sources/mongodb-v2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index 15875183ecf8..699ea85d1fcc 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -102,5 +102,5 @@ For more information regarding configuration parameters, please see [MongoDb Doc ## Changelog | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| 0.1.1 | 2021-09-21 | [](https://github.com/airbytehq/airbyte/pull/) | Added support for MongoDB source via TLS/SSL | +| 0.1.1 | 2021-09-21 | [6364](https://github.com/airbytehq/airbyte/pull/6364) | Source MongoDb: added support via TLS/SSL | | 0.1.0 | 2021-08-30 | [5530](https://github.com/airbytehq/airbyte/pull/5530) | New source: MongoDb ported to java | From 7b9093dd0b4cfaa2a01cb5f3c24c99644385ca5b Mon Sep 17 00:00:00 2001 From: irynakruk Date: Wed, 22 Sep 2021 11:30:42 -0400 Subject: [PATCH 5/8] updated README.md --- .../connectors/source-mongodb-v2/README.md | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/README.md b/airbyte-integrations/connectors/source-mongodb-v2/README.md index 88c870a9c48a..0afe15641a88 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/README.md +++ b/airbyte-integrations/connectors/source-mongodb-v2/README.md @@ -26,7 +26,29 @@ the Dockerfile. We use `JUnit` for Java tests. ### Test Configuration -No specific configuration needed for testing, MongoDb Test Container is used. + +In order to test the MongoDb source, you need to have a standalone instance, a replica set or Atlas cluster. + +## Community Contributor + +As a community contributor, you will need to have an Atlas cluster to test MongoDb source. + +1. Create `secrets/credentials.json` file + 1. Insert below json to the file with your configuration + ``` + { + "database": "database_name", + "user": "user", + "password": "password", + "cluster_url": "cluster_url" + } + ``` + +## Airbyte Employee + +1. Access the `MONGODB_TEST_CREDS` secret on LastPass +1. Create a file with the contents at `secrets/credentials.json` + #### Acceptance Tests To run acceptance and custom integration tests: From 1b08e062d6b0e921c88752402c50cac3b9e072d9 Mon Sep 17 00:00:00 2001 From: irynakruk Date: Thu, 23 Sep 2021 10:41:22 -0400 Subject: [PATCH 6/8] updated spec.json --- .../connectors/source-mongodb-v2/src/main/resources/spec.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json index 785ae5541fd0..6d617d68d241 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json @@ -11,7 +11,7 @@ "instance_type": { "type": "object", "title": "MongoDb instance type", - "description": "MongoDb instance to connect to.", + "description": "MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", "order": 0, "oneOf": [ { @@ -38,7 +38,7 @@ "tls": { "title": "TLS connection", "type": "boolean", - "description": "Indicates whether TLS encryption protocol will be used to connect to MongoDB. It is recommended to use TLS connection if possible.", + "description": "Indicates whether TLS encryption protocol will be used to connect to MongoDB. It is recommended to use TLS connection if possible. For more information see documentation.", "default": false, "order": 2 } From d8411a48572b70ecf899540730d94da9e12a291a Mon Sep 17 00:00:00 2001 From: irynakruk Date: Thu, 23 Sep 2021 21:01:39 -0400 Subject: [PATCH 7/8] Code review changes --- .../src/main/java/io/airbyte/db/mongodb/MongoUtils.java | 8 ++++++-- .../connectors/source-mongodb-v2/README.md | 3 ++- .../MongoDbSource.java | 3 ++- .../source-mongodb-v2/src/main/resources/spec.json | 2 +- docs/integrations/sources/mongodb-v2.md | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java b/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java index fb960f350bbb..585f1e2a379b 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java @@ -44,12 +44,14 @@ import org.bson.BsonInt32; import org.bson.BsonInt64; import org.bson.BsonReader; +import org.bson.BsonString; import org.bson.BsonTimestamp; import org.bson.BsonType; import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.Decimal128; import org.bson.types.ObjectId; +import org.bson.types.Symbol; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,11 +89,13 @@ public static Object getBsonValue(BsonType type, String value) { case TIMESTAMP -> new BsonTimestamp(Long.parseLong(value)); case DATE_TIME -> new BsonDateTime(new DateTime(value).getValue()); case OBJECT_ID -> new ObjectId(value); - default -> null; + case SYMBOL -> new Symbol(value); + case STRING -> new BsonString(value); + default -> value; }; } catch (Exception e) { LOGGER.error("Failed to get BsonValue for field type " + type, e.getMessage()); - return null; + return value; } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/README.md b/airbyte-integrations/connectors/source-mongodb-v2/README.md index 0afe15641a88..45570207bfbc 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/README.md +++ b/airbyte-integrations/connectors/source-mongodb-v2/README.md @@ -27,7 +27,8 @@ We use `JUnit` for Java tests. ### Test Configuration -In order to test the MongoDb source, you need to have a standalone instance, a replica set or Atlas cluster. +No specific configuration needed for testing Standalone MongoDb instance, MongoDb Test Container is used. +In order to test the MongoDb Atlas or Replica set, you need to provide configuration parameters. ## Community Contributor diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java index 41e069f51b30..6b7dc2a80183 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java @@ -94,8 +94,9 @@ public JsonNode toDatabaseConfig(JsonNode config) { JsonNode instanceConfig = config.get(INSTANCE_TYPE); if (instanceConfig.has(HOST) && instanceConfig.has(PORT)) { // Standalone MongoDb Instance + var tls = config.has(TLS) ? config.get(TLS).asBoolean() : instanceConfig.get(TLS).asBoolean(); // for backward compatibility connectionStrBuilder.append(String.format(MONGODB_SERVER_URL, credentials, instanceConfig.get(HOST).asText(), instanceConfig.get(PORT).asText(), - config.get(DATABASE).asText(), config.get(AUTH_SOURCE).asText(), instanceConfig.get(TLS).asBoolean())); + config.get(DATABASE).asText(), config.get(AUTH_SOURCE).asText(), tls)); } else if (instanceConfig.has(CLUSTER_URL)) { // MongoDB Atlas connectionStrBuilder.append( diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json index 6d617d68d241..18d5a81ab88b 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json @@ -6,7 +6,7 @@ "title": "MongoDb Source Spec", "type": "object", "required": ["database"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "instance_type": { "type": "object", diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index 699ea85d1fcc..7fb8eb108679 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -79,7 +79,7 @@ Your `READ_ONLY_USER` should now be ready for use with Airbyte. ### TLS/SSL on a Connection It is recommended to use encrypted connection. -Connection with TLS/SSL security protocol for MongoDb Atlas Cluster and Replica Set instances is enabled by default. Encryption is based on the underlying TLS/SSL support in the JDK. +Connection with TLS/SSL security protocol for MongoDb Atlas Cluster and Replica Set instances is enabled by default. To enable TSL/SSL connection with Standalone MongoDb instance, please refer to [MongoDb Documentation](https://docs.mongodb.com/manual/tutorial/configure-ssl/). ### Сonfiguration Parameters From 66f8dd422f07698b806570277ee9b0fb03851862 Mon Sep 17 00:00:00 2001 From: irynakruk Date: Thu, 23 Sep 2021 21:07:55 -0400 Subject: [PATCH 8/8] updated ci_credentials.sh --- tools/bin/ci_credentials.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index e0256f0823e8..f67b8e1b0022 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -96,7 +96,7 @@ write_standard_creds source-marketo "$SOURCE_MARKETO_TEST_CREDS" write_standard_creds source-marketo-singer "$SOURCE_MARKETO_SINGER_INTEGRATION_TEST_CONFIG" write_standard_creds source-microsoft-teams "$MICROSOFT_TEAMS_TEST_CREDS" write_standard_creds source-mixpanel "$MIXPANEL_INTEGRATION_TEST_CREDS" -write_standard_creds source-mongodb-v2 "$MONGODB_TEST_CREDS" +write_standard_creds source-mongodb-v2 "$MONGODB_TEST_CREDS" "credentials.json" write_standard_creds source-mssql "$MSSQL_RDS_TEST_CREDS" write_standard_creds source-okta "$SOURCE_OKTA_TEST_CREDS" write_standard_creds source-plaid "$PLAID_INTEGRATION_TEST_CREDS"