From 5f2604f2699b2b50b7e0e6e4752a94f83cb2f432 Mon Sep 17 00:00:00 2001 From: ulixius9 Date: Fri, 5 Jan 2024 02:23:48 +0530 Subject: [PATCH 1/2] Fix #13053: Remove Connection URI cofig MongoDB --- .../source/database/mongodb/connection.py | 6 +- .../unit/topology/database/test_mongodb.py | 6 +- .../connectors/database/mongodb/index.md | 2 - .../connectors/database/mongodb/yaml.md | 20 ++--- .../migration/mysql/v130/Migration.java | 33 ++++++++ .../migration/postgres/v130/Migration.java | 33 ++++++++ .../migration/utils/v130/MigrationUtil.java | 82 +++++++++++++++++++ .../database/mongoDB/mongoDBValues.json | 49 ----------- .../database/mongoDBConnection.json | 56 ++++++++----- .../public/locales/en-US/Database/MongoDB.md | 12 --- 10 files changed, 191 insertions(+), 108 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v130/Migration.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v130/Migration.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v130/MigrationUtil.java delete mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDB/mongoDBValues.json diff --git a/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py b/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py index f27a3c475eaf..967a0993275d 100644 --- a/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py @@ -24,7 +24,6 @@ Workflow as AutomationWorkflow, ) from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import ( - MongoConnectionString, MongoDBConnection, ) from metadata.ingestion.connections.builders import get_connection_url_common @@ -36,10 +35,7 @@ def get_connection(connection: MongoDBConnection): """ Create connection """ - if isinstance(connection.connectionDetails, MongoConnectionString): - mongo_url = connection.connectionDetails.connectionURI - else: - mongo_url = get_connection_url_common(connection.connectionDetails) + mongo_url = get_connection_url_common(connection) return MongoClient(mongo_url) diff --git a/ingestion/tests/unit/topology/database/test_mongodb.py b/ingestion/tests/unit/topology/database/test_mongodb.py index 7bf974dc7a88..91dca40e121c 100644 --- a/ingestion/tests/unit/topology/database/test_mongodb.py +++ b/ingestion/tests/unit/topology/database/test_mongodb.py @@ -47,9 +47,9 @@ "serviceConnection": { "config": { "type": "MongoDB", - "connectionDetails": { - "connectionURI": "mongodb://ulixius:dummy_password@localhost:27017", - }, + "username": "ulixius", + "password": "dummy_password", + "hostPort": "localhost:27017" }, }, "sourceConfig": { diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/index.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/index.md index 21b8286af81b..aa7959d0b4d7 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/index.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/index.md @@ -61,8 +61,6 @@ To fetch the metadata from MongoDB to OpenMetadata, the MongoDB user must have a #### Connection Details -- **MongoDB Connection Details**: Choose between MongoDB Connection String and MongoDB Connection Values to authenticate with your mongodb cluster. -- **Connection URI**: MongoDB connection string is a concise string of parameters used to establish a connection between an OpenMetadata and a MongoDB database. For ex. `mongodb://username:password@mongodb0.example.com:27017`. - **Username**: Username to connect to Mongodb. This user must have access to perform `find` operation on collection and `listCollection` operations on database available in MongoDB. - **Password**: Password to connect to MongoDB. - **Host Port**: The hostPort parameter specifies the host and port of the MongoDB. This should be specified as a string in the format `hostname:port`. E.g., `localhost:27017`. diff --git a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/yaml.md b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/yaml.md index 6b650cf100d7..b4979754fa79 100644 --- a/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/yaml.md +++ b/openmetadata-docs/content/v1.3.x-SNAPSHOT/connectors/database/mongodb/yaml.md @@ -94,12 +94,6 @@ This is a sample config for MongoDB: {% /codeInfo %} -{% codeInfo srNumber=5 %} - -**connectionURI**: MongoDB connection string is a concise string of parameters used to establish a connection between an OpenMetadata and a MongoDB database. For ex. `mongodb://username:password@mongodb0.example.com:27017`. - -{% /codeInfo %} - {% codeInfo srNumber=6 %} **databaseName**: Optional name to give to the database in OpenMetadata. If left blank, we will use default as the database name. @@ -132,23 +126,19 @@ source: serviceConnection: config: type: MongoDB - connectionDetails: ``` ```yaml {% srNumber=1 %} - username: username + username: username ``` ```yaml {% srNumber=2 %} - password: password + password: password ``` ```yaml {% srNumber=3 %} - hostPort: localhost:27017 -``` -```yaml {% srNumber=5 %} - # connectionURI: mongodb://username:password@mongodb0.example.com:27017 + hostPort: localhost:27017 ``` ```yaml {% srNumber=7 %} - # connectionOptions: - # key: value + # connectionOptions: + # key: value ``` ```yaml {% srNumber=6 %} database: custom_database_name diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v130/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v130/Migration.java new file mode 100644 index 000000000000..40e55237eb69 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v130/Migration.java @@ -0,0 +1,33 @@ +package org.openmetadata.service.migration.mysql.v130; + +import static org.openmetadata.service.migration.utils.v130.MigrationUtil.migrateMongoDBConnStr; + +import lombok.SneakyThrows; +import org.jdbi.v3.core.Handle; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + private Handle handle; + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + public void initialize(Handle handle) { + super.initialize(handle); + this.handle = handle; + } + + @Override + @SneakyThrows + public void runDataMigration() { + String updateSqlQuery = + "UPDATE dbservice_entity de SET json = :json " + + "WHERE serviceType = 'MongoDB' " + + "AND id = :id"; + migrateMongoDBConnStr(handle, updateSqlQuery); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v130/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v130/Migration.java new file mode 100644 index 000000000000..d26c97e2ebba --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v130/Migration.java @@ -0,0 +1,33 @@ +package org.openmetadata.service.migration.postgres.v130; + +import static org.openmetadata.service.migration.utils.v130.MigrationUtil.migrateMongoDBConnStr; + +import lombok.SneakyThrows; +import org.jdbi.v3.core.Handle; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + private Handle handle; + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + public void initialize(Handle handle) { + super.initialize(handle); + this.handle = handle; + } + + @Override + @SneakyThrows + public void runDataMigration() { + String updateSqlQuery = + "UPDATE dbservice_entity de SET json = :json::jsonb " + + "WHERE serviceType = 'MongoDB' " + + "AND id = :id"; + migrateMongoDBConnStr(handle, updateSqlQuery); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v130/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v130/MigrationUtil.java new file mode 100644 index 000000000000..7ec78a475a11 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v130/MigrationUtil.java @@ -0,0 +1,82 @@ +package org.openmetadata.service.migration.utils.v130; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import org.jdbi.v3.core.Handle; +import org.openmetadata.schema.entity.services.DatabaseService; +import org.openmetadata.service.util.JsonUtils; + +public class MigrationUtil { + + private static final String GET_MONGO_DB_SERVICES = + "SELECT id, json from " + "dbservice_entity de WHERE " + "serviceType = 'MongoDB'"; + + private MigrationUtil() { + /* Cannot create object util class*/ + } + + private static Map extractConnectionURIDetails(String connectionString) { + Map connectionDetailsMap = new LinkedHashMap(); + try { + // Parse the MongoDB connection string + URI uri = new URI(connectionString); + + // Extract components + String username = uri.getUserInfo().split(":")[0]; + String password = uri.getUserInfo().split(":")[1]; + String host = uri.getHost(); + String scheme = uri.getScheme(); + int port = uri.getPort(); + String query = uri.getQuery(); + Map queryMap = new HashMap<>(); + if (query != null) { + String[] queryParams = query.split("&"); + System.out.println("Query Parameters:"); + for (String param : queryParams) { + queryMap.put(param.split("=")[0], param.split("=")[1]); + } + } + + // populate connection details map the extracted components + connectionDetailsMap.put("username", username); + connectionDetailsMap.put("password", password); + connectionDetailsMap.put("hostPort", host + ":" + port); + connectionDetailsMap.put("scheme", scheme); + connectionDetailsMap.put("connectionOptions", queryMap); + + } catch (URISyntaxException e) { + e.printStackTrace(); + } + return connectionDetailsMap; + } + + public static void migrateMongoDBConnStr(Handle handle, String updateSqlQuery) { + handle + .createQuery(GET_MONGO_DB_SERVICES) + .mapToMap() + .forEach( + row -> { + DatabaseService mongoService = + JsonUtils.readValue(row.get("json").toString(), DatabaseService.class); + String id = row.get("id").toString(); + Map mongoDBConnection = (LinkedHashMap) mongoService.getConnection().getConfig(); + Map connDetails = (LinkedHashMap) mongoDBConnection.get("connectionDetails"); + + Map finalConnectionDetails; + if (connDetails.get("connectionURI") != null) { + String connectionURI = connDetails.get("connectionURI").toString(); + finalConnectionDetails = extractConnectionURIDetails(connectionURI); + } else { + finalConnectionDetails = connDetails; + } + mongoDBConnection.putAll(finalConnectionDetails); + mongoDBConnection.remove("connectionDetails"); + String json = JsonUtils.pojoToJson(mongoService); + + handle.createUpdate(updateSqlQuery).bind("json", json).bind("id", id).execute(); + }); + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDB/mongoDBValues.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDB/mongoDBValues.json deleted file mode 100644 index 35a0573a6197..000000000000 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDB/mongoDBValues.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/services/connections/database/datalake/azureConfig.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Mongo Connection Values", - "description": "Azure Datalake Storage will ingest files in container", - "type": "object", - "javaType": "org.openmetadata.schema.services.connections.database.mongo.MongoConnectionValues", - "definitions": { - "mongoDBScheme": { - "description": "Mongo connection scheme options.", - "type": "string", - "enum": [ - "mongodb", - "mongodb+srv" - ], - "default": "mongodb" - } - }, - "properties": { - "scheme": { - "title": "Connection Scheme", - "description": "Mongo connection scheme options.", - "$ref": "#/definitions/mongoDBScheme", - "default": "mongodb" - }, - "username": { - "title": "Username", - "description": "Username to connect to MongoDB. This user should have privileges to read all the metadata in MongoDB.", - "type": "string" - }, - "password": { - "title": "Password", - "description": "Password to connect to MongoDB.", - "type": "string", - "format": "password" - }, - "hostPort": { - "title": "Host and Port", - "description": "Host and port of the MongoDB service.", - "type": "string" - }, - "connectionOptions": { - "title": "Connection Options", - "$ref": "../../connectionBasicType.json#/definitions/connectionOptions" - } - }, - "additionalProperties": false, - "required": ["hostPort"] -} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json index f954a6f70a27..8fc6a817e2ef 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json @@ -14,17 +14,14 @@ ], "default": "MongoDB" }, - "MongoConnectionString": { - "type": "object", - "javaType": "org.openmetadata.schema.services.connections.database.mongo.MongoConnectionString", - "title": "Mongo Connection String", - "properties": { - "connectionURI": { - "title": "Connection URI", - "description": "Connection URI to connect to your MongoDB cluster", - "type": "string" - } - } + "mongoDBScheme": { + "description": "Mongo connection scheme options.", + "type": "string", + "enum": [ + "mongodb", + "mongodb+srv" + ], + "default": "mongodb" } }, @@ -35,17 +32,31 @@ "$ref": "#/definitions/mongoDBType", "default": "MongoDB" }, - "connectionDetails": { - "title": "MongoDB Connection Details", - "description": "MongoDB Connection Details.", - "oneOf": [ - { - "$ref": "mongoDB/mongoDBValues.json" - }, - { - "$ref": "#/definitions/MongoConnectionString" - } - ] + "scheme": { + "title": "Connection Scheme", + "description": "Mongo connection scheme options.", + "$ref": "#/definitions/mongoDBScheme", + "default": "mongodb" + }, + "username": { + "title": "Username", + "description": "Username to connect to MongoDB. This user should have privileges to read all the metadata in MongoDB.", + "type": "string" + }, + "password": { + "title": "Password", + "description": "Password to connect to MongoDB.", + "type": "string", + "format": "password" + }, + "hostPort": { + "title": "Host and Port", + "description": "Host and port of the MongoDB service.", + "type": "string" + }, + "connectionOptions": { + "title": "Connection Options", + "$ref": "../connectionBasicType.json#/definitions/connectionOptions" }, "databaseName": { "title": "Database Name", @@ -57,5 +68,6 @@ "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" } }, + "required": ["hostPort"], "additionalProperties": false } \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/MongoDB.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/MongoDB.md index e2dc4e41fd82..ba93db08c172 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/MongoDB.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/MongoDB.md @@ -8,18 +8,6 @@ You can find further information on the Hive connector in the [here](https://doc ## Connection Details -$$section -### MongoDB Connection Details $(id="connectionDetails") - -Choose between MongoDB Connection String and MongoDB Connection Values to authenticate with your mongodb cluster. -$$ - -$$section -### Connection URI $(id="connectionURI") - -MongoDB connection string is a concise string of parameters used to establish a connection between an OpenMetadata and a MongoDB database. For ex. `mongodb://username:password@mongodb0.example.com:27017` -$$ - $$section ### Username $(id="username") Username to connect to Mongodb. This user must have access to perform `find` operation on collection and `listCollection` operations on database available in MongoDB. From b11c5c2d4b6960b80b2093aba0ad412e771787ed Mon Sep 17 00:00:00 2001 From: ulixius9 Date: Fri, 5 Jan 2024 09:12:38 +0530 Subject: [PATCH 2/2] pyformat & test fixes --- ingestion/src/metadata/examples/workflows/mongodb.yaml | 6 ++++-- ingestion/tests/unit/topology/database/test_mongodb.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ingestion/src/metadata/examples/workflows/mongodb.yaml b/ingestion/src/metadata/examples/workflows/mongodb.yaml index 138f281bacdd..e69f33ee4e6c 100644 --- a/ingestion/src/metadata/examples/workflows/mongodb.yaml +++ b/ingestion/src/metadata/examples/workflows/mongodb.yaml @@ -5,8 +5,10 @@ source: config: type: MongoDB databaseName: custom_database_name - connectionDetails: - connectionURI: mongodb+srv://user:pass@localhost:27017 + scheme: mongodb+srv + username: username + password: password + hostPort: localhost:27017 sourceConfig: config: type: DatabaseMetadata diff --git a/ingestion/tests/unit/topology/database/test_mongodb.py b/ingestion/tests/unit/topology/database/test_mongodb.py index 91dca40e121c..338054d79545 100644 --- a/ingestion/tests/unit/topology/database/test_mongodb.py +++ b/ingestion/tests/unit/topology/database/test_mongodb.py @@ -49,7 +49,7 @@ "type": "MongoDB", "username": "ulixius", "password": "dummy_password", - "hostPort": "localhost:27017" + "hostPort": "localhost:27017", }, }, "sourceConfig": {