From d7d86d62d361168bc0e285a2cad77fb41737e33f Mon Sep 17 00:00:00 2001 From: sushi30 Date: Thu, 21 Nov 2024 15:31:11 +0100 Subject: [PATCH 1/2] feat(mongo): added ssl support Added SSL support for MongoDB using the SSL manager. Attached a video demo. - [Example repository for setting up mongodb with SSL](https://github.com/sushi30/mongodb-docker-ssl-example) - [MongoDB TLS documentation](https://www.mongodb.com/docs/manual/tutorial/configure-ssl/) --- .../metadata/examples/workflows/mongodb.yaml | 4 ++ .../source/database/common_nosql_source.py | 7 ++++ ingestion/src/metadata/utils/ssl_manager.py | 40 ++++++++++++++++++- .../database/mongoDBConnection.json | 6 +++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/ingestion/src/metadata/examples/workflows/mongodb.yaml b/ingestion/src/metadata/examples/workflows/mongodb.yaml index 63c1efc46957..e38c0c49379f 100644 --- a/ingestion/src/metadata/examples/workflows/mongodb.yaml +++ b/ingestion/src/metadata/examples/workflows/mongodb.yaml @@ -9,6 +9,10 @@ source: username: username password: password hostPort: localhost:27017 +# # SSL Configuration +# sslMode": verify-ca +# sslConfig: +# caCertificate": "CA certificate content" sourceConfig: config: type: DatabaseMetadata diff --git a/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py b/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py index 81ae7da59d49..15b33973c9a0 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py @@ -52,6 +52,7 @@ from metadata.utils.datalake.datalake_utils import DataFrameColumnParser from metadata.utils.filters import filter_by_schema, filter_by_table from metadata.utils.logger import ingestion_logger +from metadata.utils.ssl_manager import check_ssl_and_init logger = ingestion_logger() @@ -73,7 +74,13 @@ def __init__(self, config: WorkflowSource, metadata: OpenMetadata): ) self.metadata = metadata self.service_connection = self.config.serviceConnection.root.config + self.ssl_manager = check_ssl_and_init(self.service_connection) + if self.ssl_manager: + self.service_connection = self.ssl_manager.setup_ssl( + self.service_connection + ) self.connection_obj = get_connection(self.service_connection) + self.test_connection() def prepare(self): diff --git a/ingestion/src/metadata/utils/ssl_manager.py b/ingestion/src/metadata/utils/ssl_manager.py index 60cf08f3ce85..1e6d7a0af5ec 100644 --- a/ingestion/src/metadata/utils/ssl_manager.py +++ b/ingestion/src/metadata/utils/ssl_manager.py @@ -21,6 +21,9 @@ from pydantic import SecretStr +from metadata.generated.schema.entity.services.connections.connectionBasicType import ( + ConnectionOptions, +) from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import ( QlikSenseConnection, ) @@ -30,6 +33,9 @@ from metadata.generated.schema.entity.services.connections.database.greenplumConnection import ( GreenplumConnection, ) +from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import ( + MongoDBConnection, +) from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( MysqlConnection, ) @@ -176,6 +182,20 @@ def _(self, connection): "check_hostname": connection.validateHostName, } + @setup_ssl.register(MongoDBConnection) + def _(self, connection: MongoDBConnection): + connection.connectionOptions = ( + connection.connectionOptions or ConnectionOptions(root={}) + ) + connection.connectionOptions.root.update( + { + "tls": "true", + "tlsCertificateKeyFile": self.key_file_path, + "tlsCAFile": self.ca_file_path, + } + ) + return connection + @setup_ssl.register(KafkaConnection) def _(self, connection): connection = cast(KafkaConnection, connection) @@ -188,7 +208,7 @@ def _(self, connection): @singledispatch -def check_ssl_and_init(_) -> None: +def check_ssl_and_init(_) -> Optional[SSLManager]: return None @@ -236,6 +256,24 @@ def _(connection): return None +@check_ssl_and_init.register(MongoDBConnection) +def _(connection): + service_connection = cast(Union[MysqlConnection, DorisConnection], connection) + ssl: Optional[verifySSLConfig.SslConfig] = service_connection.sslConfig + if ssl and ssl.root.sslCertificate: + raise ValueError( + "MongoDB connection does not support SSL certificate. Only CA certificate is supported.\n" + "More information about configuring MongoDB connection can be found at:\n" + "https://www.mongodb.com/docs/manual/tutorial/configure-ssl-clients/#mongodb-shell" + ) + if ssl and (ssl.root.caCertificate or ssl.root.sslKey): + return SSLManager( + ca=ssl.root.caCertificate, + key=ssl.root.sslKey, + ) + return None + + @check_ssl_and_init.register(PostgresConnection) @check_ssl_and_init.register(RedshiftConnection) @check_ssl_and_init.register(GreenplumConnection) diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json index 7b453a16653a..7543140d2745 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json @@ -70,6 +70,12 @@ "supportsProfiler": { "title": "Supports Profiler", "$ref": "../connectionBasicType.json#/definitions/supportsProfiler" + }, + "sslMode": { + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslMode" + }, + "sslConfig": { + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" } }, "required": ["hostPort"], From 1d6a0d0d1bafd7a40436dd10cbef5d6776aa0585 Mon Sep 17 00:00:00 2001 From: sushi30 Date: Thu, 21 Nov 2024 17:34:13 +0100 Subject: [PATCH 2/2] fixed test_doris.py --- ingestion/tests/unit/topology/database/test_doris.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ingestion/tests/unit/topology/database/test_doris.py b/ingestion/tests/unit/topology/database/test_doris.py index 16e856461c70..be8aad50065f 100644 --- a/ingestion/tests/unit/topology/database/test_doris.py +++ b/ingestion/tests/unit/topology/database/test_doris.py @@ -27,6 +27,7 @@ "serviceName": "local_doris1", "serviceConnection": { "config": { + "type": "Doris", "username": "root", "hostPort": "localhost:3308", "password": "test",