From 8c7367a519c5b25358855c04e1821ba6a05f70d7 Mon Sep 17 00:00:00 2001 From: Imri Paran Date: Fri, 22 Nov 2024 17:54:13 +0100 Subject: [PATCH] [GEN-2109] feat(mongo): added ssl support (#18731) * feat(mongo): added ssl support Added SSL support for MongoDB using the SSL manager. Attached a video demo. - [Example repository for setting up mongodb with SSL](https://github.com/sushi30/mongodb-docker-ssl-example) - [MongoDB TLS documentation](https://www.mongodb.com/docs/manual/tutorial/configure-ssl/) * fixed test_doris.py (cherry picked from commit ee7d0430351a0fb6caee50c34c9fa0476fe45c0e) --- .../metadata/examples/workflows/mongodb.yaml | 4 ++ .../source/database/common_nosql_source.py | 7 ++++ ingestion/src/metadata/utils/ssl_manager.py | 40 ++++++++++++++++++- .../unit/topology/database/test_doris.py | 1 + .../database/mongoDBConnection.json | 6 +++ 5 files changed, 57 insertions(+), 1 deletion(-) diff --git a/ingestion/src/metadata/examples/workflows/mongodb.yaml b/ingestion/src/metadata/examples/workflows/mongodb.yaml index 63c1efc46957..e38c0c49379f 100644 --- a/ingestion/src/metadata/examples/workflows/mongodb.yaml +++ b/ingestion/src/metadata/examples/workflows/mongodb.yaml @@ -9,6 +9,10 @@ source: username: username password: password hostPort: localhost:27017 +# # SSL Configuration +# sslMode": verify-ca +# sslConfig: +# caCertificate": "CA certificate content" sourceConfig: config: type: DatabaseMetadata diff --git a/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py b/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py index c0b968ddd978..19c6c4a6a5c5 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_nosql_source.py @@ -54,6 +54,7 @@ from metadata.utils.datalake.datalake_utils import DataFrameColumnParser from metadata.utils.filters import filter_by_schema, filter_by_table from metadata.utils.logger import ingestion_logger +from metadata.utils.ssl_manager import check_ssl_and_init logger = ingestion_logger() @@ -75,7 +76,13 @@ def __init__(self, config: WorkflowSource, metadata: OpenMetadata): ) self.metadata = metadata self.service_connection = self.config.serviceConnection.root.config + self.ssl_manager = check_ssl_and_init(self.service_connection) + if self.ssl_manager: + self.service_connection = self.ssl_manager.setup_ssl( + self.service_connection + ) self.connection_obj = get_connection(self.service_connection) + self.test_connection() def prepare(self): diff --git a/ingestion/src/metadata/utils/ssl_manager.py b/ingestion/src/metadata/utils/ssl_manager.py index 619dc75e5930..b6af510708e6 100644 --- a/ingestion/src/metadata/utils/ssl_manager.py +++ b/ingestion/src/metadata/utils/ssl_manager.py @@ -21,6 +21,9 @@ from pydantic import SecretStr +from metadata.generated.schema.entity.services.connections.connectionBasicType import ( + ConnectionOptions, +) from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import ( QlikSenseConnection, ) @@ -30,6 +33,9 @@ from metadata.generated.schema.entity.services.connections.database.greenplumConnection import ( GreenplumConnection, ) +from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import ( + MongoDBConnection, +) from metadata.generated.schema.entity.services.connections.database.mysqlConnection import ( MysqlConnection, ) @@ -158,6 +164,20 @@ def _(self, connection): "check_hostname": connection.validateHostName, } + @setup_ssl.register(MongoDBConnection) + def _(self, connection: MongoDBConnection): + connection.connectionOptions = ( + connection.connectionOptions or ConnectionOptions(root={}) + ) + connection.connectionOptions.root.update( + { + "tls": "true", + "tlsCertificateKeyFile": self.key_file_path, + "tlsCAFile": self.ca_file_path, + } + ) + return connection + @setup_ssl.register(KafkaConnection) def _(self, connection): connection = cast(KafkaConnection, connection) @@ -170,7 +190,7 @@ def _(self, connection): @singledispatch -def check_ssl_and_init(_) -> None: +def check_ssl_and_init(_) -> Optional[SSLManager]: return None @@ -203,6 +223,24 @@ def _(connection): return None +@check_ssl_and_init.register(MongoDBConnection) +def _(connection): + service_connection = cast(Union[MysqlConnection, DorisConnection], connection) + ssl: Optional[verifySSLConfig.SslConfig] = service_connection.sslConfig + if ssl and ssl.root.sslCertificate: + raise ValueError( + "MongoDB connection does not support SSL certificate. Only CA certificate is supported.\n" + "More information about configuring MongoDB connection can be found at:\n" + "https://www.mongodb.com/docs/manual/tutorial/configure-ssl-clients/#mongodb-shell" + ) + if ssl and (ssl.root.caCertificate or ssl.root.sslKey): + return SSLManager( + ca=ssl.root.caCertificate, + key=ssl.root.sslKey, + ) + return None + + @check_ssl_and_init.register(PostgresConnection) @check_ssl_and_init.register(RedshiftConnection) @check_ssl_and_init.register(GreenplumConnection) diff --git a/ingestion/tests/unit/topology/database/test_doris.py b/ingestion/tests/unit/topology/database/test_doris.py index 16e856461c70..be8aad50065f 100644 --- a/ingestion/tests/unit/topology/database/test_doris.py +++ b/ingestion/tests/unit/topology/database/test_doris.py @@ -27,6 +27,7 @@ "serviceName": "local_doris1", "serviceConnection": { "config": { + "type": "Doris", "username": "root", "hostPort": "localhost:3308", "password": "test", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json index 7b453a16653a..7543140d2745 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mongoDBConnection.json @@ -70,6 +70,12 @@ "supportsProfiler": { "title": "Supports Profiler", "$ref": "../connectionBasicType.json#/definitions/supportsProfiler" + }, + "sslMode": { + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslMode" + }, + "sslConfig": { + "$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig" } }, "required": ["hostPort"],