Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GEN-2109] feat(mongo): added ssl support #18731

Merged
merged 2 commits into from
Nov 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ingestion/src/metadata/examples/workflows/mongodb.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ source:
username: username
password: password
hostPort: localhost:27017
# # SSL Configuration
# sslMode": verify-ca
# sslConfig:
# caCertificate": "CA certificate content"
sourceConfig:
config:
type: DatabaseMetadata
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
from metadata.utils.datalake.datalake_utils import DataFrameColumnParser
from metadata.utils.filters import filter_by_schema, filter_by_table
from metadata.utils.logger import ingestion_logger
from metadata.utils.ssl_manager import check_ssl_and_init

logger = ingestion_logger()

Expand All @@ -73,7 +74,13 @@ def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
)
self.metadata = metadata
self.service_connection = self.config.serviceConnection.root.config
self.ssl_manager = check_ssl_and_init(self.service_connection)
if self.ssl_manager:
self.service_connection = self.ssl_manager.setup_ssl(
self.service_connection
)
self.connection_obj = get_connection(self.service_connection)

self.test_connection()

def prepare(self):
Expand Down
40 changes: 39 additions & 1 deletion ingestion/src/metadata/utils/ssl_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@

from pydantic import SecretStr

from metadata.generated.schema.entity.services.connections.connectionBasicType import (
ConnectionOptions,
)
from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import (
QlikSenseConnection,
)
Expand All @@ -30,6 +33,9 @@
from metadata.generated.schema.entity.services.connections.database.greenplumConnection import (
GreenplumConnection,
)
from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import (
MongoDBConnection,
)
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
)
Expand Down Expand Up @@ -176,6 +182,20 @@ def _(self, connection):
"check_hostname": connection.validateHostName,
}

@setup_ssl.register(MongoDBConnection)
def _(self, connection: MongoDBConnection):
connection.connectionOptions = (
connection.connectionOptions or ConnectionOptions(root={})
)
connection.connectionOptions.root.update(
{
"tls": "true",
"tlsCertificateKeyFile": self.key_file_path,
"tlsCAFile": self.ca_file_path,
}
)
return connection

@setup_ssl.register(KafkaConnection)
def _(self, connection):
connection = cast(KafkaConnection, connection)
Expand All @@ -188,7 +208,7 @@ def _(self, connection):


@singledispatch
def check_ssl_and_init(_) -> None:
def check_ssl_and_init(_) -> Optional[SSLManager]:
return None


Expand Down Expand Up @@ -236,6 +256,24 @@ def _(connection):
return None


@check_ssl_and_init.register(MongoDBConnection)
def _(connection):
service_connection = cast(Union[MysqlConnection, DorisConnection], connection)
ssl: Optional[verifySSLConfig.SslConfig] = service_connection.sslConfig
if ssl and ssl.root.sslCertificate:
raise ValueError(
"MongoDB connection does not support SSL certificate. Only CA certificate is supported.\n"
"More information about configuring MongoDB connection can be found at:\n"
"https://www.mongodb.com/docs/manual/tutorial/configure-ssl-clients/#mongodb-shell"
)
if ssl and (ssl.root.caCertificate or ssl.root.sslKey):
return SSLManager(
ca=ssl.root.caCertificate,
key=ssl.root.sslKey,
)
return None


@check_ssl_and_init.register(PostgresConnection)
@check_ssl_and_init.register(RedshiftConnection)
@check_ssl_and_init.register(GreenplumConnection)
Expand Down
1 change: 1 addition & 0 deletions ingestion/tests/unit/topology/database/test_doris.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"serviceName": "local_doris1",
"serviceConnection": {
"config": {
"type": "Doris",
"username": "root",
"hostPort": "localhost:3308",
"password": "test",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@
"supportsProfiler": {
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"sslMode": {
"$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslMode"
},
"sslConfig": {
"$ref": "../../../../security/ssl/verifySSLConfig.json#/definitions/sslConfig"
}
},
"required": ["hostPort"],
Expand Down
Loading