From 5335f9228d2e2c69d8bc526a194776f941a3e446 Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Thu, 9 May 2024 14:48:08 +0100 Subject: [PATCH 1/5] feat: OBS-435 - diode-sdk-python: use root CAs for grpc secure channel and add configurable tls_verify also add metadata about platform and python version Signed-off-by: Michal Fiedorowicz --- .../netboxlabs/diode/sdk/client.py | 58 +++++++++++++++---- diode-sdk-python/pyproject.toml | 1 + 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/diode-sdk-python/netboxlabs/diode/sdk/client.py b/diode-sdk-python/netboxlabs/diode/sdk/client.py index db489c98..c6a36972 100644 --- a/diode-sdk-python/netboxlabs/diode/sdk/client.py +++ b/diode-sdk-python/netboxlabs/diode/sdk/client.py @@ -4,20 +4,49 @@ import logging import os +import platform import uuid from typing import Iterable, Optional +from urllib.parse import urlparse +import certifi import grpc from netboxlabs.diode.sdk.diode.v1 import ingester_pb2, ingester_pb2_grpc from netboxlabs.diode.sdk.exceptions import DiodeClientError, DiodeConfigError _DIODE_API_KEY_ENVVAR_NAME = "DIODE_API_KEY" +_DIODE_API_TLS_VERIFY_ENVVAR_NAME = "DIODE_API_TLS_VERIFY" _DIODE_SDK_LOG_LEVEL_ENVVAR_NAME = "DIODE_SDK_LOG_LEVEL" _DEFAULT_STREAM = "latest" _LOGGER = logging.getLogger(__name__) +def _certs() -> bytes: + with open(certifi.where(), "rb") as f: + return f.read() + + +def _api_key(api_key: Optional[str] = None) -> str: + if api_key is None: + api_key = os.getenv(_DIODE_API_KEY_ENVVAR_NAME) + if api_key is None: + raise DiodeConfigError( + f"api_key param or {_DIODE_API_KEY_ENVVAR_NAME} environment variable required" + ) + return api_key + + +def _tls_verify(tls_verify: Optional[bool]) -> bool: + if tls_verify is None: + tls_verify_env_var = os.getenv(_DIODE_API_TLS_VERIFY_ENVVAR_NAME, "false") + return tls_verify_env_var.lower() in ["true", "1", "yes"] + if not isinstance(tls_verify, bool): + raise DiodeConfigError("tls_verify must be a boolean") + + return tls_verify + + class DiodeClient: """Diode Client.""" @@ -34,6 +63,7 @@ def __init__( app_name: str, app_version: str, api_key: Optional[str] = None, + tls_verify: bool = None, ): """Initiate a new client.""" log_level = os.getenv(_DIODE_SDK_LOG_LEVEL_ENVVAR_NAME, "INFO").upper() @@ -45,16 +75,24 @@ def __init__( self._app_name = app_name self._app_version = app_version - if api_key is None: - api_key = os.getenv(_DIODE_API_KEY_ENVVAR_NAME) - if api_key is None: - raise DiodeConfigError("API key is required") + api_key = _api_key(api_key) + self._metadata = ( + ("diode-api-key", api_key), + ("platform", platform.platform()), + ("python-version", platform.python_version()), + ) + + if _tls_verify(tls_verify): + self._channel = grpc.secure_channel( + self._target, + grpc.ssl_channel_credentials( + root_certificates=_certs(), + ), + ) + else: + self._channel = grpc.insecure_channel(self._target) - self._auth_metadata = (("diode-api-key", api_key),) - # TODO: add support for secure channel (TLS verify flag and cert) - self._channel = grpc.insecure_channel(target) self._stub = ingester_pb2_grpc.IngesterServiceStub(self._channel) - # TODO: obtain meta data about the environment; Python version, CPU arch, OS @property def name(self) -> str: @@ -103,7 +141,7 @@ def ingest( entities: Iterable[ingester_pb2.Entity], stream: Optional[str] = _DEFAULT_STREAM, ) -> ingester_pb2.IngestResponse: - """Push a message.""" + """Ingest entities.""" try: request = ingester_pb2.IngestRequest( stream=stream, @@ -115,6 +153,6 @@ def ingest( producer_app_version=self.app_version, ) - return self._stub.Ingest(request, metadata=self._auth_metadata) + return self._stub.Ingest(request, metadata=self._metadata) except grpc.RpcError as err: raise DiodeClientError(err) from err diff --git a/diode-sdk-python/pyproject.toml b/diode-sdk-python/pyproject.toml index 501007b8..2e39d8de 100644 --- a/diode-sdk-python/pyproject.toml +++ b/diode-sdk-python/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ # Optional ] dependencies = [ + "certifi==2024.2.2", "grpcio==1.62.1", "grpcio-status==1.62.1", ] From 512a79b3dd33889810f6fd9f3ec754ac8852e697 Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Thu, 9 May 2024 14:57:39 +0100 Subject: [PATCH 2/5] feat: OBS-435 - diode-sdk-python: fix client test Signed-off-by: Michal Fiedorowicz --- diode-sdk-python/tests/test_client.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/diode-sdk-python/tests/test_client.py b/diode-sdk-python/tests/test_client.py index b949548f..a6a8ce12 100644 --- a/diode-sdk-python/tests/test_client.py +++ b/diode-sdk-python/tests/test_client.py @@ -11,7 +11,12 @@ def test_init(): """Ensure we can initiate a client configuration.""" - config = DiodeClient(target="localhost:8081", app_name="my-producer", app_version="0.0.1", api_key="abcde") + config = DiodeClient( + target="localhost:8081", + app_name="my-producer", + app_version="0.0.1", + api_key="abcde", + ) assert config.target == "localhost:8081" assert config.name == "diode-sdk-python" assert config.version == "0.0.1" @@ -22,13 +27,23 @@ def test_init(): def test_config_error(): """Ensure we can raise a config error.""" with pytest.raises(DiodeConfigError) as err: - DiodeClient(target="localhost:8081", app_name="my-producer", app_version="0.0.1") - assert str(err.value) == "API key is required" + DiodeClient( + target="localhost:8081", app_name="my-producer", app_version="0.0.1" + ) + assert ( + str(err.value) + == f"api_key param or DIODE_API_KEY environment variable required" + ) def test_client_error(): """Ensure we can raise a client error.""" with pytest.raises(DiodeClientError) as err: - client = DiodeClient(target="invalid:8081", app_name="my-producer", app_version="0.0.1", api_key="abcde") + client = DiodeClient( + target="invalid:8081", + app_name="my-producer", + app_version="0.0.1", + api_key="abcde", + ) client.ingest(entities=[]) assert err.value.status_code == grpc.StatusCode.UNAVAILABLE From bd14de9997d38f7d6c7d73dd337b8a8a464e8f60 Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Thu, 9 May 2024 19:58:18 +0100 Subject: [PATCH 3/5] feat: OBS-435 - diode-sdk-python: add client call interceptor for prefixing grpc methods with paths provided in the target Signed-off-by: Michal Fiedorowicz --- .../netboxlabs/diode/sdk/client.py | 101 ++++++++++++++++-- 1 file changed, 93 insertions(+), 8 deletions(-) diff --git a/diode-sdk-python/netboxlabs/diode/sdk/client.py b/diode-sdk-python/netboxlabs/diode/sdk/client.py index c6a36972..7b4bf752 100644 --- a/diode-sdk-python/netboxlabs/diode/sdk/client.py +++ b/diode-sdk-python/netboxlabs/diode/sdk/client.py @@ -1,13 +1,12 @@ #!/usr/bin/env python # Copyright 2024 NetBox Labs Inc """NetBox Labs, Diode - SDK - Client.""" - +import collections import logging import os import platform import uuid -from typing import Iterable, Optional -from urllib.parse import urlparse +from typing import Dict, Iterable, Optional import certifi import grpc @@ -47,6 +46,25 @@ def _tls_verify(tls_verify: Optional[bool]) -> bool: return tls_verify +def parse_target(target: str) -> Dict[str, str]: + """Parse target.""" + if target.startswith(("http://", "https://")): + raise ValueError("target should not contain http:// or https://") + + parts = [str(part) for part in target.split("/") if part != ""] + + authority = ":".join([str(part) for part in parts[0].split(":") if part != ""]) + + if ":" not in authority: + authority += ":443" + + path = "" + if len(parts) > 1: + path = "/" + "/".join(parts[1:]) + + return authority, path + + class DiodeClient: """Diode Client.""" @@ -69,9 +87,7 @@ def __init__( log_level = os.getenv(_DIODE_SDK_LOG_LEVEL_ENVVAR_NAME, "INFO").upper() logging.basicConfig(level=log_level) - # TODO: validate target - self._target = target - + self._target, self._path = parse_target(target) self._app_name = app_name self._app_version = app_version @@ -90,9 +106,21 @@ def __init__( ), ) else: - self._channel = grpc.insecure_channel(self._target) + self._channel = grpc.insecure_channel( + target=self._target, + ) + + channel = self._channel - self._stub = ingester_pb2_grpc.IngesterServiceStub(self._channel) + if self._path: + rpc_method_interceptor = DiodeMethodClientInterceptor(subpath=self._path) + + intercept_channel = grpc.intercept_channel( + self._channel, rpc_method_interceptor + ) + channel = intercept_channel + + self._stub = ingester_pb2_grpc.IngesterServiceStub(channel) @property def name(self) -> str: @@ -156,3 +184,60 @@ def ingest( return self._stub.Ingest(request, metadata=self._metadata) except grpc.RpcError as err: raise DiodeClientError(err) from err + + +class _ClientCallDetails( + collections.namedtuple( + "_ClientCallDetails", + ( + "method", + "timeout", + "metadata", + "credentials", + "wait_for_ready", + "compression", + ), + ), + grpc.ClientCallDetails, +): + """Client Call Details.""" + + pass + + +class DiodeMethodClientInterceptor( + grpc.UnaryUnaryClientInterceptor, grpc.StreamUnaryClientInterceptor +): + """Diode Method Client Interceptor.""" + + def __init__(self, subpath): + """Initiate a new interceptor.""" + self._subpath = subpath + + def _intercept_call(self, continuation, client_call_details, request_or_iterator): + """Intercept call.""" + method = client_call_details.method + if client_call_details.method is not None: + method = f"{self._subpath}{client_call_details.method}" + + client_call_details = _ClientCallDetails( + method, + client_call_details.timeout, + client_call_details.metadata, + client_call_details.credentials, + client_call_details.wait_for_ready, + client_call_details.compression, + ) + + response = continuation(client_call_details, request_or_iterator) + return response + + def intercept_unary_unary(self, continuation, client_call_details, request): + """Intercept unary unary.""" + return self._intercept_call(continuation, client_call_details, request) + + def intercept_stream_unary( + self, continuation, client_call_details, request_iterator + ): + """Intercept stream unary.""" + return self._intercept_call(continuation, client_call_details, request_iterator) From f2b9d09d83aa3ab99f7ceea18ffb95d19965bf73 Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Thu, 9 May 2024 20:20:04 +0100 Subject: [PATCH 4/5] feat: OBS-435 - diode-sdk-python: redundant f-string Signed-off-by: Michal Fiedorowicz --- diode-sdk-python/tests/test_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/diode-sdk-python/tests/test_client.py b/diode-sdk-python/tests/test_client.py index a6a8ce12..5cfce8d0 100644 --- a/diode-sdk-python/tests/test_client.py +++ b/diode-sdk-python/tests/test_client.py @@ -31,8 +31,7 @@ def test_config_error(): target="localhost:8081", app_name="my-producer", app_version="0.0.1" ) assert ( - str(err.value) - == f"api_key param or DIODE_API_KEY environment variable required" + str(err.value) == "api_key param or DIODE_API_KEY environment variable required" ) From 55c55fca1b84398792375da8dd4c7acc467a9be8 Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Thu, 9 May 2024 20:28:08 +0100 Subject: [PATCH 5/5] feat: OBS-435 - diode-sdk-python: tidy up + tests Signed-off-by: Michal Fiedorowicz --- diode-sdk-python/netboxlabs/diode/sdk/client.py | 14 +++++++++++++- diode-sdk-python/tests/test_client.py | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/diode-sdk-python/netboxlabs/diode/sdk/client.py b/diode-sdk-python/netboxlabs/diode/sdk/client.py index 7b4bf752..31810cbd 100644 --- a/diode-sdk-python/netboxlabs/diode/sdk/client.py +++ b/diode-sdk-python/netboxlabs/diode/sdk/client.py @@ -98,7 +98,9 @@ def __init__( ("python-version", platform.python_version()), ) - if _tls_verify(tls_verify): + self._tls_verify = _tls_verify(tls_verify) + + if self._tls_verify: self._channel = grpc.secure_channel( self._target, grpc.ssl_channel_credentials( @@ -137,6 +139,16 @@ def target(self) -> str: """Retrieve the target.""" return self._target + @property + def path(self) -> str: + """Retrieve the path.""" + return self._path + + @property + def tls_verify(self) -> str: + """Retrieve the tls_verify.""" + return self._tls_verify + @property def app_name(self) -> str: """Retrieve the app name.""" diff --git a/diode-sdk-python/tests/test_client.py b/diode-sdk-python/tests/test_client.py index 5cfce8d0..43e0dff8 100644 --- a/diode-sdk-python/tests/test_client.py +++ b/diode-sdk-python/tests/test_client.py @@ -22,6 +22,8 @@ def test_init(): assert config.version == "0.0.1" assert config.app_name == "my-producer" assert config.app_version == "0.0.1" + assert config.tls_verify is False + assert config.path == "" def test_config_error():