From 2739fd448af119ed6e5a9acf84bb78836899fd04 Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN" Date: Tue, 14 Jan 2025 10:45:56 +0100 Subject: [PATCH 1/7] CONTRIBUTORS: fix email syntax ... and fix my own identity while at it. Fixes: #258 Signed-off-by: Yann E. MORIN --- CONTRIBUTORS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 4198abc3..c0907007 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -3,7 +3,7 @@ Authors * Frédéric GARDES * Nicolas BUFFON -* Yann MORIN -* Mathieu LEFEBVRE +* Mathieu LEFEBVRE +* François SUC +* Pierre-Yves LAPERSONNE From 57779e276e5bb6edadac58e51d91f6c72d645401 Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN" Date: Fri, 17 Jan 2025 09:27:43 +0100 Subject: [PATCH 2/7] python/iot3: bump dependency to its-quadkeys Signed-off-by: Yann E. MORIN --- python/iot3/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/iot3/pyproject.toml b/python/iot3/pyproject.toml index 6577d99d..15ecc6be 100644 --- a/python/iot3/pyproject.toml +++ b/python/iot3/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ dependencies = [ "paho-mqtt>=2.1.0", "requests>=2.31.0", - "its-quadkeys @ git+https://github.com/Orange-OpenSource/its-client@2083d20a59c5191a1258ece823c0741fce672443#subdirectory=python/its-quadkeys" + "its-quadkeys @ git+https://github.com/Orange-OpenSource/its-client@4ef72eaabacc7504640becb86974a2cfbf9846b3#subdirectory=python/its-quadkeys" ] [project.urls] From 5f33fdfd425c07a4ce4f29ae16dd8c94462b5010 Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN" Date: Tue, 14 Jan 2025 10:28:29 +0100 Subject: [PATCH 3/7] python/iot3/core: fix typo in docstring Signed-off-by: Yann E. MORIN --- python/iot3/src/iot3/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/iot3/src/iot3/core/__init__.py b/python/iot3/src/iot3/core/__init__.py index 340b5778..1a047374 100644 --- a/python/iot3/src/iot3/core/__init__.py +++ b/python/iot3/src/iot3/core/__init__.py @@ -276,7 +276,7 @@ def is_ready() -> bool: def wait_for_ready(): - """Wait until the IoT2 Core SDK is ready. + """Wait until the IoT3 Core SDK is ready. Beware that this may take an indeterminate amount of time; in case the MQTT client can't connect at all, wait_for_ready() From 98eb225a888d6619acf7e762c940c963c062cf34 Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN" Date: Tue, 14 Jan 2025 08:48:23 +0100 Subject: [PATCH 4/7] python/iot3/core: allow creating an MQTT client with explicitly no OTLP Currently, we can create an MQTT client either by passing an explicit OTLP span manager, whether real or fake, or by not passing anything, in which case no OTLP traces would be sent. However, for some callers, it might be simpler to explicitly pass 'None' to not send OTLP traces, rather than pass a fake span manager. Change the MQTT signature to accept this new use-case, in a backward compatible way. Signed-off-by: Yann E. MORIN --- python/iot3/src/iot3/core/mqtt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/iot3/src/iot3/core/mqtt.py b/python/iot3/src/iot3/core/mqtt.py index 4f266bbc..11418ef7 100644 --- a/python/iot3/src/iot3/core/mqtt.py +++ b/python/iot3/src/iot3/core/mqtt.py @@ -32,7 +32,7 @@ def __init__( password: Optional[str] = None, msg_cb: Optional[MsgCallbackType] = None, msg_cb_data: Any = None, - span_ctxmgr_cb: Optional[SpanCallableType] = otel.Otel.noexport_span, + span_ctxmgr_cb: Optional[SpanCallableType] = None, ): """ Create an MQTT client @@ -121,7 +121,7 @@ def my_cb( if tls is None: tls = port != 1883 - self.span_ctxmgr_cb = span_ctxmgr_cb + self.span_ctxmgr_cb = span_ctxmgr_cb or otel.Otel.noexport_span self.client = paho.mqtt.client.Client( callback_api_version=paho.mqtt.enums.CallbackAPIVersion.VERSION2, From 03146d17d918df5044e731e864474e26a5609d08 Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN" Date: Tue, 14 Jan 2025 09:20:15 +0100 Subject: [PATCH 5/7] python/iot3/core: allow using no OTLP in simple API Currently, using OTLP is mandatory in the simple API. However, it is perfectly legit for a user to only be interested in sending MQTT messages. Similarly, it is possible that a bootstrap reply does not contain any OTLP setup when the IoT3 instance does not offer an OTLP collector Signed-off-by: Yann E. MORIN --- python/iot3/src/iot3/core/__init__.py | 90 ++++++++++++++------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/python/iot3/src/iot3/core/__init__.py b/python/iot3/src/iot3/core/__init__.py index 1a047374..5293033e 100644 --- a/python/iot3/src/iot3/core/__init__.py +++ b/python/iot3/src/iot3/core/__init__.py @@ -156,24 +156,21 @@ def bootstrap( # can be prefixed wth "internal-"... Sigh... :-( otlp_proto = "internal-" + otlp_proto if otlp_proto in bootstrap["protocols"]: + config["otel"] = { + "endpoint": bootstrap["protocols"][otlp_proto], + # In practice, there will *always* be credentials provided in + # the bootstrap response, so we'll always have authentication + # and we know the backend only implements BasicAuth. Prove me + # wrong! ;-) + "auth": _otel.Auth.BASIC, + "username": bootstrap["psk_run_login"], + "password": bootstrap["psk_run_password"], + "service_name": service_name, + "batch_period": 5, + "max_backlog": 100, + "compression": "gzip", + } break - else: - raise RuntimeError("No known OTLP protocol available") - - config["otel"] = { - "endpoint": bootstrap["protocols"][otlp_proto], - # In practice, there will *always* be credentials provided in - # the bootstrap response, so we'll always have authentication - # and we know the backend only implements BasicAuth. Prove me - # wrong! ;-) - "auth": _otel.Auth.BASIC, - "username": bootstrap["psk_run_login"], - "password": bootstrap["psk_run_password"], - "service_name": service_name, - "batch_period": 5, - "max_backlog": 100, - "compression": "gzip", - } return config @@ -201,33 +198,39 @@ def start( if _core is not None: raise RuntimeError("IoT3 Core SDK already intialised.") + _core = dict() o_kwargs = dict() - for auth in _otel.Auth: - if config["otel"]["auth"] == auth.value: - o_kwargs["auth"] = auth - if auth != otel.Auth.NONE: - o_kwargs["username"] = config["otel"]["username"] - o_kwargs["password"] = config["otel"]["password"] - break - else: - raise ValueError(f"unknown authentication {config['otel']['auth']}") + if "otel" in config: + for auth in _otel.Auth: + if config["otel"]["auth"] == auth.value: + o_kwargs["auth"] = auth + if auth != otel.Auth.NONE: + o_kwargs["username"] = config["otel"]["username"] + o_kwargs["password"] = config["otel"]["password"] + break + else: + raise ValueError(f"unknown authentication {config['otel']['auth']}") - for comp in _otel.Compression: - if config["otel"]["compression"] == comp.value: - o_kwargs["compression"] = comp - break + for comp in _otel.Compression: + if config["otel"]["compression"] == comp.value: + o_kwargs["compression"] = comp + break + else: + raise ValueError(f"unknown compression {config['otel']['compression']}") + + o = _otel.Otel( + service_name=config["otel"]["service_name"], + endpoint=config["otel"]["endpoint"], + batch_period=config["otel"]["batch_period"], + max_backlog=config["otel"]["max_backlog"], + **o_kwargs, + ) + o.start() + _core["otel"] = o + span_ctxmgr = o.span else: - raise ValueError(f"unknown compression {config['otel']['compression']}") - - o = _otel.Otel( - service_name=config["otel"]["service_name"], - endpoint=config["otel"]["endpoint"], - batch_period=config["otel"]["batch_period"], - max_backlog=config["otel"]["max_backlog"], - **o_kwargs, - ) - o.start() + span_ctxmgr = None # Wrap the callback to avoid leaking telemetry into the simple API def _msg_cb(*, data, topic, payload, **_kwargs): @@ -241,11 +244,11 @@ def _msg_cb(*, data, topic, payload, **_kwargs): password=config["mqtt"]["password"], msg_cb=_msg_cb if message_callback else None, msg_cb_data=callback_data, - span_ctxmgr_cb=o.span, + span_ctxmgr_cb=span_ctxmgr, ) m.start() - _core = dict([("otel", o), ("mqtt", m)]) + _core["mqtt"] = m def stop(): @@ -256,7 +259,8 @@ def stop(): raise RuntimeError("IoT3 Core SDK not initialised.") _core["mqtt"].stop() - _core["otel"].stop() + if "otel" in _core: + _core["otel"].stop() _core = None From 876154fb5e3eb77372565d4a6e75ddc3d22fc28b Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN" Date: Tue, 14 Jan 2025 09:48:48 +0100 Subject: [PATCH 6/7] python/iot3/core: simplify OTLP config in simple API We currently use a loop to iterate over all the possible compression and authentication schemes, and compare their string representation against the valude from the configuration, which is a string. However, python's enum.strEnum() constructor does accept a string as a parameter, where that string is the str() representation of one of the enum values, which incidentally also plays the role of validating the value from the configuration. Use that, rather than our canned, ugly for-loop. Signed-off-by: Yann E. MORIN --- python/iot3/src/iot3/core/__init__.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/python/iot3/src/iot3/core/__init__.py b/python/iot3/src/iot3/core/__init__.py index 5293033e..115a678f 100644 --- a/python/iot3/src/iot3/core/__init__.py +++ b/python/iot3/src/iot3/core/__init__.py @@ -202,22 +202,12 @@ def start( o_kwargs = dict() if "otel" in config: - for auth in _otel.Auth: - if config["otel"]["auth"] == auth.value: - o_kwargs["auth"] = auth - if auth != otel.Auth.NONE: - o_kwargs["username"] = config["otel"]["username"] - o_kwargs["password"] = config["otel"]["password"] - break - else: - raise ValueError(f"unknown authentication {config['otel']['auth']}") + o_kwargs["auth"] = _otel.Auth(config["otel"]["auth"]) + if o_kwargs["auth"] != _otel.Auth.NONE: + o_kwargs["username"] = config["otel"]["username"] + o_kwargs["password"] = config["otel"]["password"] - for comp in _otel.Compression: - if config["otel"]["compression"] == comp.value: - o_kwargs["compression"] = comp - break - else: - raise ValueError(f"unknown compression {config['otel']['compression']}") + o_kwargs["compression"] = _otel.Compression(config["otel"]["compression"]) o = _otel.Otel( service_name=config["otel"]["service_name"], From 91b9356bd1f1363958a619061ceb98a7a8e67313 Mon Sep 17 00:00:00 2001 From: "Yann E. MORIN" Date: Tue, 14 Jan 2025 11:09:00 +0100 Subject: [PATCH 7/7] python/iot3/core: add test of simple API without OTLP collector Signed-off-by: Yann E. MORIN --- python/iot3/tests/test-iot3-core | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/python/iot3/tests/test-iot3-core b/python/iot3/tests/test-iot3-core index 968fbb57..3417beb0 100755 --- a/python/iot3/tests/test-iot3-core +++ b/python/iot3/tests/test-iot3-core @@ -6,9 +6,11 @@ import time def recv(data, topic, payload): - print(f"{topic[:16]}: {payload[:16]}") + print(f"{topic}: {payload[:16]}") +print("IoT3 Core SDK with MQTT and OTLP") + config = iot3.core.sample_config config["mqtt"] = { "host": "test.mosquitto.org", @@ -34,3 +36,25 @@ iot3.core.publish(f"{topic}/passed", "passed") time.sleep(1) iot3.core.stop() + + +print("IoT3 Core SDK with MQTT and no OTLP") + +del config["otel"] + +topic = "test/" + random.randbytes(16).hex() + "/iot3/no-otlp" + +iot3.core.start( + config=config, + message_callback=recv, +) + +iot3.core.subscribe(f"{topic}/+") + +iot3.core.publish(f"{topic}/dropped", "dropped") +time.sleep(1) + +iot3.core.publish(f"{topic}/passed", "passed") +time.sleep(1) + +iot3.core.stop()