[ntfy] Fix submitting newline characters in notification message text

CHANGES.rst

@@ -12,6 +12,7 @@ in progress
 - Documentation: Clarify data type of "target address descriptor"
 - [Pushover]: Switch to using ``base64.b64decode()`` to support decoding
   a string. Thanks, @sumnerboy12.
+- [ntfy] Fix submitting newline characters in notification message text
 
 
 2023-05-15 0.34.1

docs/notifier-catalog.md

@@ -1992,9 +1992,21 @@ targets  = {
 
 :::{note}
 [ntfy publishing options] outlines different ways to marshal data to the ntfy
-HTTP API. mqttwarn is using the HTTP PUT method, where the HTTP body is used
-for the attachment file, and HTTP headers are used for all other ntfy option 
-fields, encoded with [RFC 2047] MIME [quoted-printable encoding].
+HTTP API. mqttwarn utilizes two variants to submit the notification to ntfy,
+using both the HTTP PUT and POST methods, and encoding ntfy option fields into
+HTTP headers with [RFC 2047] MIME [quoted-printable encoding].
+
+- Per default, send the message as HTTP body, enabling line breaks.
+- When submitting a local attachment without a text message, encode the
+  attachment data into the HTTP body, and all other fields into HTTP headers.
+- When it is a notification with both a local attachment, and a text message,
+  also encode the attachment data into the HTTP body, but replace all newline
+  characters `\n` of the text message, because they can not be encoded into
+  HTTP headers.
+
+Effectively, this means you can not submit notification message texts including
+newline characters and local attachments at the same time. When adding a local
+attachment, all newline characters will implicitly be replaced by space characters.
 :::
 
 {#ntfy-remote-attachments}

examples/frigate/test_frigate.py

@@ -166,11 +166,11 @@ def test_frigate_with_notification(mosquitto, ntfy_service, caplog, capmqtt, jso
     assert "Section [frigate/events] matches message on frigate/events, processing it" in caplog.messages
     assert "Message on frigate/events going to ntfy:test" in caplog.messages
     assert "Invoking service plugin for `ntfy'" in caplog.messages
+    assert "Body:    b'goat was in barn before'" in caplog.messages
     assert (
         "Headers: {"
         "'Click': 'https://httpbin.org/anything?camera=cam-testdrive&label=goat&zone=lawn', "
-        "'Title': '=?utf-8?q?goat_entered_lawn_at_2023-04-06_14=3A31=3A46=2E638857+00=3A00?=', "
-        "'Message': '=?utf-8?q?goat_was_in_barn_before?='}" in caplog.messages
+        "'Title': '=?utf-8?q?goat_entered_lawn_at_2023-04-06_14=3A31=3A46=2E638857+00=3A00?='}" in caplog.messages
     )
 
     # assert "Sent ntfy notification to 'http://localhost:5555'." in caplog.messages

mqttwarn/services/ntfy.py

@@ -4,6 +4,7 @@
 
 import dataclasses
 import logging
+import re
 from collections import OrderedDict
 import typing as t
 from email.header import Header
@@ -74,7 +75,7 @@ class NtfyRequest:
     attachment_path: t.Optional[str]
     attachment_data: t.Optional[t.Union[bytes, t.IO]]
 
-    def to_http_headers(self) -> t.Dict[str, str]:
+    def to_http_headers(self, no_message: t.Optional[bool] = False) -> t.Dict[str, str]:
         """
         Provide a variant for `fields` to be submitted as HTTP headers to the ntfy API.
 
@@ -84,30 +85,66 @@ def to_http_headers(self) -> t.Dict[str, str]:
         In this spirit, the header transport does not permit any fancy UTF-8 characters
         within any field, so they will be replaced with placeholder characters `?`.
         """
-        return dict_with_titles(encode_ntfy_fields(self.fields))
+        data = dict_with_titles(encode_ntfy_fields(self.fields))
+        if no_message and "Message" in data:
+            del data["Message"]
+        return data
 
 
 def plugin(srv: Service, item: ProcessorItem) -> bool:
     """
     mqttwarn service plugin for ntfy.
+
+    Regarding newline support, this procedure implements the following suggestion by @codebude from [1]:
+
+    - Per default, send the message as HTTP body, enabling line breaks.
+    - When submitting a local attachment without a text message, encode the
+      attachment data into the HTTP body, and all other fields into HTTP headers.
+    - When it is a notification with both a local attachment, and a text message,
+      also encode the attachment data into the HTTP body, but replace all newline
+      characters `\n` of the text message, because they can not be encoded into
+      HTTP headers.
+
+    [1] https://github.com/mqtt-tools/mqttwarn/issues/677#issuecomment-1575060446
     """
 
     srv.logging.debug("*** MODULE=%s: service=%s, target=%s", __file__, item.service, item.target)
 
     # Decode inbound mqttwarn job item into `NtfyRequest`.
     ntfy_request = decode_jobitem(item)
 
-    # Convert field dictionary to HTTP header dictionary.
-    headers = ntfy_request.to_http_headers()
+    # Submit request to ntfy HTTP API.
+    srv.logging.info("Sending notification to ntfy. target=%s, options=%s", item.target, ntfy_request.options)
+    try:
+        if ntfy_request.attachment_data is not None:
+            # HTTP PUT: Use body for attachment, convert field dictionary to HTTP header dictionary.
+            headers = ntfy_request.to_http_headers()
+            body = ntfy_request.attachment_data
+            response = http.put(
+                ntfy_request.url,
+                data=body,
+                headers=headers,
+            )
+            srv.logging.debug(f"Headers: {dict(headers)}")
+        else:
+            # HTTP POST: Use body for message, other fields via HTTP headers.
+            headers = ntfy_request.to_http_headers(no_message=True)
+            body = to_string(ntfy_request.fields["message"]).encode("utf-8")
+            response = http.post(
+                ntfy_request.url,
+                data=body,
+                headers=headers,
+            )
+    except Exception:
+        srv.logging.exception("Request to ntfy API failed")
+        return False
+    srv.logging.debug(f"Body:    {body!r}")
     srv.logging.debug(f"Headers: {dict(headers)}")
 
-    # Submit request to ntfy HTTP API.
     try:
-        srv.logging.info("Sending notification to ntfy. target=%s, options=%s", item.target, ntfy_request.options)
-        response = http.put(ntfy_request.url, data=ntfy_request.attachment_data, headers=headers)
         response.raise_for_status()
     except Exception:
-        srv.logging.exception("Request to ntfy API failed")
+        srv.logging.exception(f"Error response from ntfy API:\n{response.text}")
         return False
 
     # Report about ntfy response.
@@ -217,13 +254,21 @@ def obtain_ntfy_fields(item: ProcessorItem) -> DataDict:
     return fields
 
 
+def to_string(value: t.Union[str, bytes]) -> str:
+    """
+    Cast from string or bytes to string.
+    """
+    if isinstance(value, bytes):
+        value = value.decode()
+    return value
+
+
 def ascii_clean(data: t.Union[str, bytes]) -> str:
     """
     Return ASCII-clean variant of input string.
     https://stackoverflow.com/a/18430817
     """
-    if isinstance(data, bytes):
-        data = data.decode()
+    data = to_string(data)
     if isinstance(data, str):
         return data.encode("ascii", errors="replace").decode()
     else:
@@ -284,9 +329,12 @@ def encode_ntfy_fields(data: DataDict) -> t.Dict[str, str]:
     - https://docs.python.org/3/library/email.header.html
     """
 
+    rm_newlines = re.compile(r"\r?\n")
+
     outdata = OrderedDict()
     for key, value in data.items():
         key = ascii_clean(key).strip()
+        value = re.sub(rm_newlines, " ", to_string(value))
         if key in NTFY_RFC2047_FIELDS:
             value = encode_rfc2047(value)
         else:

tests/conftest.py

@@ -1,7 +1,12 @@
+# -*- coding: utf-8 -*-
+# (c) 2018-2023 The mqttwarn developers
 import importlib
+import os
 import pathlib
 import shutil
 import sys
+import typing as t
+from tempfile import NamedTemporaryFile
 
 import pytest
 
@@ -76,3 +81,15 @@ def tmp_ini(tmp_path) -> pathlib.Path:
     """
     filepath = tmp_path.joinpath("testdrive.ini")
     return filepath
+
+
+@pytest.fixture
+def attachment_dummy() -> t.Generator[t.IO[bytes], None, None]:
+    """
+    Provide a temporary file to the test cases to be used as an attachment with defined content.
+    """
+    tmp = NamedTemporaryFile(suffix=".txt", delete=False)
+    tmp.write(b"foo")
+    tmp.close()
+    yield tmp
+    os.unlink(tmp.name)

tests/fixtures/ntfy.py

@@ -73,5 +73,6 @@ def ntfy_service():
         yield "localhost", 5555
         return
 
-    yield ntfy_image.run()
+    ntfy_image.run()
+    yield "localhost", 5555
     ntfy_image.stop()