Handle UnicodeDecodeError in logger when console encoding is not utf-8

src/tribler-common/tribler_common/logger/config.yaml

@@ -76,13 +76,13 @@ handlers:
         level: DEBUG
         formatter: standard
         filters: [info_filter]
-        stream: ext://sys.stdout
+        stream: ext://tribler_common.logger.streams.stdout_wrapper
 
     error_console:
         class: logging.StreamHandler
         level: ERROR
         formatter: error
-        stream: ext://sys.stderr
+        stream: ext://tribler_common.logger.streams.stderr_wrapper
 
 
 # Root Logger Configuration

src/tribler-common/tribler_common/logger/streams.py

@@ -0,0 +1,28 @@
+import sys
+from typing import TextIO
+
+
+class StreamWrapper:
+    """
+    Used by logger to wrap stderr & stdout streams. Handles UnicodeDecodeError if console encoding is not utf-8.
+    """
+    def __init__(self, stream: TextIO):
+        self.stream = stream
+
+    def flush(self):
+        self.stream.flush()
+
+    def write(self, s: str):
+        try:
+            self.stream.write(s)
+        except UnicodeEncodeError:
+            encoding = self.stream.encoding
+            s2 = s.encode(encoding, errors='backslashreplace').decode(encoding)
+            self.stream.write(s2)
+
+    def close(self):
+        self.stream.close()
+
+
+stdout_wrapper = StreamWrapper(sys.stdout)  # specified in logger.yaml for `console` handler
+stderr_wrapper = StreamWrapper(sys.stderr)  # specified in logger.yaml for `error_console` handler

src/tribler-common/tribler_common/tests/test_logger.py

@@ -0,0 +1,60 @@
+from io import BytesIO, TextIOWrapper
+from unittest.mock import MagicMock, call
+
+from tribler_common.logger.streams import StreamWrapper
+
+
+def test_stream_wrapper_write_ascii():
+    stream = MagicMock()
+    wrapper = StreamWrapper(stream)
+    wrapper.write('hello')
+    stream.write.assert_called_once_with('hello')
+
+    byte_stream = BytesIO()
+    stream = TextIOWrapper(byte_stream, encoding='ascii')
+    wrapper = StreamWrapper(stream)
+    wrapper.write("hello")
+    wrapper.flush()
+    assert byte_stream.getvalue() == b"hello"
+
+def test_stream_wrapper_write_non_ascii_without_exception():
+    stream = MagicMock()
+    wrapper = StreamWrapper(stream)
+    wrapper.write('hello привет')
+    stream.write.assert_called_once_with('hello привет')
+
+    byte_stream = BytesIO()
+    stream = TextIOWrapper(byte_stream, encoding='cp1251')
+    wrapper = StreamWrapper(stream)
+    wrapper.write("hello привет")
+    wrapper.flush()
+    assert byte_stream.getvalue() == 'hello привет'.encode('cp1251')
+
+
+def test_stream_wrapper_write_non_ascii_with_exception():
+    stream = MagicMock(encoding='ascii')
+    stream.write.side_effect = [UnicodeEncodeError('ascii','zzz', 0, 1, 'error message'), None]
+    wrapper = StreamWrapper(stream)
+    wrapper.write('hello привет')
+    stream.write.assert_has_calls([call('hello привет'), call('hello \\u043f\\u0440\\u0438\\u0432\\u0435\\u0442')])
+
+    byte_stream = BytesIO()
+    stream = TextIOWrapper(byte_stream, encoding='ascii')
+    wrapper = StreamWrapper(stream)
+    wrapper.write("hello привет")
+    wrapper.flush()
+    assert byte_stream.getvalue() == b'hello \\u043f\\u0440\\u0438\\u0432\\u0435\\u0442'
+
+
+def test_stream_flush_and_close():
+    stream = MagicMock()
+    wrapper = StreamWrapper(stream)
+    wrapper.write('hello')
+
+    stream.flush.assert_not_called()
+    wrapper.flush()
+    stream.flush.assert_called_once()
+
+    stream.close.assert_not_called()
+    wrapper.close()
+    stream.close.assert_called_once()

src/tribler-core/tribler_core/logger.yaml

@@ -53,13 +53,13 @@ handlers:
         level: INFO
         formatter: standard
         filters: [info_filter]
-        stream: ext://sys.stdout
+        stream: ext://tribler_common.logger.streams.stdout_wrapper
 
     error_console:
         class: logging.StreamHandler
         level: ERROR
         formatter: error
-        stream: ext://sys.stderr
+        stream: ext://tribler_common.logger.streams.stderr_wrapper
 
 # Root Logger Configuration
 root:

src/tribler-gui/tribler_gui/logger.yaml

@@ -53,13 +53,13 @@ handlers:
         level: DEBUG
         formatter: standard
         filters: [info_filter]
-        stream: ext://sys.stdout
+        stream: ext://tribler_common.logger.streams.stdout_wrapper
 
     error_console:
         class: logging.StreamHandler
         level: ERROR
         formatter: error
-        stream: ext://sys.stderr
+        stream: ext://tribler_common.logger.streams.stderr_wrapper
 
 # Root Logger Configuration
 root: