Merge pull request #15 from plone/datakurre-fix-decode

Fix marshaler decode to decode raw value as unicode (IText)

docs/HISTORY.rst

@@ -4,6 +4,20 @@ Changelog
 1.2.5 (unreleased)
 ------------------
 
+- Fix marshaler decode to always decode raw value into unicode
+  [datakurre]
+
+- Remove utils.getSiteEncoding, which was deprecated and not used anywhere.
+  [thet]
+
+- For Plone 5, support getting markup control panel settings from the registry,
+  while still supporting normal portal_properties access for Plone < 5.
+  [thet]
+
+- Resolved an interesting circular import case, which wasnt effective because
+  of sort order of imports
+  [thet]
+
 - For Plone 5, support getting markup control panel settings from the registry,
   while still supporting normal portal_properties access for Plone < 5.
   [thet]

plone/app/textfield/marshaler.py

@@ -32,8 +32,12 @@ def decode(
                 charset='utf-8',
                 contentType=None,
                 primary=False):
+            try:
+                unicode_value = value.decode(charset)
+            except UnicodeEncodeError:
+                unicode_value = value  # was already unicode
             return RichTextValue(
-                raw=value,
+                raw=unicode_value,
                 mimeType=contentType or self.field.default_mime_type,
                 outputMimeType=self.field.output_mime_type,
                 encoding=charset

plone/app/textfield/marshaler.rst

@@ -53,7 +53,7 @@ We can now look up and test the marshaler:
     'Some \xc3\x98 plain text'
     >>> decoded = marshaler.decode('Some \xc3\x98 plain text', charset='utf-8', contentType='text/plain')
     >>> decoded.raw
-    'Some \xc3\x98 plain text'
+    u'Some \xd8 plain text'
     >>> decoded.mimeType
     'text/plain'
     >>> decoded.outputMimeType
@@ -72,7 +72,7 @@ default type is used.
 
     >>> decoded = marshaler.decode('Some \xc3\x98 plain text')
     >>> decoded.raw
-    'Some \xc3\x98 plain text'
+    u'Some \xd8 plain text'
     >>> decoded.mimeType
     'text/plain'
     >>> decoded.outputMimeType
@@ -108,7 +108,7 @@ Let's now use this message to construct a new object.
     >>> from plone.rfc822 import initializeObjectFromSchema
     >>> initializeObjectFromSchema(newContent, ITestContent, inputMessage)
     >>> newContent._text.raw
-    'Some \xc3\x98 plain text'
+    u'Some \xd8 plain text'
     >>> newContent._text.mimeType
     'text/plain'
     >>> newContent._text.outputMimeType