more test fixes

status:
py2: 6 failures, 0 errors
py3: 7 failures, 0 errors

Products/PortalTransforms/tests/base.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
-import six
 import unittest
 
+from Products.CMFPlone.utils import safe_unicode
 from Products.PortalTransforms.testing import PRODUCTS_PORTALTRANSFORMS_INTEGRATION_TESTING  # noqa
 
 
@@ -13,16 +13,10 @@ def setUp(self):
         self.portal = self.layer['portal']
         self.transforms = self.portal.portal_transforms
 
-    def _decode(self, first, second):
-        if isinstance(first, six.binary_type):
-            first = first.decode('unicode-escape')
-        if isinstance(second, six.binary_type):
-            second = second.decode('unicode-escape')
-
     def _baseAssertEqual(self, first, second, msg=None):
-        self._decode(first, second)
-        return unittest.TestCase._baseAssertEqual(self, first, second, msg)
+        return unittest.TestCase._baseAssertEqual(
+            self, safe_unicode(first), safe_unicode(second), msg)
 
     def assertMultiLineEqual(self, first, second, msg=None):
-        self._decode(first, second)
-        return unittest.TestCase.assertMultiLineEqual(self, first, second, msg)
+        return unittest.TestCase.assertMultiLineEqual(
+            self, safe_unicode(first), safe_unicode(second), msg)

Products/PortalTransforms/tests/test_transforms.py

@@ -2,6 +2,7 @@
 from __future__ import print_function
 from Products.CMFCore.utils import getToolByName
 from Products.CMFPlone.interfaces import IFilterSchema
+from Products.CMFPlone.utils import safe_unicode
 from Products.PortalTransforms.data import datastream
 from Products.PortalTransforms.interfaces import IDataStream
 from Products.PortalTransforms.libtransforms.utils import MissingBinary
@@ -75,14 +76,13 @@ def do_convert(self, filename=None):
                 fd.write(got)
             self.assertTrue(0)
 
+        if six.PY3:
+            got = safe_unicode(got)
+            expected = safe_unicode(expected)
+
         if self.normalize is not None:
-            expected = self.normalize(expected)
             got = self.normalize(got)
-
-        if isinstance(got, six.binary_type):
-            got = got.decode('unicode-escape')
-        if isinstance(expected, six.binary_type):
-            expected = expected.decode('unicode-escape')
+            expected = self.normalize(expected)
 
         # show the first character ord table for debugging
         got_start = got.strip()[:40]

Products/PortalTransforms/transforms/safe_html.py

@@ -66,16 +66,17 @@ def decode_charref(s):
 
 def decode_entityref(s):
     s = s.group(1)
+    # python3 has its own html5 entitydef translation dict
+    # unfortunytely not backported in six for python 2
+    entitydefs = six.PY3 and six.moves.html_entities.html5 or html5entities
     try:
-        c = html5entities[s + ';']
+        c = entitydefs[s + ';']
     except KeyError:
         try:
-            c = html5entities[s]
+            c = entitydefs[s]
         except KeyError:
             # strip unrecognized entities
             c = u''
-    if isinstance(s, six.text_type):
-        c = c.encode('utf8')
     return c
 
 
@@ -2428,7 +2429,7 @@ def strip_outer(s):
 
         valid_tags = self.settings.valid_tags
         nasty_tags = [
-            tag for tag in self.settings.nasty_tags if tag not in valid_tags]
+            t for t in self.settings.nasty_tags if t not in valid_tags]
         if six.PY2:
             safe_attrs = [attr.decode() for attr in html.defs.safe_attrs]
         else: