bpo-40597: email: Use CTE if lines are longer than max_line_length co…

…nsistently (gh-20038) (gh-20084)

raw_data_manager (default for EmailPolicy, EmailMessage)
does correct wrapping of 'text' parts as long as the message contains
characters outside of 7bit US-ASCII set: base64 or qp
Content-Transfer-Encoding is applied if the lines would be too long
without it.  It did not, however, do this for ascii-only text,
which could result in lines that were longer than
policy.max_line_length or even the rfc 998  maximum.

This changeset fixes the heuristic so that if lines are longer than
policy.max_line_length, it will always apply a
content-transfer-encoding so that the lines are wrapped correctly.
(cherry picked from commit 6f2f475)

Co-authored-by: Arkadiusz Hiler <arek.l1@gmail.com>

Lib/email/contentmanager.py

@@ -146,13 +146,13 @@ def embedded_body(lines): return linesep.join(lines) + linesep
  def normal_body(lines): return b'\n'.join(lines) + b'\n'
  if cte==None:
  # Use heuristics to decide on the "best" encoding.
- try:
- return '7bit', normal_body(lines).decode('ascii')
- except UnicodeDecodeError:
- pass
- if (policy.cte_type == '8bit' and
-  max(len(x) for x in lines) <= policy.max_line_length):
- return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
+ if max(len(x) for x in lines) <= policy.max_line_length:
+ try:
+  return '7bit', normal_body(lines).decode('ascii')
+ except UnicodeDecodeError:
+  pass
+ if policy.cte_type == '8bit':
+  return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
  sniff = embedded_body(lines[:10])
  sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'),
  policy.max_line_length)

Lib/test/test_email/test_contentmanager.py

@@ -329,6 +329,21 @@ def test_set_text_charset_latin_1(self):
  self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
  self.assertEqual(m.get_content(), content)
 
+ def test_set_text_plain_long_line_heuristics(self):
+ m = self._make_message()
+ content = ("Simple but long message that is over 78 characters"
+ " long to force transfer encoding.\n")
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(str(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: quoted-printable
+
+ Simple but long message that is over 78 characters long to =
+ force transfer encoding.
+ """))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
  def test_set_text_short_line_minimal_non_ascii_heuristics(self):
  m = self._make_message()
  content = "et là il est monté sur moi et il commence à m'éto.\n"

Misc/NEWS.d/next/Library/2020-05-11-19-17-23.bpo-40597.4SGfgm.rst

@@ -0,0 +1 @@
+If text content lines are longer than policy.max_line_length, always use a content-encoding to make sure they are wrapped.