bpo-40597: email: Use CTE if lines are longer than max_line_length co…

…nsistently RFC5322 in section 2.1.1 mandates that the line cannot be longer than 998 characters and should not be longer than 78 characters (excluding CRLF). When we use raw_data_manager (default for EmailPolicy, EmailMessage) it does the correct thing as long as the message contains characters outside of 7bit US-ASCII set - base64 or qp Content-Transfer-Encoding is applied if the lines would be too long without it. However if our message is limited to the characters from the 7bit US-ASCII set no transfer encoding is applied, and such messages can easily go beyond 78 or even 998 characters. Let's fix the CTE heuristic so it doesn't care about 7bit vs 8bit.
python · bitdancer · May 14, 2020 · May 11, 2020 · May 11, 2020 · 97e200007d84cb6d3fc066f2dec43c9eb3aa3af4
commit 97e200007d84cb6d3fc066f2dec43c9eb3aa3af4
diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py
@@ -146,13 +146,13 @@ def embedded_body(lines): return linesep.join(lines) + linesep
     def normal_body(lines): return b'\n'.join(lines) + b'\n'
     if cte==None:
         # Use heuristics to decide on the "best" encoding.
-        try:
-            return '7bit', normal_body(lines).decode('ascii')
-        except UnicodeDecodeError:
-            pass
-        if (policy.cte_type == '8bit' and
-                max(len(x) for x in lines) <= policy.max_line_length):
-            return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
+        if max(len(x) for x in lines) <= policy.max_line_length:
+            try:
+                return '7bit', normal_body(lines).decode('ascii')
+            except UnicodeDecodeError:
+                pass
+            if policy.cte_type == '8bit':
+                return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
         sniff = embedded_body(lines[:10])
         sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'),
                                           policy.max_line_length)

diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py
@@ -329,6 +329,21 @@ def test_set_text_charset_latin_1(self):
         self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
         self.assertEqual(m.get_content(), content)
 
+    def test_set_text_plain_long_line_heuristics(self):
+        m = self._make_message()
+        content = ("Simple but long message that is over 78 characters"
+                   " long to force transfer encoding.\n")
+        raw_data_manager.set_content(m, content)
+        self.assertEqual(str(m), textwrap.dedent("""\
+            Content-Type: text/plain; charset="utf-8"
+            Content-Transfer-Encoding: quoted-printable
+
+            Simple but long message that is over 78 characters long to =
+            force transfer encoding.
+            """))
+        self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+        self.assertEqual(m.get_content(), content)
+
     def test_set_text_short_line_minimal_non_ascii_heuristics(self):
         m = self._make_message()
         content = "et là il est monté sur moi et il commence à m'éto.\n"

diff --git a/Misc/NEWS.d/next/Library/2020-05-11-19-17-23.bpo-40597.4SGfgm.rst b/Misc/NEWS.d/next/Library/2020-05-11-19-17-23.bpo-40597.4SGfgm.rst
@@ -0,0 +1 @@
+If text content lines are longer than policy.max_line_length, always use a content-encoding to make sure they are wrapped.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		If text content lines are longer than policy.max_line_length, always use a content-encoding to make sure they are wrapped.