helper: replace email_as_* with email builtins

Python 3.3 added a new feature to the email module, policies
(https://docs.python.org/3.5/library/email.policy.html). Policy objects
allow precise control over how numerous features work when converting to
and from str or bytes. With the `email.policy.SMTP` the behavior of
email_as_bytes and email_as_string can be achieved using the builtin
`.as_string()` and `.as_bytes()` methods, without custom code or the
need to test it. Additionally these methods handle corner cases that we
don't currently handle, such as multi-part messages with different
encodings.

Fixes #1257

8 files changed, 16 insertions, 75 deletions

diff --git a/NEWS b/NEWS
index 492b4708..6ac982e8 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,7 @@
 * Port to python 3. Python 2.x no longer supported
 * feature: Add a new 'namedqueries' buffer type for displaying named queries.
 * feature: Replace twisted with asyncio
+* bug fix: correct handling of subparts with different encodings
 
 0.7:
 * info: missing html mailcap entry now reported as mail body text
diff --git a/alot/commands/envelope.py b/alot/commands/envelope.py
index 8198b1fd..77858e25 100644
--- a/alot/commands/envelope.py
+++ b/alot/commands/envelope.py
@@ -5,6 +5,7 @@
 import argparse
 import datetime
 import email
+import email.policy
 import glob
 import logging
 import os
@@ -22,7 +23,6 @@ from .. import crypto
 from ..account import SendingMailFailed, StoreMailError
 from ..db.errors import DatabaseError
 from ..errors import GPGProblem
-from ..helper import email_as_string
 from ..helper import string_decode
 from ..settings.const import settings
 from ..settings.errors import NoMatchingAccount
@@ -130,7 +130,8 @@ class SaveCommand(Command):
         # store mail locally
         # add Date header
         mail['Date'] = email.utils.formatdate(localtime=True)
-        path = account.store_draft_mail(email_as_string(mail))
+        path = account.store_draft_mail(
+            mail.as_string(policy=email.policy.SMTP))
 
         msg = 'draft saved successfully'
 
@@ -210,7 +211,7 @@ class SendCommand(Command):
             try:
                 self.mail = self.envelope.construct_mail()
                 self.mail['Date'] = email.utils.formatdate(localtime=True)
-                self.mail = email_as_string(self.mail)
+                self.mail = self.mail.as_string(policy=email.policy.SMTP)
             except GPGProblem as e:
                 ui.clear_notify([clearme])
                 ui.notify(str(e), priority='error')
diff --git a/alot/commands/thread.py b/alot/commands/thread.py
index 801ce2f0..6513872c 100644
--- a/alot/commands/thread.py
+++ b/alot/commands/thread.py
@@ -9,6 +9,8 @@ import os
 import re
 import subprocess
 import tempfile
+import email
+import email.policy
 from email.utils import getaddresses, parseaddr, formataddr
 from email.message import Message
 
@@ -34,7 +36,6 @@ from ..db.errors import DatabaseROError
 from ..settings.const import settings
 from ..helper import parse_mailcap_nametemplate
 from ..helper import split_commandstring
-from ..helper import email_as_string
 from ..utils import argparse as cargparse
 from ..widgets.globals import AttachmentWidget
 
@@ -382,7 +383,7 @@ class ForwardCommand(Command):
             original_mail = Message()
             original_mail.set_type('message/rfc822')
             original_mail['Content-Disposition'] = 'attachment'
-            original_mail.set_payload(email_as_string(mail))
+            original_mail.set_payload(mail.as_string(policy=email.policy.SMTP))
             envelope.attach(Attachment(original_mail))
 
         # copy subject
diff --git a/alot/db/envelope.py b/alot/db/envelope.py
index 89c1f984..4c2e5cd9 100644
--- a/alot/db/envelope.py
+++ b/alot/db/envelope.py
@@ -6,6 +6,7 @@ import logging
 import os
 import re
 import email
+import email.policy
 from email.encoders import encode_7or8bit
 from email.mime.text import MIMEText
 from email.mime.multipart import MIMEMultipart
@@ -191,7 +192,7 @@ class Envelope(object):
             inner_msg = textpart
 
         if self.sign:
-            plaintext = helper.email_as_bytes(inner_msg)
+            plaintext = inner_msg.as_bytes(policy=email.policy.SMTP)
             logging.debug('signing plaintext: %s', plaintext)
 
             try:
@@ -236,7 +237,7 @@ class Envelope(object):
             unencrypted_msg = inner_msg
 
         if self.encrypt:
-            plaintext = helper.email_as_bytes(unencrypted_msg)
+            plaintext = unencrypted_msg.as_bytes(policy=email.policy.SMTP)
             logging.debug('encrypting plaintext: %s', plaintext)
 
             try:
diff --git a/alot/db/utils.py b/alot/db/utils.py
index 37ef4c48..e750bfbb 100644
--- a/alot/db/utils.py
+++ b/alot/db/utils.py
@@ -8,6 +8,7 @@ import email
 import email.charset as charset
 from email.header import Header
 from email.iterators import typed_subpart_iterator
+import email.policy
 import email.utils
 import tempfile
 import re
@@ -132,7 +133,7 @@ def _handle_signatures(original, message, params):
     if not malformed:
         try:
             sigs = crypto.verify_detached(
-                helper.email_as_bytes(message.get_payload(0)),
+                message.get_payload(0).as_bytes(policy=email.policy.SMTP),
                 message.get_payload(1).get_payload(decode=True))
         except GPGProblem as e:
             malformed = str(e)
diff --git a/alot/helper.py b/alot/helper.py
index b7b7d25d..2b076964 100644
--- a/alot/helper.py
+++ b/alot/helper.py
@@ -595,49 +595,6 @@ def RFC3156_canonicalize(text):
     return text
 
 
-def email_as_string(mail):
-    """
-    Converts the given message to a string, without mangling "From" lines
-    (like as_string() does).
-
-    :param mail: email to convert to string
-    :rtype: str
-    """
-    fp = StringIO()
-    g = Generator(fp, mangle_from_=False, maxheaderlen=78)
-    g.flatten(mail)
-    as_string = RFC3156_canonicalize(fp.getvalue())
-    return as_string
-
-
-def email_as_bytes(mail):
-    string = email_as_string(mail)
-    charset = mail.get_charset()
-    if charset:
-        charset = str(charset)
-    else:
-        charsets = set(mail.get_charsets())
-        if None in charsets:
-            # None is equal to US-ASCII
-            charsets.discard(None)
-            charsets.add('ascii')
-
-        if len(charsets) == 1:
-            charset = list(charsets)[0]
-        elif 'ascii' in charsets:
-            # If we get here and the assert triggers it means that different
-            # parts of the email are encoded differently. I don't think we're
-            # likely to see that, but it's possible
-            if not {'utf-8', 'ascii', 'us-ascii'}.issuperset(charsets):
-                raise RuntimeError(
-                    "different encodings detected: {}".format(charsets))
-            charset = 'utf-8'  # It's a strict super-set
-        else:
-            charset = 'utf-8'
-
-    return string.encode(charset)
-
-
 def get_xdg_env(env_name, fallback):
     """ Used for XDG_* env variables to return fallback if unset *or* empty """
     env = os.environ.get(env_name)
diff --git a/tests/db/utils_test.py b/tests/db/utils_test.py
index 23885067..7d50d888 100644
--- a/tests/db/utils_test.py
+++ b/tests/db/utils_test.py
@@ -8,6 +8,7 @@ import codecs
 import email
 import email.header
 import email.mime.application
+import email.policy
 import io
 import os
 import os.path
@@ -465,7 +466,7 @@ class TestMessageFromFile(TestCaseClassCleanup):
         text = b'This is some text'
         t = email.mime.text.MIMEText(text, 'plain', 'utf-8')
         _, sig = crypto.detached_signature_for(
-            helper.email_as_bytes(t), self.keys)
+            t.as_bytes(policy=email.policy.SMTP), self.keys)
         s = email.mime.application.MIMEApplication(
             sig, 'pgp-signature', email.encoders.encode_7or8bit)
         m = email.mime.multipart.MIMEMultipart('signed', None, [t, s])
@@ -555,7 +556,7 @@ class TestMessageFromFile(TestCaseClassCleanup):
         else:
             text = b'This is some text'
             t = email.mime.text.MIMEText(text, 'plain', 'utf-8')
-        enc = crypto.encrypt(helper.email_as_bytes(t), self.keys)
+        enc = crypto.encrypt(t.as_bytes(policy=email.policy.SMTP), self.keys)
         e = email.mime.application.MIMEApplication(
             enc, 'octet-stream', email.encoders.encode_7or8bit)
 
diff --git a/tests/helper_test.py b/tests/helper_test.py
index b47e5eff..b87d77af 100644
--- a/tests/helper_test.py
+++ b/tests/helper_test.py
@@ -383,28 +383,6 @@ class TestCallCmd(unittest.TestCase):
         self.assertEqual(code, 42)
 
 
-class TestEmailAsString(unittest.TestCase):
-
-    def test_empty_message(self):
-        message = email.message.Message()
-        actual = helper.email_as_string(message)
-        expected = '\r\n'
-        self.assertEqual(actual, expected)
-
-    def test_empty_message_with_unicode_header(self):
-        """Test if unicode header keys can be used in an email that is
-        converted to string with email_as_string()."""
-        # This is what alot.db.envelope.Envelope.construct_mail() currently
-        # does: It constructs a message object and then copies all headers from
-        # the envelope to the message object.  Some header names are stored as
-        # unicode in the envelope.
-        message = email.message.Message()
-        message[u'X-Unicode-Header'] = 'dummy value'
-        actual = helper.email_as_string(message)
-        expected = 'X-Unicode-Header: dummy value\r\n\r\n'
-        self.assertEqual(actual, expected)
-
-
 class TestShorten(unittest.TestCase):
 
     def test_lt_maxlen(self):