Merge e332e5e245 into 512c44bca6

django_mailbox/models.py

@@ -26,7 +26,7 @@ from django.core.mail.message import make_msgid
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
-from .utils import convert_header_to_unicode, get_body_from_message
+from .utils import convert_header_to_unicode, get_body_from_message, cleanup_message
 from django_mailbox.signals import message_received
 from django_mailbox.transports import Pop3Transport, ImapTransport, \
     MaildirTransport, MboxTransport, BabylTransport, MHTransport, \
@@ -336,8 +336,7 @@ class Mailbox(models.Model):
                 msg.get_payload(decode=True).decode(content_charset)
             except LookupError:
                 logger.warning(
-                    "Unknown encoding %s; interpreting as ASCII!",
-                    content_charset
+                    "Unknown encoding %s; interpreting as ASCII!"
                 )
                 msg.set_payload(
                     msg.get_payload(decode=True).decode(
@@ -347,8 +346,7 @@ class Mailbox(models.Model):
                 )
             except ValueError:
                 logger.warning(
-                    "Decoding error encountered; interpreting as ASCII!",
-                    content_charset
+                    "Decoding error encountered; interpreting as ASCII!"
                 )
                 msg.set_payload(
                     msg.get_payload(decode=True).decode(
@@ -375,7 +373,7 @@ class Mailbox(models.Model):
         elif 'Delivered-To' in message:
             msg.to_header = convert_header_to_unicode(message['Delivered-To'])
         msg.save()
-        message = self._get_dehydrated_message(message, msg)
+        message = cleanup_message(self._get_dehydrated_message(message, msg))
         msg.set_body(message.as_string())
         if message['in-reply-to']:
             try:

django_mailbox/utils.py

@@ -1,6 +1,7 @@
 import email.header
 import logging
-
+from email import utils
+from email import charset
 import six
 
 from django.conf import settings
@@ -78,3 +79,120 @@ def get_body_from_message(message, maintype, subtype):
             body += this_part
 
     return body
+
+# From http://tools.ietf.org/html/rfc5322#section-3.6
+ADDR_HEADERS = ('resent-from',
+                'resent-sender',
+                'resent-to',
+                'resent-cc',
+                'resent-bcc',
+                'from',
+                'sender',
+                'reply-to',
+                'to',
+                'cc',
+                'bcc')
+
+PARAM_HEADERS = ('content-type',
+                 'content-disposition')
+
+
+def cleanup_message(message, addr_headers=None, param_headers=None):
+    """
+    Cleanup a `Message` handling header and payload charsets.
+
+    Headers are handled in the most sane way possible.  Address names
+    are left in `ascii` if possible or encoded to `latin_1` or `utf-8`
+    and finally encoded according to RFC 2047 without encoding the
+    address, something the `email` stdlib package doesn't do.
+    Parameterized headers such as `filename` in the
+    `Content-Disposition` header, have their values encoded properly
+    while leaving the rest of the header to be handled without
+    encoding.  Finally, all other header are left in `ascii` if
+    possible or encoded to `latin_1` or `utf-8` as a whole.
+
+    The message is modified in place and is also returned in such a
+    state that it can be safely encoded to ascii.
+    """
+
+    if addr_headers is None:
+        addr_headers = ADDR_HEADERS
+    if param_headers is None:
+        param_headers = PARAM_HEADERS
+
+    for key, value in message.items():
+        if key.lower() in addr_headers:
+            addrs = []
+            for name, addr in utils.getaddresses([value]):
+                best, encoded = best_charset(name)
+                if six.PY2:
+                    name = encoded
+                name = charset.Charset(best).header_encode(name)
+                addrs.append(utils.formataddr((name, addr)))
+            value = ', '.join(addrs)
+            message.replace_header(key, value)
+        if key.lower() in param_headers:
+            for param_key, param_value in message.get_params(header=key):
+                if param_value:
+                    best, encoded = best_charset(param_value)
+                    if six.PY2:
+                        param_value = encoded
+                    if best == 'ascii':
+                        best = None
+                    message.set_param(param_key, param_value,
+                                      header=key, charset=best)
+        else:
+            best, encoded = best_charset(value)
+            if six.PY2:
+                value = encoded
+            value = charset.Charset(best).header_encode(value)
+            message.replace_header(key, value)
+
+    payload = message.get_payload()
+    if payload and isinstance(payload, six.text_type):
+        best, encoded = best_charset(payload)
+        if six.PY2:
+            payload = encoded
+        message.set_payload(payload, charset=best)
+    elif isinstance(payload, list):
+        for part in payload:
+            cleanup_message(part)
+
+    return message
+
+
+def encode_message(message,
+                   addr_headers=ADDR_HEADERS, param_headers=PARAM_HEADERS):
+    """
+    Encode a `Message` handling headers and payloads.
+
+    Headers are handled in the most sane way possible.  Address names
+    are left in `ascii` if possible or encoded to `latin_1` or `utf-8`
+    and finally encoded according to RFC 2047 without encoding the
+    address, something the `email` stdlib package doesn't do.
+    Parameterized headers such as `filename` in the
+    `Content-Disposition` header, have their values encoded properly
+    while leaving the rest of the header to be handled without
+    encoding.  Finally, all other header are left in `ascii` if
+    possible or encoded to `latin_1` or `utf-8` as a whole.
+
+    The return is a byte string of the whole message.
+    """
+    cleanup_message(message)
+    return message.as_string().encode('ascii')
+
+
+def best_charset(text):
+    """
+    Find the most human-readable and/or conventional encoding for unicode text.
+
+    Prefers `ascii` or `latin_1` and falls back to `utf_8`.
+    """
+    encoded = text
+    for charset in 'ascii', 'latin_1', 'utf_8':
+        try:
+            encoded = text.encode(charset)
+        except UnicodeError:
+            pass
+        else:
+            return charset, encoded