From e33515979a9994b4285d48405daf260752ccda34 Mon Sep 17 00:00:00 2001 From: PaoloC68 Date: Thu, 25 Feb 2016 15:38:41 -0300 Subject: [PATCH 1/2] Proposal for issue #82 --- django_mailbox/models.py | 4 +- django_mailbox/utils.py | 120 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 3 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index ce218de..36c6bea 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -26,7 +26,7 @@ from django.core.mail.message import make_msgid from django.db import models from django.utils.translation import ugettext_lazy as _ -from .utils import convert_header_to_unicode, get_body_from_message +from .utils import convert_header_to_unicode, get_body_from_message, cleanup_message from django_mailbox.signals import message_received from django_mailbox.transports import Pop3Transport, ImapTransport, \ MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ @@ -375,7 +375,7 @@ class Mailbox(models.Model): elif 'Delivered-To' in message: msg.to_header = convert_header_to_unicode(message['Delivered-To']) msg.save() - message = self._get_dehydrated_message(message, msg) + message = cleanup_message(self._get_dehydrated_message(message, msg)) msg.set_body(message.as_string()) if message['in-reply-to']: try: diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index 75b1ee2..13633e3 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -1,6 +1,7 @@ import email.header import logging - +from email import utils +from email import charset import six from django.conf import settings @@ -78,3 +79,120 @@ def get_body_from_message(message, maintype, subtype): body += this_part return body + +# From http://tools.ietf.org/html/rfc5322#section-3.6 +ADDR_HEADERS = ('resent-from', + 'resent-sender', + 'resent-to', + 'resent-cc', + 'resent-bcc', + 'from', + 'sender', + 'reply-to', + 'to', + 'cc', + 'bcc') + +PARAM_HEADERS = ('content-type', + 'content-disposition') + + +def cleanup_message(message, addr_headers=None, param_headers=None): + """ + Cleanup a `Message` handling header and payload charsets. + + Headers are handled in the most sane way possible. Address names + are left in `ascii` if possible or encoded to `latin_1` or `utf-8` + and finally encoded according to RFC 2047 without encoding the + address, something the `email` stdlib package doesn't do. + Parameterized headers such as `filename` in the + `Content-Disposition` header, have their values encoded properly + while leaving the rest of the header to be handled without + encoding. Finally, all other header are left in `ascii` if + possible or encoded to `latin_1` or `utf-8` as a whole. + + The message is modified in place and is also returned in such a + state that it can be safely encoded to ascii. + """ + + if addr_headers is None: + addr_headers = ADDR_HEADERS + if param_headers is None: + param_headers = PARAM_HEADERS + + for key, value in message.items(): + if key.lower() in addr_headers: + addrs = [] + for name, addr in utils.getaddresses([value]): + best, encoded = best_charset(name) + if six.PY2: + name = encoded + name = charset.Charset(best).header_encode(name) + addrs.append(utils.formataddr((name, addr))) + value = ', '.join(addrs) + message.replace_header(key, value) + if key.lower() in param_headers: + for param_key, param_value in message.get_params(header=key): + if param_value: + best, encoded = best_charset(param_value) + if six.PY2: + param_value = encoded + if best == 'ascii': + best = None + message.set_param(param_key, param_value, + header=key, charset=best) + else: + best, encoded = best_charset(value) + if six.PY2: + value = encoded + value = charset.Charset(best).header_encode(value) + message.replace_header(key, value) + + payload = message.get_payload() + if payload and isinstance(payload, six.text_type): + best, encoded = best_charset(payload) + if six.PY2: + payload = encoded + message.set_payload(payload, charset=best) + elif isinstance(payload, list): + for part in payload: + cleanup_message(part) + + return message + + +def encode_message(message, + addr_headers=ADDR_HEADERS, param_headers=PARAM_HEADERS): + """ + Encode a `Message` handling headers and payloads. + + Headers are handled in the most sane way possible. Address names + are left in `ascii` if possible or encoded to `latin_1` or `utf-8` + and finally encoded according to RFC 2047 without encoding the + address, something the `email` stdlib package doesn't do. + Parameterized headers such as `filename` in the + `Content-Disposition` header, have their values encoded properly + while leaving the rest of the header to be handled without + encoding. Finally, all other header are left in `ascii` if + possible or encoded to `latin_1` or `utf-8` as a whole. + + The return is a byte string of the whole message. + """ + cleanup_message(message) + return message.as_string().encode('ascii') + + +def best_charset(text): + """ + Find the most human-readable and/or conventional encoding for unicode text. + + Prefers `ascii` or `latin_1` and falls back to `utf_8`. + """ + encoded = text + for charset in 'ascii', 'latin_1', 'utf_8': + try: + encoded = text.encode(charset) + except UnicodeError: + pass + else: + return charset, encoded From e332e5e24538417a1afd7365398343306d6d8d61 Mon Sep 17 00:00:00 2001 From: PaoloC68 Date: Thu, 25 Feb 2016 15:59:53 -0300 Subject: [PATCH 2/2] fix logging bug. Only one string could be passed as logger argument! --- django_mailbox/models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 36c6bea..079327d 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -336,8 +336,7 @@ class Mailbox(models.Model): msg.get_payload(decode=True).decode(content_charset) except LookupError: logger.warning( - "Unknown encoding %s; interpreting as ASCII!", - content_charset + "Unknown encoding %s; interpreting as ASCII!" ) msg.set_payload( msg.get_payload(decode=True).decode( @@ -347,8 +346,7 @@ class Mailbox(models.Model): ) except ValueError: logger.warning( - "Decoding error encountered; interpreting as ASCII!", - content_charset + "Decoding error encountered; interpreting as ASCII!" ) msg.set_payload( msg.get_payload(decode=True).decode(