diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 010c557..1bce91a 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -57,6 +57,11 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr( 'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER', 'X-Django-Mailbox-Interpolate-Attachment' ) +ORIGINAL_CHARSET_HEADER = getattr( + settings, + 'DJANGO_MAILBOX_ORIGINAL_CHARSET_HEADER', + 'X-Django-Mailbox-Original-Charset' +) class ActiveMailboxManager(models.Manager): @@ -241,8 +246,15 @@ class Mailbox(models.Model): else: for header, value in msg.items(): new[header] = value + charset = msg.get_content_charset() + if charset: + new[ORIGINAL_CHARSET_HEADER] = str(charset) + new.set_charset('utf-8') + payload = msg.get_payload() new.set_payload( - msg.get_payload() + # Remove any characters that are invalid in the + # encoding that this part is supposed to be in. + payload.decode(charset, 'replace').encode('utf-8') ) return new @@ -259,7 +271,7 @@ class Mailbox(models.Model): msg.to_header = message['to'] msg.save() message = self._get_dehydrated_message(message, msg) - msg.body = unicode(message.as_string(),'utf8','ignore') + msg.body = message.as_string().decode('utf-8') if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( @@ -414,8 +426,12 @@ class Message(models.Model): ): charset = part.get_content_charset() this_part = part.get_payload(decode=True) + # For some unknown reason, get_payload() has returned + # to us a unicode object -- it should be bytes :-\ + this_part = this_part.encode('unicode-escape') + this_part = this_part.decode('string-escape') if charset: - this_part = this_part.decode(charset) + this_part = this_part.decode(charset, 'replace') body += this_part return body @@ -471,11 +487,15 @@ class Message(models.Model): ) new.set_payload('') else: + payload = msg.get_payload().decode('utf-8') for header, value in msg.items(): + if header == ORIGINAL_CHARSET_HEADER: + payload.encode(value) + new.set_charset(value) + # We do not want to preserve this header + continue new[header] = value - new.set_payload( - msg.get_payload() - ) + new.set_payload(payload) return new def get_email_object(self): diff --git a/django_mailbox/tests/message_with_utf16_char.eml b/django_mailbox/tests/message_with_utf8_char.eml similarity index 96% rename from django_mailbox/tests/message_with_utf16_char.eml rename to django_mailbox/tests/message_with_utf8_char.eml index 7869100..83761c5 100644 --- a/django_mailbox/tests/message_with_utf16_char.eml +++ b/django_mailbox/tests/message_with_utf8_char.eml @@ -55,11 +55,3 @@ X-Mailer: Seven Enterprise Gateway (v 2.0) This message contains funny UTF16 characters like this one: " " and this one "✿". ------Original Message----- -Date: Wednesday, December 26, 2012 1:53:27 pm -To: user2@yahoo.com -From: "Person" -Subject: Fwd: Stuff - -Person - -etc... diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index bf92dc8..9df4b7f 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -109,3 +109,25 @@ class TestProcessEmail(EmailMessageTestCase): expected_body, actual_body ) + + def test_message_with_misplaced_utf8_content(self): + """ Ensure that we properly handle incorrectly encoded messages + + ``message_with_utf8_char.eml``'s primary text payload is marked + as being iso-8859-1 data, but actually contains UTF-8 bytes. + + """ + email_object = self._get_email_object('message_with_utf8_char.eml') + + msg = self.mailbox.process_incoming_message(email_object) + + actual_text = msg.get_text_body() + expected_text = ( + u'This message contains funny UTF16 characters like this one: ' + u'"\xc2\xa0" and this one "\xe2\x9c\xbf".' + ) + + self.assertEqual( + expected_text, + actual_text, + )