From 3c33a22c3eb15213aea45cd8728c8763a42ce10a Mon Sep 17 00:00:00 2001 From: antcorro Date: Mon, 4 Apr 2016 21:28:11 +0200 Subject: [PATCH] Fix for #82; Improper handling of badly-encoded documents. * fix logging warning message * decode as ascii on Decode error * Add test case for #88 #82 * add test case bis for #88 #82 --- django_mailbox/models.py | 4 +- .../tests/messages/email_issue_82.eml | 44 ++++++++++++ django_mailbox/tests/test_process_email.py | 70 +++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 django_mailbox/tests/messages/email_issue_82.eml diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 9eaa518..6b76d26 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -347,12 +347,12 @@ class Mailbox(models.Model): ) except ValueError: logger.warning( - "Decoding error encountered; interpreting as ASCII!", + "Decoding error encountered; interpreting %s as ASCII!", content_charset ) msg.set_payload( msg.get_payload(decode=True).decode( - content_charset, + 'ascii', 'ignore' ) ) diff --git a/django_mailbox/tests/messages/email_issue_82.eml b/django_mailbox/tests/messages/email_issue_82.eml new file mode 100644 index 0000000..13cb98c --- /dev/null +++ b/django_mailbox/tests/messages/email_issue_82.eml @@ -0,0 +1,44 @@ +From nobody Wed Mar 30 12:15:10 2016 +Delivered-To: info@test.org +X-Orig-To: info@test.org +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="----_=_NextPart_001_01D18A74.C17A9D2D" +Subject: character issue. +Date: Wed, 30 Mar 2016 08:10:23 -0300 +Message-ID: <2A4988FCD265FE438FD0848B3ED91ABA2C0AAF3E@issue> +From: "Me" +To: + +This is a multi-part message in MIME format. + +------_=_NextPart_001_01D18A74.C17A9D2D +Content-Type: multipart/alternative; + boundary="----_=_NextPart_002_01D18A74.C17A9D2D" + + +------_=_NextPart_002_01D18A74.C17A9D2D +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: base64 + + +QXRlbmNpw7NuIGFsdXN0b21lciBTZXJ2aWNlIEYW9ubGluZQ0KCVIA0KCURJ +aWxlZ2lhZGFzIChwcm90ZWdpZGFzIHBvciBzaWdpbG8gcHJvZmlzc2lvbmFs +IGNvbnRyYXR1YWwsIHBvciBsZWkgZGUKcHJvcHJpZWRhZGUgaW50ZWxlY3R1 +YWwvaW5kdXN0cmlhbCBvdSBjdWphIGRpdnVsZ2Hn428gc2VqYSBwcm9pYmlk +YSBwb3Igc2V1IHByb3ByaWV04XJpbykgcXVlIHPjbyBkZSBpbnRlcmVzc2Ug +ZXhjbHVzaXZvIGRlIHNldShzKSByZW1ldGVudGUocykgZQpkZXN0aW5hdOFy +aW8ocykuIENhc28gbuNvIGVzdGVqYSBlbnZvbHZpZG8gbm8gcHJvY2Vzc28g +ZG8gcXVhbCB0cmF0YSBlc3RhIG1lbnNhZ2VtIGUvb3UgcmVjZWJldS1hIHBv +ciBlbmdhbm8sIHBvciBmYXZvciwgZXhjbHVhLWEgZGEgc3VhIGNhaXhhIHBv +c3RhbCBlCmNvbXVuaXF1ZSBvIG9jb3JyaWRvIGFvIHNldSBwcm9wcmlldOFy +aW8uIE8gdXNvLCByZXZlbGHn428sIGRpc3RyaWJ1aefjbyxpbXByZXNz428g +b3UgY/NwaWEgZGUgdG9kYSBvdSBhbGd1bWEgcGFydGUgZGFzIGluZm9ybWHn +9WVzLCBzZW0gcHLpdmlhCmF1dG9yaXph5+NvLCBlc3ThIHN1amVpdG8g4HMg +cGVuYWxpZGFkZXMgY2Fi7XZlaXMuIEFzIGluZm9ybWHn9WVzIGFxdWkgY29u +dGlkYXMgbuNvIG5lY2Vzc2FyaWFtZW50ZSByZWZsZXRlbSBhIG9waW5p428g +ZGEgTUVUQUdBTCBJTkQuIENPTS4gTFREQS4sCmZpY2FuZG8gbyByZW1ldGVu +dGUgaW50ZWlyYW1lbnRlIHJlc3BvbnPhdmVsIHBlbG8gc2V1IGNvbnRl+mRv +LiAKCg== + +------_=_NextPart_002_01D18A74.C17A9D2D-- diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index e3d4778..7a39749 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -5,6 +5,7 @@ import mock import six from django_mailbox.models import Mailbox, Message +from django_mailbox.utils import convert_header_to_unicode from django_mailbox.tests.base import EmailMessageTestCase from django.core.mail import EmailMessage @@ -140,6 +141,75 @@ class TestProcessEmail(EmailMessageTestCase): actual_body ) + def test_message_issue_82(self): + """ Ensure that we properly handle incorrectly encoded messages + + """ + email_object = self._get_email_object('email_issue_82.eml') + it = 'works' + try: + # it's ok to call as_string() before passing email_object + # to _get_dehydrated_message() + email_object.as_string() + except: + it = 'do not works' + + success = True + try: + msg = self.mailbox.process_incoming_message(email_object) + except ValueError: + success = False + + self.assertEqual(it, 'works') + self.assertEqual(True, success) + + def test_message_issue_82_bis(self): + """ Ensure that the email object is good before and after + calling _get_dehydrated_message() + + """ + message = self._get_email_object('email_issue_82.eml') + + success = True + + # this is the code of _process_message() + msg = Message() + # if STORE_ORIGINAL_MESSAGE: + # msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message), save=False) + msg.mailbox = self.mailbox + if 'subject' in message: + msg.subject = convert_header_to_unicode(message['subject'])[0:255] + if 'message-id' in message: + msg.message_id = message['message-id'][0:255] + if 'from' in message: + msg.from_header = convert_header_to_unicode(message['from']) + if 'to' in message: + msg.to_header = convert_header_to_unicode(message['to']) + elif 'Delivered-To' in message: + msg.to_header = convert_header_to_unicode(message['Delivered-To']) + msg.save() + + #here the message is ok + str_msg = message.as_string() + message = self.mailbox._get_dehydrated_message(message, msg) + try: + # here as_string raises UnicodeEncodeError + str_msg = message.as_string() + except: + success = False + + msg.set_body(str_msg) + if message['in-reply-to']: + try: + msg.in_reply_to = Message.objects.filter( + message_id=message['in-reply-to'] + )[0] + except IndexError: + pass + msg.save() + + self.assertEqual(True, success) + def test_message_with_misplaced_utf8_content(self): """ Ensure that we properly handle incorrectly encoded messages