Fix for #82; Improper handling of badly-encoded documents.

* fix logging warning message

* decode as ascii on Decode error

* Add test case for #88 #82

* add test case bis for #88 #82

django_mailbox/models.py

@@ -347,12 +347,12 @@ class Mailbox(models.Model):
                 )
             except ValueError:
                 logger.warning(
-                    "Decoding error encountered; interpreting as ASCII!",
+                    "Decoding error encountered; interpreting %s as ASCII!",
                     content_charset
                 )
                 msg.set_payload(
                     msg.get_payload(decode=True).decode(
-                        content_charset,
+                        'ascii',
                         'ignore'
                     )
                 )

django_mailbox/tests/messages/email_issue_82.eml

@@ -0,0 +1,44 @@
+From nobody Wed Mar 30 12:15:10 2016
+Delivered-To: info@test.org
+X-Orig-To: info@test.org
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="----_=_NextPart_001_01D18A74.C17A9D2D"
+Subject: character issue.
+Date: Wed, 30 Mar 2016 08:10:23 -0300
+Message-ID: <2A4988FCD265FE438FD0848B3ED91ABA2C0AAF3E@issue>
+From: "Me" <me@test.org>
+To: <info@test.org>
+
+This is a multi-part message in MIME format.
+
+------_=_NextPart_001_01D18A74.C17A9D2D
+Content-Type: multipart/alternative;
+ boundary="----_=_NextPart_002_01D18A74.C17A9D2D"
+
+
+------_=_NextPart_002_01D18A74.C17A9D2D
+Content-Type: text/plain;
+	charset="utf-8"
+Content-Transfer-Encoding: base64
+
+
+QXRlbmNpw7NuIGFsdXN0b21lciBTZXJ2aWNlIEYW9ubGluZQ0KCVIA0KCURJ
+aWxlZ2lhZGFzIChwcm90ZWdpZGFzIHBvciBzaWdpbG8gcHJvZmlzc2lvbmFs
+IGNvbnRyYXR1YWwsIHBvciBsZWkgZGUKcHJvcHJpZWRhZGUgaW50ZWxlY3R1
+YWwvaW5kdXN0cmlhbCBvdSBjdWphIGRpdnVsZ2Hn428gc2VqYSBwcm9pYmlk
+YSBwb3Igc2V1IHByb3ByaWV04XJpbykgcXVlIHPjbyBkZSBpbnRlcmVzc2Ug
+ZXhjbHVzaXZvIGRlIHNldShzKSByZW1ldGVudGUocykgZQpkZXN0aW5hdOFy
+aW8ocykuIENhc28gbuNvIGVzdGVqYSBlbnZvbHZpZG8gbm8gcHJvY2Vzc28g
+ZG8gcXVhbCB0cmF0YSBlc3RhIG1lbnNhZ2VtIGUvb3UgcmVjZWJldS1hIHBv
+ciBlbmdhbm8sIHBvciBmYXZvciwgZXhjbHVhLWEgZGEgc3VhIGNhaXhhIHBv
+c3RhbCBlCmNvbXVuaXF1ZSBvIG9jb3JyaWRvIGFvIHNldSBwcm9wcmlldOFy
+aW8uIE8gdXNvLCByZXZlbGHn428sIGRpc3RyaWJ1aefjbyxpbXByZXNz428g
+b3UgY/NwaWEgZGUgdG9kYSBvdSBhbGd1bWEgcGFydGUgZGFzIGluZm9ybWHn
+9WVzLCBzZW0gcHLpdmlhCmF1dG9yaXph5+NvLCBlc3ThIHN1amVpdG8g4HMg
+cGVuYWxpZGFkZXMgY2Fi7XZlaXMuIEFzIGluZm9ybWHn9WVzIGFxdWkgY29u
+dGlkYXMgbuNvIG5lY2Vzc2FyaWFtZW50ZSByZWZsZXRlbSBhIG9waW5p428g
+ZGEgTUVUQUdBTCBJTkQuIENPTS4gTFREQS4sCmZpY2FuZG8gbyByZW1ldGVu
+dGUgaW50ZWlyYW1lbnRlIHJlc3BvbnPhdmVsIHBlbG8gc2V1IGNvbnRl+mRv
+LiAKCg==
+
+------_=_NextPart_002_01D18A74.C17A9D2D--

django_mailbox/tests/test_process_email.py

@@ -5,6 +5,7 @@ import mock
 import six
 
 from django_mailbox.models import Mailbox, Message
+from django_mailbox.utils import convert_header_to_unicode
 from django_mailbox.tests.base import EmailMessageTestCase
 from django.core.mail import EmailMessage
 
@@ -140,6 +141,75 @@ class TestProcessEmail(EmailMessageTestCase):
             actual_body
         )
 
+    def test_message_issue_82(self):
+        """ Ensure that we properly handle incorrectly encoded messages
+
+        """
+        email_object = self._get_email_object('email_issue_82.eml')
+        it = 'works'
+        try:
+            # it's ok to call as_string() before passing email_object
+            # to _get_dehydrated_message()
+            email_object.as_string()
+        except:
+            it = 'do not works'
+
+        success = True
+        try:
+            msg = self.mailbox.process_incoming_message(email_object)
+        except ValueError:
+            success = False
+
+        self.assertEqual(it, 'works')
+        self.assertEqual(True, success)
+
+    def test_message_issue_82_bis(self):
+        """ Ensure that the email object is good before and after
+        calling _get_dehydrated_message()
+
+        """
+        message = self._get_email_object('email_issue_82.eml')
+
+        success = True
+
+        # this is the code of _process_message()
+        msg = Message()
+        # if STORE_ORIGINAL_MESSAGE:
+        #     msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message), save=False)
+        msg.mailbox = self.mailbox
+        if 'subject' in message:
+            msg.subject = convert_header_to_unicode(message['subject'])[0:255]
+        if 'message-id' in message:
+            msg.message_id = message['message-id'][0:255]
+        if 'from' in message:
+            msg.from_header = convert_header_to_unicode(message['from'])
+        if 'to' in message:
+            msg.to_header = convert_header_to_unicode(message['to'])
+        elif 'Delivered-To' in message:
+            msg.to_header = convert_header_to_unicode(message['Delivered-To'])
+        msg.save()
+
+        #here the message is ok
+        str_msg = message.as_string()
+        message = self.mailbox._get_dehydrated_message(message, msg)
+        try:
+            # here as_string raises UnicodeEncodeError
+            str_msg = message.as_string()
+        except:
+            success = False
+
+        msg.set_body(str_msg)
+        if message['in-reply-to']:
+            try:
+                msg.in_reply_to = Message.objects.filter(
+                    message_id=message['in-reply-to']
+                )[0]
+            except IndexError:
+                pass
+        msg.save()
+
+        self.assertEqual(True, success)
+
     def test_message_with_misplaced_utf8_content(self):
         """ Ensure that we properly handle incorrectly encoded messages