Adding tests and minor code changes to ensure that we properly handle the distinction between bytes and text when dehydrating/rehydrating messages.

django_mailbox/models.py

@@ -244,20 +244,26 @@ class Mailbox(models.Model):
             placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk)
             new = placeholder
         else:
+            payload = msg.get_payload()
             for header, value in msg.items():
                 new[header] = value
             charset = msg.get_content_charset()
             if charset:
-                new[ORIGINAL_CHARSET_HEADER] = str(charset)
+                try:
-                new.set_charset('utf-8')
+                    payload = payload.decode(charset, 'replace')
-            else:
+                    new[ORIGINAL_CHARSET_HEADER] = charset
-                charset = 'ascii'
+                    new.set_charset('utf-8')
-            payload = msg.get_payload()
+                except LookupError:
-            new.set_payload(
+                    payload = payload.decode('ascii', 'replace')
-                # Remove any characters that are invalid in the
+                    new.set_charset('utf-8')
-                # encoding that this part is supposed to be in.
+                    new[ALTERED_MESSAGE_HEADER] = (
-                payload.decode(charset, 'replace').encode('utf-8')
+                        'Message declared unknown charset \'%s\'' % (
-            )
+                            charset
+                        )
+                    )
+            # Remove any characters that are invalid in the
+            # encoding that this part is supposed to be in.
+            new.set_payload(payload.encode('utf-8'))
         return new
 
     def _process_message(self, message):
@@ -428,10 +434,11 @@ class Message(models.Model):
                 ):
                     charset = part.get_content_charset()
                     this_part = part.get_payload(decode=True)
-                    # For some unknown reason, get_payload() has returned
+                    if isinstance(this_part, six.text_type):
-                    # to us a unicode object -- it should be bytes :-\
+                        # For some unknown reason, get_payload() sometimes
-                    this_part = this_part.encode('unicode-escape')
+                        # returns a unicode object -- it should always be bytes
-                    this_part = this_part.decode('string-escape')
+                        this_part = this_part.encode('unicode-escape')
+                        this_part = this_part.decode('string-escape')
                     if charset:
                         this_part = this_part.decode(charset, 'replace')
                     body += this_part
@@ -490,20 +497,23 @@ class Message(models.Model):
                 new.set_payload('')
         else:
             payload = msg.get_payload().decode('utf-8')
-            charset = None
+            charset = msg.get_content_charset()
+            if not charset:
+                charset = 'utf-8'
             for header, value in msg.items():
                 if header == ORIGINAL_CHARSET_HEADER:
-                    payload.encode(value)
+                    # If we process ORIGINAL_CHARSET_HEADER before other
+                    # headers, due to some idiosyncrasies of python's email
+                    # module, we'll end up with duplicated headers --
+                    # set_charset sets a handful of headers if they do not
+                    # already exist. We'll store this value and re-encode
+                    # the payload afterward.
                     charset = value
                     # We do not want to preserve this header.
                     continue
                 new[header] = value
-            if charset:
+            payload = payload.encode(charset, 'replace')
-                # If we process ORIGINAL_CHARSET_HEADER before other headers
+            new.set_charset(charset)
-                # (in the above), due to some idiosyncrasies of python's email
-                # module, we'll end up with duplicated headers -- set_charset
-                # sets a handful of headers if they do not already exist.
-                new.set_charset(charset)
             new.set_payload(payload)
         return new
 

django_mailbox/tests/messages/generic_message.eml

@@ -7,8 +7,7 @@ Message-ID: <CAMdmm+hGH8Dgn-_0xnXJCd=PhyNAiouOYm5zFP0z-foqTO60zA@mail.gmail.com>
 Subject: Message Without Attachment
 From: Adam Coddington <test@adamcoddington.net>
 To: Adam Coddington <test@adamcoddington.net>
-Content-Type: text/plain;
+Content-Type: text/plain; charset="iso-8859-1"
-	charset="iso-8859-1"
 Content-Transfer-Encoding: 7bit
 
 Hello there.

django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml

@@ -0,0 +1,9 @@
+Date: Fri, 30 Jan 2012 11:30:01 PST
+Subject: Tjest
+From: "Somebody" <somebody@somewhere.com>
+To: idontknow@somewhere.com
+MIME-Version: 1.0
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Ýòî ñîîáùåíèå èìååò íåïðàâèëüíóþ êîäèðîâêà.

django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml

@@ -0,0 +1,9 @@
+Date: Fri, 30 Jan 2012 11:30:01 PST
+Subject: Tjest
+From: "Somebody" <somebody@somewhere.com>
+To: idontknow@somewhere.com
+MIME-Version: 1.0
+Content-Type: text/plain; charset="raspberries"
+Content-Transfer-Encoding: 7bit
+
+Это сообщение имеет неверная кодировка.

django_mailbox/tests/test_process_email.py

@@ -131,3 +131,61 @@ class TestProcessEmail(EmailMessageTestCase):
             expected_text,
             actual_text,
         )
+
+    def test_message_with_unknown_charset(self):
+        """ Ensure that we gracefully fail at handling unknown encodings.
+
+        Should an unknown encoding be used, we should:
+
+        - Add a X-Django-Mailbox-Altered-Message header
+        - Use UTF-8 (with replacement) as the de-facto (and probably incorrect)
+          encoding for this payload part.
+
+        """
+        email_object = self._get_email_object(
+            'message_with_invalid_payload_encoding.eml'
+        )
+
+        msg = self.mailbox.process_incoming_message(email_object)
+
+        actual_body = msg.get_text_body()
+
+        expected_body = (
+            u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
+            u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
+            u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
+            u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
+            u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
+            u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
+            u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
+            u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd.'
+        )
+
+        self.assertEqual(
+            expected_body,
+            actual_body,
+        )
+
+    def test_message_with_invalid_content_for_declared_encoding(self):
+        """ Ensure that we gracefully handle mis-encoded bodies.
+
+        Should a payload body be misencoded, we should:
+
+        - Decode the message (with replacement) using the declared encoding.
+        - Always return a payload body in the declared encoding, using python's
+          usual replacement mechanisms.
+
+        """
+        email_object = self._get_email_object(
+            'message_with_invalid_content_for_declared_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(email_object)
+
+        actual_body = msg.get_text_body()
+        expected_body = '??? ????????? ????? ???????????? ?????????.'
+
+        self.assertEqual(
+            actual_body,
+            expected_body,
+        )