diff --git a/django_mailbox/models.py b/django_mailbox/models.py index dbe3424..fe6e295 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -244,20 +244,26 @@ class Mailbox(models.Model): placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk) new = placeholder else: + payload = msg.get_payload() for header, value in msg.items(): new[header] = value charset = msg.get_content_charset() if charset: - new[ORIGINAL_CHARSET_HEADER] = str(charset) - new.set_charset('utf-8') - else: - charset = 'ascii' - payload = msg.get_payload() - new.set_payload( - # Remove any characters that are invalid in the - # encoding that this part is supposed to be in. - payload.decode(charset, 'replace').encode('utf-8') - ) + try: + payload = payload.decode(charset, 'replace') + new[ORIGINAL_CHARSET_HEADER] = charset + new.set_charset('utf-8') + except LookupError: + payload = payload.decode('ascii', 'replace') + new.set_charset('utf-8') + new[ALTERED_MESSAGE_HEADER] = ( + 'Message declared unknown charset \'%s\'' % ( + charset + ) + ) + # Remove any characters that are invalid in the + # encoding that this part is supposed to be in. + new.set_payload(payload.encode('utf-8')) return new def _process_message(self, message): @@ -428,10 +434,11 @@ class Message(models.Model): ): charset = part.get_content_charset() this_part = part.get_payload(decode=True) - # For some unknown reason, get_payload() has returned - # to us a unicode object -- it should be bytes :-\ - this_part = this_part.encode('unicode-escape') - this_part = this_part.decode('string-escape') + if isinstance(this_part, six.text_type): + # For some unknown reason, get_payload() sometimes + # returns a unicode object -- it should always be bytes + this_part = this_part.encode('unicode-escape') + this_part = this_part.decode('string-escape') if charset: this_part = this_part.decode(charset, 'replace') body += this_part @@ -490,20 +497,23 @@ class Message(models.Model): new.set_payload('') else: payload = msg.get_payload().decode('utf-8') - charset = None + charset = msg.get_content_charset() + if not charset: + charset = 'utf-8' for header, value in msg.items(): if header == ORIGINAL_CHARSET_HEADER: - payload.encode(value) + # If we process ORIGINAL_CHARSET_HEADER before other + # headers, due to some idiosyncrasies of python's email + # module, we'll end up with duplicated headers -- + # set_charset sets a handful of headers if they do not + # already exist. We'll store this value and re-encode + # the payload afterward. charset = value # We do not want to preserve this header. continue new[header] = value - if charset: - # If we process ORIGINAL_CHARSET_HEADER before other headers - # (in the above), due to some idiosyncrasies of python's email - # module, we'll end up with duplicated headers -- set_charset - # sets a handful of headers if they do not already exist. - new.set_charset(charset) + payload = payload.encode(charset, 'replace') + new.set_charset(charset) new.set_payload(payload) return new diff --git a/django_mailbox/tests/messages/generic_message.eml b/django_mailbox/tests/messages/generic_message.eml index 438eea7..b3c9f33 100644 --- a/django_mailbox/tests/messages/generic_message.eml +++ b/django_mailbox/tests/messages/generic_message.eml @@ -7,8 +7,7 @@ Message-ID: Subject: Message Without Attachment From: Adam Coddington To: Adam Coddington -Content-Type: text/plain; - charset="iso-8859-1" +Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: 7bit Hello there. diff --git a/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml b/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml new file mode 100644 index 0000000..8860e9d --- /dev/null +++ b/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml @@ -0,0 +1,9 @@ +Date: Fri, 30 Jan 2012 11:30:01 PST +Subject: Tjest +From: "Somebody" +To: idontknow@somewhere.com +MIME-Version: 1.0 +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Это сообщение имеет неправильную кодировка. diff --git a/django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml b/django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml new file mode 100644 index 0000000..3cf61cd --- /dev/null +++ b/django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml @@ -0,0 +1,9 @@ +Date: Fri, 30 Jan 2012 11:30:01 PST +Subject: Tjest +From: "Somebody" +To: idontknow@somewhere.com +MIME-Version: 1.0 +Content-Type: text/plain; charset="raspberries" +Content-Transfer-Encoding: 7bit + +Это сообщение имеет неверная РєРѕРґРёСЂРѕРІРєР°. diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 5932799..6ba8c7c 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -131,3 +131,61 @@ class TestProcessEmail(EmailMessageTestCase): expected_text, actual_text, ) + + def test_message_with_unknown_charset(self): + """ Ensure that we gracefully fail at handling unknown encodings. + + Should an unknown encoding be used, we should: + + - Add a X-Django-Mailbox-Altered-Message header + - Use UTF-8 (with replacement) as the de-facto (and probably incorrect) + encoding for this payload part. + + """ + email_object = self._get_email_object( + 'message_with_invalid_payload_encoding.eml' + ) + + msg = self.mailbox.process_incoming_message(email_object) + + actual_body = msg.get_text_body() + + expected_body = ( + u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd ' + u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd' + u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd ' + u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd ' + u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd' + u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd ' + u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd' + u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd.' + ) + + self.assertEqual( + expected_body, + actual_body, + ) + + def test_message_with_invalid_content_for_declared_encoding(self): + """ Ensure that we gracefully handle mis-encoded bodies. + + Should a payload body be misencoded, we should: + + - Decode the message (with replacement) using the declared encoding. + - Always return a payload body in the declared encoding, using python's + usual replacement mechanisms. + + """ + email_object = self._get_email_object( + 'message_with_invalid_content_for_declared_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(email_object) + + actual_body = msg.get_text_body() + expected_body = '??? ????????? ????? ???????????? ?????????.' + + self.assertEqual( + actual_body, + expected_body, + )