federico/django-mailbox
1
0
Fork 0
forked from mirror/django-mailbox
Code Activity

Adding tests and minor code changes to ensure that we properly handle the distinction between bytes and text when dehydrating/rehydrating messages.

Browse source
This commit is contained in:
Adam Coddington 2013-07-26 19:13:35 -07:00
parent 3ee4b43936
commit adb17e7d67
5 changed files with 109 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

54
django_mailbox/models.py
View file

@ -244,20 +244,26 @@ class Mailbox(models.Model):
placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk)
new = placeholder
else:
payload = msg.get_payload()
for header, value in msg.items():
new[header] = value
charset = msg.get_content_charset()
if charset:
new[ORIGINAL_CHARSET_HEADER] = str(charset)
new.set_charset('utf-8')
else:
charset = 'ascii'
payload = msg.get_payload()
new.set_payload(
# Remove any characters that are invalid in the
# encoding that this part is supposed to be in.
payload.decode(charset, 'replace').encode('utf-8')
)
try:
payload = payload.decode(charset, 'replace')
new[ORIGINAL_CHARSET_HEADER] = charset
new.set_charset('utf-8')
except LookupError:
payload = payload.decode('ascii', 'replace')
new.set_charset('utf-8')
new[ALTERED_MESSAGE_HEADER] = (
'Message declared unknown charset \'%s\'' % (
charset
)
)
# Remove any characters that are invalid in the
# encoding that this part is supposed to be in.
new.set_payload(payload.encode('utf-8'))
return new
def _process_message(self, message):
@ -428,10 +434,11 @@ class Message(models.Model):
):
charset = part.get_content_charset()
this_part = part.get_payload(decode=True)
# For some unknown reason, get_payload() has returned
# to us a unicode object -- it should be bytes :-\
this_part = this_part.encode('unicode-escape')
this_part = this_part.decode('string-escape')
if isinstance(this_part, six.text_type):
# For some unknown reason, get_payload() sometimes
# returns a unicode object -- it should always be bytes
this_part = this_part.encode('unicode-escape')
this_part = this_part.decode('string-escape')
if charset:
this_part = this_part.decode(charset, 'replace')
body += this_part
@ -490,20 +497,23 @@ class Message(models.Model):
new.set_payload('')
else:
payload = msg.get_payload().decode('utf-8')
charset = None
charset = msg.get_content_charset()
if not charset:
charset = 'utf-8'
for header, value in msg.items():
if header == ORIGINAL_CHARSET_HEADER:
payload.encode(value)
# If we process ORIGINAL_CHARSET_HEADER before other
# headers, due to some idiosyncrasies of python's email
# module, we'll end up with duplicated headers --
# set_charset sets a handful of headers if they do not
# already exist. We'll store this value and re-encode
# the payload afterward.
charset = value
# We do not want to preserve this header.
continue
new[header] = value
if charset:
# If we process ORIGINAL_CHARSET_HEADER before other headers
# (in the above), due to some idiosyncrasies of python's email
# module, we'll end up with duplicated headers -- set_charset
# sets a handful of headers if they do not already exist.
new.set_charset(charset)
payload = payload.encode(charset, 'replace')
new.set_charset(charset)
new.set_payload(payload)
return new

3
django_mailbox/tests/messages/generic_message.eml
View file

@ -7,8 +7,7 @@ Message-ID: <CAMdmm+hGH8Dgn-_0xnXJCd=PhyNAiouOYm5zFP0z-foqTO60zA@mail.gmail.com>
Subject: Message Without Attachment
From: Adam Coddington <test@adamcoddington.net>
To: Adam Coddington <test@adamcoddington.net>
Content-Type: text/plain;
charset="iso-8859-1"
Content-Type: text/plain; charset="iso-8859-1"
Content-Transfer-Encoding: 7bit
Hello there.

9
django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml Normal file
View file

@ -0,0 +1,9 @@
Date: Fri, 30 Jan 2012 11:30:01 PST
Subject: Tjest
From: "Somebody" <somebody@somewhere.com>
To: idontknow@somewhere.com
MIME-Version: 1.0
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Ýòî ñîîáùåíèå èìååò íåïðàâèëüíóþ êîäèðîâêà.

9
django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml Normal file
View file

@ -0,0 +1,9 @@
Date: Fri, 30 Jan 2012 11:30:01 PST
Subject: Tjest
From: "Somebody" <somebody@somewhere.com>
To: idontknow@somewhere.com
MIME-Version: 1.0
Content-Type: text/plain; charset="raspberries"
Content-Transfer-Encoding: 7bit
Это сообщение имеет неверная кодировка.

58
django_mailbox/tests/test_process_email.py
View file

@ -131,3 +131,61 @@ class TestProcessEmail(EmailMessageTestCase):
expected_text,
actual_text,
)
def test_message_with_unknown_charset(self):
""" Ensure that we gracefully fail at handling unknown encodings.
Should an unknown encoding be used, we should:
- Add a X-Django-Mailbox-Altered-Message header
- Use UTF-8 (with replacement) as the de-facto (and probably incorrect)
encoding for this payload part.
"""
email_object = self._get_email_object(
'message_with_invalid_payload_encoding.eml'
)
msg = self.mailbox.process_incoming_message(email_object)
actual_body = msg.get_text_body()
expected_body = (
u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd.'
)
self.assertEqual(
expected_body,
actual_body,
)
def test_message_with_invalid_content_for_declared_encoding(self):
""" Ensure that we gracefully handle mis-encoded bodies.
Should a payload body be misencoded, we should:
- Decode the message (with replacement) using the declared encoding.
- Always return a payload body in the declared encoding, using python's
usual replacement mechanisms.
"""
email_object = self._get_email_object(
'message_with_invalid_content_for_declared_encoding.eml',
)
msg = self.mailbox.process_incoming_message(email_object)
actual_body = msg.get_text_body()
expected_body = '??? ????????? ????? ???????????? ?????????.'
self.assertEqual(
actual_body,
expected_body,
)