forked from mirror/django-mailbox
Major refactor of message attachment handling; no longer stores verbatim
message copy in database. Bumping version number to 2.1 * Walks through incoming message, write attachments to disk as they are found, and alters message body removing actual attachment body, and adding header 'X-Django-Mailbox-Interpolate-Attachment' referencing the ID of the stored attachment. * When calling ``get_email_object``, will walk through stored message, and 're-hydrate' the message by finding said headers, searching for the appropriate record in the MessageAttachments table, and rebuild the message object as closely as possible. Minor fixes: * Properly collect text/plain content from any part of the message; previously would only check the message's first level of payloads, now walks through all payloads, building a string of all text/plain content. * Remove use of deprecated `assertEquals` message.
This commit is contained in:
Adam Coddington
parent
548cc66c52
commit
5f289289ce
10 changed files with 525 additions and 126 deletions
|
|
@ -4,7 +4,7 @@ import os.path
|
|||
from django.test import TestCase
|
||||
import six
|
||||
|
||||
import django_mailbox
|
||||
from django_mailbox import models
|
||||
from django_mailbox.models import Mailbox, Message
|
||||
|
||||
|
||||
|
|
@ -16,22 +16,96 @@ class TestMailbox(TestCase):
|
|||
expected_protocol = 'alpha'
|
||||
actual_protocol = mailbox._protocol_info.scheme
|
||||
|
||||
self.assertEquals(
|
||||
self.assertEqual(
|
||||
expected_protocol,
|
||||
actual_protocol,
|
||||
)
|
||||
|
||||
|
||||
class EmailMessageTestCase(TestCase):
|
||||
def setUp(self):
|
||||
self._ALLOWED_MIMETYPES = models.ALLOWED_MIMETYPES
|
||||
self._STRIP_UNALLOWED_MIMETYPES = models.STRIP_UNALLOWED_MIMETYPES
|
||||
self._TEXT_STORED_MIMETYPES = models.TEXT_STORED_MIMETYPES
|
||||
|
||||
self.mailbox = Mailbox.objects.create()
|
||||
super(EmailMessageTestCase, self).setUp()
|
||||
|
||||
def _get_email_object(self, name):
|
||||
with open(os.path.join(os.path.dirname(__file__), name), 'r') as f:
|
||||
return email.message_from_string(
|
||||
f.read()
|
||||
)
|
||||
|
||||
def compare_email_objects(self, left, right):
|
||||
# Compare headers
|
||||
for key, value in left.items():
|
||||
if not right[key]:
|
||||
raise AssertionError("Extra header '%s'" % key)
|
||||
if right[key].replace('\n\t', ' ') != value.replace('\n\t', ' '):
|
||||
raise AssertionError(
|
||||
"Header '%s' unequal:\n%s\n%s" % (
|
||||
key,
|
||||
repr(value),
|
||||
repr(right[key]),
|
||||
)
|
||||
)
|
||||
for key, value in right.items():
|
||||
if not left[key]:
|
||||
raise AssertionError("Extra header '%s'" % key)
|
||||
if left[key].replace('\n\t', ' ') != value.replace('\n\t', ' '):
|
||||
raise AssertionError(
|
||||
"Header '%s' unequal:\n%s\n%s" % (
|
||||
key,
|
||||
repr(value),
|
||||
repr(right[key]),
|
||||
)
|
||||
)
|
||||
if left.is_multipart() != right.is_multipart():
|
||||
self._raise_mismatched(left, right)
|
||||
if left.is_multipart():
|
||||
left_payloads = left.get_payload()
|
||||
right_payloads = right.get_payload()
|
||||
if len(left_payloads) != len(right_payloads):
|
||||
self._raise_mismatched(left, right)
|
||||
for n in range(len(left_payloads)):
|
||||
self.compare_email_objects(
|
||||
left_payloads[n],
|
||||
right_payloads[n]
|
||||
)
|
||||
else:
|
||||
if left.get_payload() is None or right.get_payload() is None:
|
||||
if left.get_payload() is None:
|
||||
if right.get_payload is not None:
|
||||
self._raise_mismatched(left, right)
|
||||
if right.get_payload() is None:
|
||||
if left.get_payload is not None:
|
||||
self._raise_mismatched(left, right)
|
||||
elif left.get_payload().strip() != right.get_payload().strip():
|
||||
self._raise_mismatched(left, right)
|
||||
|
||||
def _raise_mismatched(self, left, right):
|
||||
raise AssertionError(
|
||||
"Message payloads do not match:\n%s\n%s" % (
|
||||
left.as_string(),
|
||||
right.as_string()
|
||||
)
|
||||
)
|
||||
|
||||
def assertEqual(self, left, right):
|
||||
if not isinstance(left, email.message.Message):
|
||||
return super(EmailMessageTestCase, self).assertEqual(left, right)
|
||||
return self.compare_email_objects(left, right)
|
||||
|
||||
def tearDown(self):
|
||||
for message in Message.objects.all():
|
||||
message.delete()
|
||||
models.ALLOWED_MIMETYPES = self._ALLOWED_MIMETYPES
|
||||
models.STRIP_UNALLOWED_MIMETYPES = self._STRIP_UNALLOWED_MIMETYPES
|
||||
models.TEXT_STORED_MIMETYPES = self._TEXT_STORED_MIMETYPES
|
||||
|
||||
self.mailbox.delete()
|
||||
super(EmailMessageTestCase, self).tearDown()
|
||||
|
||||
|
||||
class TestProcessEmail(EmailMessageTestCase):
|
||||
|
|
@ -48,7 +122,10 @@ class TestProcessEmail(EmailMessageTestCase):
|
|||
self.assertEqual(msg.subject, 'Message Without Attachment')
|
||||
self.assertEqual(
|
||||
msg.message_id,
|
||||
'<CAMdmm+hGH8Dgn-_0xnXJCd=PhyNAiouOYm5zFP0z-foqTO60zA@mail.gmail.com>'
|
||||
(
|
||||
'<CAMdmm+hGH8Dgn-_0xnXJCd=PhyNAiouOYm5zFP0z'
|
||||
'-foqTO60zA@mail.gmail.com>'
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
msg.from_header,
|
||||
|
|
@ -68,14 +145,14 @@ class TestProcessEmail(EmailMessageTestCase):
|
|||
expected_count = 1
|
||||
actual_count = msg.attachments.count()
|
||||
|
||||
self.assertEquals(
|
||||
self.assertEqual(
|
||||
expected_count,
|
||||
actual_count,
|
||||
)
|
||||
|
||||
attachment = msg.attachments.all()[0]
|
||||
self.assertEquals(
|
||||
os.path.basename(attachment.document.name),
|
||||
self.assertEqual(
|
||||
attachment.get_filename(),
|
||||
'heart.png',
|
||||
)
|
||||
|
||||
|
|
@ -88,59 +165,12 @@ class TestProcessEmail(EmailMessageTestCase):
|
|||
expected_results = 'Hello there!'
|
||||
actual_results = msg.get_text_body().strip()
|
||||
|
||||
self.assertEquals(
|
||||
self.assertEqual(
|
||||
expected_results,
|
||||
actual_results,
|
||||
)
|
||||
|
||||
|
||||
class TestFilterMessageBody(EmailMessageTestCase):
|
||||
def setUp(self):
|
||||
django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = True
|
||||
super(TestFilterMessageBody, self).setUp()
|
||||
|
||||
def tearDown(self):
|
||||
django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = False
|
||||
super(TestFilterMessageBody, self).tearDown()
|
||||
|
||||
def test_filter_message_does_not_filter_message_if_disabled(self):
|
||||
django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = False
|
||||
message = self._get_email_object('message_with_attachment.eml')
|
||||
mailbox = Mailbox.objects.create()
|
||||
|
||||
self.assertEquals(
|
||||
message.as_string(),
|
||||
mailbox._filter_message_body(message).as_string()
|
||||
)
|
||||
|
||||
def test_filter_message_removes_unknown_content_if_disabled(self):
|
||||
# The below is the _same_ as message_with_attachment.eml, but missing
|
||||
# its attached png image, and adding the expected message altered header.
|
||||
message_without_non_plaintext = (
|
||||
"MIME-Version: 1.0\n"
|
||||
"Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 12:07:07 -0800 (PST)\n"
|
||||
"X-Originating-IP: [24.22.122.177]\n"
|
||||
"Date: Sun, 20 Jan 2013 12:07:07 -0800\n"
|
||||
"Delivered-To: test@adamcoddington.net\n"
|
||||
"Message-ID: <CAMdmm+jYCgrxrekAxszmDnBjAytcBym-Ec+uM-+HEtzuKy=M_g@mail.gmail.com>\n"
|
||||
"Subject: Message With Attachment\n"
|
||||
"From: Adam Coddington <test@adamcoddington.net>\n"
|
||||
"To: Adam Coddington <test@adamcoddington.net>\n"
|
||||
"Content-Type: multipart/mixed; boundary=047d7b33dd729737fe04d3bde348\n"
|
||||
"X-Django-Mailbox-Altered-Message: Stripped image/png*1, multipart/mixed*1\n"
|
||||
"\n--047d7b33dd729737fe04d3bde348\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\n\n"
|
||||
"This message has an attachment.\n\n--047d7b33dd729737fe04d3bde348--"
|
||||
)
|
||||
message = self._get_email_object('message_with_attachment.eml')
|
||||
mailbox = Mailbox.objects.create()
|
||||
|
||||
self.assertEquals(
|
||||
message_without_non_plaintext,
|
||||
mailbox._filter_message_body(message).as_string()
|
||||
)
|
||||
|
||||
|
||||
class TestGetMessage(EmailMessageTestCase):
|
||||
def test_get_text_body_properly_recomposes_line_continuations(self):
|
||||
message = Message()
|
||||
|
|
@ -156,12 +186,93 @@ class TestGetMessage(EmailMessageTestCase):
|
|||
'but a man stopped to help us and gave us his pump.'
|
||||
)
|
||||
|
||||
self.assertEquals(
|
||||
self.assertEqual(
|
||||
actual_text,
|
||||
expected_text
|
||||
)
|
||||
|
||||
|
||||
class TestMessageFlattening(EmailMessageTestCase):
|
||||
def test_quopri_message_is_properly_rehydrated(self):
|
||||
incoming_email_object = self._get_email_object(
|
||||
'message_with_many_multiparts.eml',
|
||||
)
|
||||
# Note: this is identical to the above, but it appears that
|
||||
# while reading-in an e-mail message, we do alter it slightly
|
||||
expected_email_object = self._get_email_object(
|
||||
'message_with_many_multiparts.eml',
|
||||
)
|
||||
models.TEXT_STORED_MIMETYPES = ['text/plain']
|
||||
|
||||
msg = self.mailbox.process_incoming_message(incoming_email_object)
|
||||
|
||||
actual_email_object = msg.get_email_object()
|
||||
|
||||
self.assertEqual(
|
||||
actual_email_object,
|
||||
expected_email_object,
|
||||
)
|
||||
|
||||
def test_base64_message_is_properly_rehydrated(self):
|
||||
incoming_email_object = self._get_email_object(
|
||||
'message_with_attachment.eml',
|
||||
)
|
||||
# Note: this is identical to the above, but it appears that
|
||||
# while reading-in an e-mail message, we do alter it slightly
|
||||
expected_email_object = self._get_email_object(
|
||||
'message_with_attachment.eml',
|
||||
)
|
||||
|
||||
msg = self.mailbox.process_incoming_message(incoming_email_object)
|
||||
|
||||
actual_email_object = msg.get_email_object()
|
||||
|
||||
self.assertEqual(
|
||||
actual_email_object,
|
||||
expected_email_object,
|
||||
)
|
||||
|
||||
def test_message_handles_rehydration_problems(self):
|
||||
incoming_email_object = self._get_email_object(
|
||||
'message_with_defective_attachment_association.eml',
|
||||
)
|
||||
expected_email_object = self._get_email_object(
|
||||
'message_with_defective_attachment_association_result.eml',
|
||||
)
|
||||
# Note: this is identical to the above, but it appears that
|
||||
# while reading-in an e-mail message, we do alter it slightly
|
||||
message = Message()
|
||||
message.body = incoming_email_object.as_string()
|
||||
|
||||
msg = self.mailbox.process_incoming_message(incoming_email_object)
|
||||
|
||||
actual_email_object = msg.get_email_object()
|
||||
|
||||
self.assertEqual(
|
||||
actual_email_object,
|
||||
expected_email_object,
|
||||
)
|
||||
|
||||
def test_message_content_type_stripping(self):
|
||||
incoming_email_object = self._get_email_object(
|
||||
'message_with_many_multiparts.eml',
|
||||
)
|
||||
expected_email_object = self._get_email_object(
|
||||
'message_with_many_multiparts_stripped_html.eml',
|
||||
)
|
||||
models.STRIP_UNALLOWED_MIMETYPES = True
|
||||
models.ALLOWED_MIMETYPES = ['text/plain']
|
||||
|
||||
msg = self.mailbox.process_incoming_message(incoming_email_object)
|
||||
|
||||
actual_email_object = msg.get_email_object()
|
||||
|
||||
self.assertEqual(
|
||||
actual_email_object,
|
||||
expected_email_object,
|
||||
)
|
||||
|
||||
|
||||
class TestMessageGetEmailObject(TestCase):
|
||||
def test_get_body_properly_handles_unicode_body(self):
|
||||
with open(
|
||||
|
|
@ -178,7 +289,7 @@ class TestMessageGetEmailObject(TestCase):
|
|||
expected_body = unicode_body
|
||||
actual_body = message.get_email_object().as_string()
|
||||
|
||||
self.assertEquals(
|
||||
self.assertEqual(
|
||||
expected_body,
|
||||
actual_body
|
||||
)
|
||||
|
|
|
|||
|
|
@ -25,4 +25,5 @@ ICESkFAJkRAJSIgEpEQCEqYfu6QUkn7sCcyDGQiSACKSKCAkGwBJwhDwZQNMEiYAIBdQvk7rfaHf
|
|||
AO8NBJwCxTGhtFgTHVNaNaJeWFu44AXEHzKCktc7zZ0vss+bMoHSiM2b9mQoX1eZCgGqnWskY3gi
|
||||
XXAAxb8BqFiUgBNY7k49Tu/kV7UKPsefrjEOT9GmghYzrk9V03pjDGYKj3d0c06dKZkpTboRaD9o
|
||||
B+1m2m81d2Az948xzgdjLaFe95e83AAAAABJRU5ErkJggg==
|
||||
|
||||
--047d7b33dd729737fe04d3bde348--
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
X-sender: <sender@sendersdomain.com>
|
||||
X-receiver: <somerecipient@recipientdomain.com>
|
||||
From: "Senders Name" <sender@sendersdomain.com>
|
||||
To: "Recipient Name" <somerecipient@recipientdomain.com>
|
||||
Message-ID: <5bec11c119194c14999e592feb46e3cf@sendersdomain.com>
|
||||
Date: Sat, 24 Sep 2005 15:06:49 -0400
|
||||
Subject: Sample Multi-Part
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/alternative; boundary="----=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D"
|
||||
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D
|
||||
Content-type: text/plain; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
Sample Text Content
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D
|
||||
Content-Type: multipart/alternative; boundary="----=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C"
|
||||
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C
|
||||
Content-Type: text/plain; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
Multipart inside of multipart!
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C
|
||||
X-Django-Mailbox-Interpolate-Attachment: 9013
|
||||
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C--
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D--
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
X-sender: <sender@sendersdomain.com>
|
||||
X-receiver: <somerecipient@recipientdomain.com>
|
||||
From: "Senders Name" <sender@sendersdomain.com>
|
||||
To: "Recipient Name" <somerecipient@recipientdomain.com>
|
||||
Message-ID: <5bec11c119194c14999e592feb46e3cf@sendersdomain.com>
|
||||
Date: Sat, 24 Sep 2005 15:06:49 -0400
|
||||
Subject: Sample Multi-Part
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/alternative; boundary="----=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D"
|
||||
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D
|
||||
Content-type: text/plain; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
Sample Text Content
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D
|
||||
Content-Type: multipart/alternative; boundary="----=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C"
|
||||
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C
|
||||
Content-Type: text/plain; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
Multipart inside of multipart!
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C
|
||||
X-Django-Mailbox-Altered-Message: Missing; Attachment 9013 not found
|
||||
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C--
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D--
|
||||
33
django_mailbox/tests/message_with_many_multiparts.eml
Normal file
33
django_mailbox/tests/message_with_many_multiparts.eml
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
X-sender: <sender@sendersdomain.com>
|
||||
X-receiver: <somerecipient@recipientdomain.com>
|
||||
From: "Senders Name" <sender@sendersdomain.com>
|
||||
To: "Recipient Name" <somerecipient@recipientdomain.com>
|
||||
Message-ID: <5bec11c119194c14999e592feb46e3cf@sendersdomain.com>
|
||||
Date: Sat, 24 Sep 2005 15:06:49 -0400
|
||||
Subject: Sample Multi-Part
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/alternative; boundary="----=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D"
|
||||
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D
|
||||
Content-type: text/plain; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
Sample Text Content
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D
|
||||
Content-Type: multipart/alternative; boundary="----=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C"
|
||||
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C
|
||||
Content-Type: text/plain; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
Multipart inside of multipart!
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C
|
||||
Content-type: text/html; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
<html>
|
||||
<h1>Hello!</h1>
|
||||
</html>
|
||||
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C--
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D--
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
X-sender: <sender@sendersdomain.com>
|
||||
X-receiver: <somerecipient@recipientdomain.com>
|
||||
From: "Senders Name" <sender@sendersdomain.com>
|
||||
To: "Recipient Name" <somerecipient@recipientdomain.com>
|
||||
Message-ID: <5bec11c119194c14999e592feb46e3cf@sendersdomain.com>
|
||||
Date: Sat, 24 Sep 2005 15:06:49 -0400
|
||||
Subject: Sample Multi-Part
|
||||
MIME-Version: 1.0
|
||||
Content-Type: multipart/alternative; boundary="----=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D"
|
||||
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D
|
||||
Content-type: text/plain; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
Sample Text Content
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D
|
||||
Content-Type: multipart/alternative; boundary="----=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C"
|
||||
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C
|
||||
Content-Type: text/plain; charset=iso-8859-1
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
Multipart inside of multipart!
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C
|
||||
Content-type: text/html; charset=iso-8859-1
|
||||
X-Django-Mailbox-Altered-Message: Stripped; Content type text/html not allowed
|
||||
|
||||
------=_OtherPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB011C--
|
||||
------=_NextPart_DC7E1BB5_1105_4DB3_BAE3_2A6208EB099D--
|
||||
Loading…
Add table
Add a link
Reference in a new issue