diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/.hgignore b/.hgignore index acaa74b..6a8eec5 100755 --- a/.hgignore +++ b/.hgignore @@ -2,3 +2,4 @@ .*egg-info.* build/.* dist/.* +\.git.* diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 8c65521..f954f92 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -1,4 +1,5 @@ import email +from email.message import Message as EmailMessage from email.utils import formatdate import rfc822 import urllib @@ -20,6 +21,24 @@ SKIPPED_EXTENSIONS = getattr( 'DJANGO_MAILBOX_SKIPPED_EXTENSIONS', ['.p7s'] ) +STRIP_UNALLOWED_MIMETYPES = getattr( + settings, + 'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES', + False +) +ALLOWED_MIMETYPES = getattr( + settings, + 'DJANGO_MAILBOX_ALLOWED_MIMETYPES', + [ + 'text/plain', + 'text/html' + ] +) +ALTERED_MESSAGE_HEADER = getattr( + settings, + 'DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER', + 'X-Django-Mailbox-Altered-Message' +) class ActiveMailboxManager(models.Manager): def get_query_set(self): @@ -155,6 +174,29 @@ class Mailbox(models.Model): msg.save() return msg + def _filter_message_body(self, message): + if not message.is_multipart() or not STRIP_UNALLOWED_MIMETYPES: + return message + stripped_content = {} + new = EmailMessage() + for header, value in message.items(): + new[header] = value + for part in message.walk(): + content_type = part.get_content_type() + print content_type + if not content_type in ALLOWED_MIMETYPES: + if content_type not in stripped_content: + stripped_content[content_type] = 0 + stripped_content[content_type] = ( + stripped_content[content_type] + 1 + ) + continue + new.attach(part) + new[ALTERED_MESSAGE_HEADER] = 'Stripped ' + ', '.join( + ['%s*%s' % (key, value) for key, value in stripped_content.items()] + ) + return new + def _process_message(self, message): msg = Message() msg.mailbox = self @@ -162,6 +204,7 @@ class Mailbox(models.Model): msg.message_id = message['message-id'][0:255] msg.from_header = message['from'] msg.to_header = message['to'] + message = self._filter_message_body(message) msg.body = message.as_string() if message['in-reply-to']: try: @@ -335,7 +378,7 @@ class Message(models.Model): return get_body_from_message( self.get_email_object() - ) + ).replace('=\n', '').rstrip('\n') def get_email_object(self): return email.message_from_string(self.body) diff --git a/django_mailbox/tests/__init__.py b/django_mailbox/tests/__init__.py index 7972100..f00bdb2 100644 --- a/django_mailbox/tests/__init__.py +++ b/django_mailbox/tests/__init__.py @@ -4,10 +4,16 @@ import shutil from django.db import models from django.test import TestCase +import mimic +import django_mailbox from django_mailbox.models import Mailbox, Message -class TestProcessMessage(TestCase): +class EmailMessageTestCase(TestCase): + def setUp(self): + super(EmailMessageTestCase, self).setUp() + self.mimic = mimic.Mimic() + def _get_email_object(self, name): with open(os.path.join(os.path.dirname(__file__), name), 'r') as f: return email.message_from_string( @@ -17,7 +23,10 @@ class TestProcessMessage(TestCase): def tearDown(self): for message in Message.objects.all(): message.delete() - + + self.mimic.verify_all() + +class TestProcessEmail(EmailMessageTestCase): def test_message_without_attachments(self): message = self._get_email_object('generic_message.eml') @@ -75,3 +84,72 @@ class TestProcessMessage(TestCase): expected_results, actual_results, ) + +class TestFilterMessageBody(EmailMessageTestCase): + def setUp(self): + django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = True + super(TestFilterMessageBody, self).setUp() + + def tearDown(self): + django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = False + super(TestFilterMessageBody, self).tearDown() + + def test_filter_message_does_not_filter_message_if_disabled(self): + django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = False + message = self._get_email_object('message_with_attachment.eml') + mailbox = Mailbox.objects.create() + + self.assertEquals( + message.as_string(), + mailbox._filter_message_body(message).as_string() + ) + + def test_filter_message_removes_unknown_content_if_disabled(self): + # The below is the _same_ as message_with_attachment.eml, but missing + # its attached png image, and adding the expected message altered header. + message_without_non_plaintext = ( + "MIME-Version: 1.0\n" + "Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 12:07:07 -0800 (PST)\n" + "X-Originating-IP: [24.22.122.177]\n" + "Date: Sun, 20 Jan 2013 12:07:07 -0800\n" + "Delivered-To: test@adamcoddington.net\n" + "Message-ID: \n" + "Subject: Message With Attachment\n" + "From: Adam Coddington \n" + "To: Adam Coddington \n" + "Content-Type: multipart/mixed; boundary=047d7b33dd729737fe04d3bde348\n" + "X-Django-Mailbox-Altered-Message: Stripped image/png*1, multipart/mixed*1\n" + "\n--047d7b33dd729737fe04d3bde348\n" + "Content-Type: text/plain; charset=UTF-8\n\n" + "This message has an attachment.\n\n--047d7b33dd729737fe04d3bde348--" + ) + message = self._get_email_object('message_with_attachment.eml') + mailbox = Mailbox.objects.create() + + self.assertEquals( + message_without_non_plaintext, + mailbox._filter_message_body(message).as_string() + ) + +class TestGetMessage(EmailMessageTestCase): + def test_get_text_body_properly_recomposes_line_continuations(self): + message = Message() + email_object = self._get_email_object( + 'message_with_long_text_lines.eml' + ) + + self.mimic.stub_out_with_mock(message, 'get_email_object') + message.get_email_object().and_return(email_object) + + self.mimic.replay_all() + + actual_text = message.get_text_body() + expected_text = ( + u'The one of us with a bike pump is far ahead, ' + u'but a man stopped to help us and gave us his pump.' + ) + + self.assertEquals( + actual_text, + expected_text + ) diff --git a/django_mailbox/tests/message_with_long_text_lines.eml b/django_mailbox/tests/message_with_long_text_lines.eml new file mode 100644 index 0000000..db739ad --- /dev/null +++ b/django_mailbox/tests/message_with_long_text_lines.eml @@ -0,0 +1,15 @@ +Return-path: +Delivery-date: Sat, 01 Jun 2013 00:30:16 +0000 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: quoted-printable +Subject: Flat tire +From: Somebody +Message-Id: <8E4EA9F3-4F4D-45D5-94F8-D3B91B68ABC8@somewhere.net> +Date: Fri, 31 May 2013 16:39:45 -0700 +To: something@somewhere.net +Mime-Version: 1.0 (1.0) +X-Mailer: iPhone Mail (10B329) + +The one of us with a bike pump is far ahead, but a man stopped to help us an= +d gave us his pump. + diff --git a/docs/conf.py b/docs/conf.py index a234e8c..c5ef780 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ copyright = u'2013, Adam Coddington' # built documents. # # The short X.Y version. -version = '1.8.1' +version = '1.9' # The full version, including alpha/beta/rc tags. -release = '1.8.1' +release = '1.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9d01f55..18585b4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2,11 +2,20 @@ Settings ======== -+---------------------------------------+--------------+-------------------------------------------------------------------------+ -| Setting | Default | Notes | -+=======================================+==============+=========================================================================+ -| ``DJANGO_MAILBOX_ADMIN_ENABLED`` | ``True`` | Controls whether mailboxes appear in the Django Admin. | -+---------------------------------------+--------------+-------------------------------------------------------------------------+ -| ``DJANGO_MAILBOX_SKIPPED_EXTENSIONS`` | ``['.p7s']`` | A list of extensions to skip when processing email message attachments. | -+---------------------------------------+--------------+-------------------------------------------------------------------------+ +* ``DJANGO_MAILBOX_ADMIN_ENABLED`` + * Default: ``True`` + * Type: ``boolean`` + * Controls whether mailboxes appear in the Django Admin. +* ``DJANGO_MAILBOX_SKIPPED_EXTENSIONS`` + * Default: ``['.p7s']`` + * Type: ``list`` + * A list of extensions to skip when processing e-mail message attachments. +* ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES`` + * Default: ``False`` + * Type: ``boolean`` + * Controls whether or not we remove mimetypes not specified in ``DJANGO_MAILBOX_PRESERVED_MIMETYPES``. +* ``DJANGO_MAILBOX_ALLOWED_MIMETYPES`` + * Default ``['text/html', 'text/plain']`` + * Type: ``list`` + * A list of mimetypes that will remain and be stored in the message payload of the message object. Has no effect unless ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES`` is set to ``True``. diff --git a/setup.py b/setup.py index a786adc..378ab62 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,13 @@ from setuptools import setup -tests_require=['django'] +tests_require=[ + 'django', + 'mimic', +] setup( name='django-mailbox', - version='1.8.3', + version='1.9.1', url='http://bitbucket.org/latestrevision/django-mailbox/', description='Import mail from POP3, IMAP, local mailboxes or directly from Postfix or Exim4 into your Django application automatically.', author='Adam Coddington',