Merging in upstream changes.

.gitignore

@@ -0,0 +1 @@
+*.pyc

.hgignore

@@ -2,3 +2,4 @@
 .*egg-info.*
 build/.*
 dist/.*
+\.git.*

django_mailbox/models.py

@@ -1,4 +1,5 @@
 import email
+from email.message import Message as EmailMessage
 from email.utils import formatdate
 import rfc822
 import urllib
@@ -20,6 +21,24 @@ SKIPPED_EXTENSIONS = getattr(
     'DJANGO_MAILBOX_SKIPPED_EXTENSIONS', 
     ['.p7s']
 )
+STRIP_UNALLOWED_MIMETYPES = getattr(
+    settings,
+    'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES',
+    False
+)
+ALLOWED_MIMETYPES = getattr(
+    settings,
+    'DJANGO_MAILBOX_ALLOWED_MIMETYPES',
+    [
+        'text/plain',
+        'text/html'
+    ]
+)
+ALTERED_MESSAGE_HEADER = getattr(
+    settings,
+    'DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER',
+    'X-Django-Mailbox-Altered-Message'
+)
 
 class ActiveMailboxManager(models.Manager):
     def get_query_set(self):
@@ -155,6 +174,29 @@ class Mailbox(models.Model):
         msg.save()
         return msg
 
+    def _filter_message_body(self, message):
+        if not message.is_multipart() or not STRIP_UNALLOWED_MIMETYPES:
+            return message
+        stripped_content = {}
+        new = EmailMessage()
+        for header, value in message.items():
+            new[header] = value
+        for part in message.walk():
+            content_type = part.get_content_type()
+            print content_type
+            if not content_type in ALLOWED_MIMETYPES:
+                if content_type not in stripped_content:
+                    stripped_content[content_type] = 0
+                stripped_content[content_type] = (
+                    stripped_content[content_type] + 1
+                )
+                continue
+            new.attach(part)
+        new[ALTERED_MESSAGE_HEADER] = 'Stripped ' + ', '.join(
+            ['%s*%s' % (key, value) for key, value in stripped_content.items()]
+        )
+        return new
+
     def _process_message(self, message):
         msg = Message()
         msg.mailbox = self
@@ -162,6 +204,7 @@ class Mailbox(models.Model):
         msg.message_id = message['message-id'][0:255]
         msg.from_header = message['from']
         msg.to_header = message['to']
+        message = self._filter_message_body(message)
         msg.body = message.as_string()
         if message['in-reply-to']:
             try:
@@ -335,7 +378,7 @@ class Message(models.Model):
 
         return get_body_from_message(
             self.get_email_object()
-        )
+        ).replace('=\n', '').rstrip('\n')
 
     def get_email_object(self):
         return email.message_from_string(self.body)

django_mailbox/tests/__init__.py

@@ -4,10 +4,16 @@ import shutil
 
 from django.db import models
 from django.test import TestCase
+import mimic
 
+import django_mailbox
 from django_mailbox.models import Mailbox, Message
 
-class TestProcessMessage(TestCase):
+class EmailMessageTestCase(TestCase):
+    def setUp(self):
+        super(EmailMessageTestCase, self).setUp()
+        self.mimic = mimic.Mimic()
+
     def _get_email_object(self, name):
         with open(os.path.join(os.path.dirname(__file__), name), 'r') as f:
             return email.message_from_string(
@@ -17,7 +23,10 @@ class TestProcessMessage(TestCase):
     def tearDown(self):
         for message in Message.objects.all():
             message.delete()
-    
+
+        self.mimic.verify_all()
+
+class TestProcessEmail(EmailMessageTestCase):
     def test_message_without_attachments(self):
         message = self._get_email_object('generic_message.eml')
 
@@ -75,3 +84,72 @@ class TestProcessMessage(TestCase):
             expected_results,
             actual_results,
         )
+
+class TestFilterMessageBody(EmailMessageTestCase):
+    def setUp(self):
+        django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = True
+        super(TestFilterMessageBody, self).setUp()
+
+    def tearDown(self):
+        django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = False
+        super(TestFilterMessageBody, self).tearDown()
+
+    def test_filter_message_does_not_filter_message_if_disabled(self):
+        django_mailbox.models.STRIP_UNALLOWED_MIMETYPES = False
+        message = self._get_email_object('message_with_attachment.eml')
+        mailbox = Mailbox.objects.create()
+
+        self.assertEquals(
+            message.as_string(),
+            mailbox._filter_message_body(message).as_string()
+        )
+
+    def test_filter_message_removes_unknown_content_if_disabled(self):
+        # The below is the _same_ as message_with_attachment.eml, but missing
+        # its attached png image, and adding the expected message altered header.
+        message_without_non_plaintext = (
+            "MIME-Version: 1.0\n"
+            "Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 12:07:07 -0800 (PST)\n"
+            "X-Originating-IP: [24.22.122.177]\n"
+            "Date: Sun, 20 Jan 2013 12:07:07 -0800\n"
+            "Delivered-To: test@adamcoddington.net\n"
+            "Message-ID: <CAMdmm+jYCgrxrekAxszmDnBjAytcBym-Ec+uM-+HEtzuKy=M_g@mail.gmail.com>\n"
+            "Subject: Message With Attachment\n"
+            "From: Adam Coddington <test@adamcoddington.net>\n"
+            "To: Adam Coddington <test@adamcoddington.net>\n"
+            "Content-Type: multipart/mixed; boundary=047d7b33dd729737fe04d3bde348\n"
+            "X-Django-Mailbox-Altered-Message: Stripped image/png*1, multipart/mixed*1\n"
+            "\n--047d7b33dd729737fe04d3bde348\n"
+            "Content-Type: text/plain; charset=UTF-8\n\n"
+            "This message has an attachment.\n\n--047d7b33dd729737fe04d3bde348--"
+        )
+        message = self._get_email_object('message_with_attachment.eml')
+        mailbox = Mailbox.objects.create()
+
+        self.assertEquals(
+            message_without_non_plaintext,
+            mailbox._filter_message_body(message).as_string()
+        )
+
+class TestGetMessage(EmailMessageTestCase):
+    def test_get_text_body_properly_recomposes_line_continuations(self):
+        message = Message()
+        email_object = self._get_email_object(
+            'message_with_long_text_lines.eml'
+        )
+
+        self.mimic.stub_out_with_mock(message, 'get_email_object')
+        message.get_email_object().and_return(email_object)
+
+        self.mimic.replay_all()
+
+        actual_text = message.get_text_body()
+        expected_text = (
+            u'The one of us with a bike pump is far ahead, '
+            u'but a man stopped to help us and gave us his pump.'
+        )
+
+        self.assertEquals(
+            actual_text,
+            expected_text
+        )

django_mailbox/tests/message_with_long_text_lines.eml

@@ -0,0 +1,15 @@
+Return-path: <who@somewhere.net>
+Delivery-date: Sat, 01 Jun 2013 00:30:16 +0000
+Content-Type: text/plain; charset=us-ascii
+Content-Transfer-Encoding: quoted-printable
+Subject: Flat tire
+From: Somebody <somebody@somewhere.com>
+Message-Id: <8E4EA9F3-4F4D-45D5-94F8-D3B91B68ABC8@somewhere.net>
+Date: Fri, 31 May 2013 16:39:45 -0700
+To: something@somewhere.net 
+Mime-Version: 1.0 (1.0)
+X-Mailer: iPhone Mail (10B329)
+
+The one of us with a bike pump is far ahead, but a man stopped to help us an=
+d gave us his pump.
+

docs/conf.py

@@ -48,9 +48,9 @@ copyright = u'2013, Adam Coddington'
 # built documents.
 #
 # The short X.Y version.
-version = '1.8.1'
+version = '1.9'
 # The full version, including alpha/beta/rc tags.
-release = '1.8.1'
+release = '1.9'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/topics/settings.rst

@@ -2,11 +2,20 @@
 Settings
 ========
 
-+---------------------------------------+--------------+-------------------------------------------------------------------------+
-| Setting                               | Default      | Notes                                                                   |
-+=======================================+==============+=========================================================================+
-| ``DJANGO_MAILBOX_ADMIN_ENABLED``      | ``True``     | Controls whether mailboxes appear in the Django Admin.                  |
-+---------------------------------------+--------------+-------------------------------------------------------------------------+
-| ``DJANGO_MAILBOX_SKIPPED_EXTENSIONS`` | ``['.p7s']`` | A list of extensions to skip when processing email message attachments. |
-+---------------------------------------+--------------+-------------------------------------------------------------------------+
+* ``DJANGO_MAILBOX_ADMIN_ENABLED``
+    * Default: ``True``
+    * Type: ``boolean``
+    * Controls whether mailboxes appear in the Django Admin.
+* ``DJANGO_MAILBOX_SKIPPED_EXTENSIONS``
+    * Default: ``['.p7s']``
+    * Type: ``list``
+    * A list of extensions to skip when processing e-mail message attachments.
+* ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES``
+    * Default: ``False``
+    * Type: ``boolean``
+    * Controls whether or not we remove mimetypes not specified in ``DJANGO_MAILBOX_PRESERVED_MIMETYPES``.
+* ``DJANGO_MAILBOX_ALLOWED_MIMETYPES``
+    * Default ``['text/html', 'text/plain']``
+    * Type: ``list``
+    * A list of mimetypes that will remain and be stored in the message payload of the message object.  Has no effect unless ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES`` is set to ``True``.
 

setup.py

@@ -1,10 +1,13 @@
 from setuptools import setup
 
-tests_require=['django']
+tests_require=[
+    'django',
+    'mimic',
+]
 
 setup(
     name='django-mailbox',
-    version='1.8.3',
+    version='1.9.1',
     url='http://bitbucket.org/latestrevision/django-mailbox/',
     description='Import mail from POP3, IMAP, local mailboxes or directly from Postfix or Exim4 into your Django application automatically.',
     author='Adam Coddington',