Properly decode encoded filename headers. Fixes #96.

.travis.yml

@@ -38,7 +38,8 @@ matrix:
     python: '3.4'
 install:
 - pip install -q Django$DJANGO
+- pip install -r test_requirements.txt
 - pip install -q -e .
 script:
-- python setup.py test
+- py.test
 sudo: false

MANIFEST

@@ -1,32 +0,0 @@
-# file GENERATED by distutils, do NOT edit
-setup.py
-django_mailbox/__init__.py
-django_mailbox/admin.py
-django_mailbox/models.py
-django_mailbox/signals.py
-django_mailbox/management/__init__.py
-django_mailbox/management/commands/__init__.py
-django_mailbox/management/commands/getmail.py
-django_mailbox/management/commands/processincomingmessage.py
-django_mailbox/migrations/0001_initial.py
-django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py
-django_mailbox/migrations/0003_auto__add_field_mailbox_active.py
-django_mailbox/migrations/0004_auto__add_field_message_outgoing.py
-django_mailbox/migrations/0005_rename_fields.py
-django_mailbox/migrations/0006_auto__add_field_message_in_reply_to.py
-django_mailbox/migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py
-django_mailbox/migrations/0008_populate_from_to_fields.py
-django_mailbox/migrations/0009_remove_references_table.py
-django_mailbox/migrations/0010_auto__add_field_mailbox_from_email.py
-django_mailbox/migrations/0011_auto__add_field_message_read.py
-django_mailbox/migrations/0012_auto__add_messageattachment.py
-django_mailbox/migrations/__init__.py
-django_mailbox/transports/__init__.py
-django_mailbox/transports/babyl.py
-django_mailbox/transports/generic.py
-django_mailbox/transports/imap.py
-django_mailbox/transports/maildir.py
-django_mailbox/transports/mbox.py
-django_mailbox/transports/mh.py
-django_mailbox/transports/mmdf.py
-django_mailbox/transports/pop3.py

django_mailbox/models.py

@@ -20,13 +20,13 @@ import uuid
 import six
 from six.moves.urllib.parse import parse_qs, unquote, urlparse
 
-from django.conf import settings
+from django.conf import settings as django_settings
 from django.core.files.base import ContentFile
 from django.core.mail.message import make_msgid
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
-from .utils import convert_header_to_unicode, get_body_from_message
+from django_mailbox import utils
 from django_mailbox.signals import message_received
 from django_mailbox.transports import Pop3Transport, ImapTransport, \
     MaildirTransport, MboxTransport, BabylTransport, MHTransport, \
@@ -35,55 +35,6 @@ from django_mailbox.transports import Pop3Transport, ImapTransport, \
 logger = logging.getLogger(__name__)
 
 
-STRIP_UNALLOWED_MIMETYPES = getattr(
-    settings,
-    'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES',
-    False
-)
-
-ALLOWED_MIMETYPES = getattr(
-    settings,
-    'DJANGO_MAILBOX_ALLOWED_MIMETYPES',
-    [
-        'text/plain',
-        'text/html'
-    ]
-)
-
-TEXT_STORED_MIMETYPES = getattr(
-    settings,
-    'DJANGO_MAILBOX_TEXT_STORED_MIMETYPES',
-    [
-        'text/plain',
-        'text/html'
-    ]
-)
-
-ALTERED_MESSAGE_HEADER = getattr(
-    settings,
-    'DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER',
-    'X-Django-Mailbox-Altered-Message'
-)
-
-ATTACHMENT_INTERPOLATION_HEADER = getattr(
-    settings,
-    'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER',
-    'X-Django-Mailbox-Interpolate-Attachment'
-)
-
-ATTACHMENT_UPLOAD_TO = getattr(
-    settings,
-    'DJANGO_MAILBOX_ATTACHMENT_UPLOAD_TO',
-    'mailbox_attachments/%Y/%m/%d/'
-)
-
-STORE_ORIGINAL_MESSAGE = getattr(
-    settings,
-    'DJANGO_MAILBOX_STORE_ORIGINAL_MESSAGE',
-    False
-)
-
-
 class ActiveMailboxManager(models.Manager):
     def get_queryset(self):
         return super(ActiveMailboxManager, self).get_queryset().filter(
@@ -272,6 +223,8 @@ class Mailbox(models.Model):
         return msg
 
     def _get_dehydrated_message(self, msg, record):
+        settings = utils.get_settings()
+
         new = EmailMessage()
         if msg.is_multipart():
             for header, value in msg.items():
@@ -281,25 +234,27 @@ class Mailbox(models.Model):
                     self._get_dehydrated_message(part, record)
                 )
         elif (
-            STRIP_UNALLOWED_MIMETYPES
-            and not msg.get_content_type() in ALLOWED_MIMETYPES
+            settings['strip_unallowed_mimetypes']
+            and not msg.get_content_type() in settings['allowed_mimetypes']
         ):
             for header, value in msg.items():
                 new[header] = value
             # Delete header, otherwise when attempting to  deserialize the
             # payload, it will be expecting a body for this.
             del new['Content-Transfer-Encoding']
-            new[ALTERED_MESSAGE_HEADER] = (
+            new[settings['altered_message_header']] = (
                 'Stripped; Content type %s not allowed' % (
                     msg.get_content_type()
                 )
             )
             new.set_payload('')
         elif (
-            (msg.get_content_type() not in TEXT_STORED_MIMETYPES) or
+            (
+                msg.get_content_type() not in settings['text_stored_mimetypes']
+            ) or
             ('attachment' in msg.get('Content-Disposition', ''))
         ):
-            filename = msg.get_filename()
+            filename = utils.convert_header_to_unicode(msg.get_filename())
             if not filename:
                 extension = mimetypes.guess_extension(msg.get_content_type())
             else:
@@ -323,7 +278,9 @@ class Mailbox(models.Model):
             attachment.save()
 
             placeholder = EmailMessage()
-            placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk)
+            placeholder[
+                settings['attachment_interpolation_header']
+            ] = str(attachment.pk)
             new = placeholder
         else:
             content_charset = msg.get_content_charset()
@@ -361,19 +318,29 @@ class Mailbox(models.Model):
 
     def _process_message(self, message):
         msg = Message()
-        if STORE_ORIGINAL_MESSAGE:
-            msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message.as_string(unixfrom=True)), save=False)
+        settings = utils.get_settings()
+
+        if settings['store_original_message']:
+            msg.eml.save(
+                '%s.eml' % uuid.uuid4(),
+                ContentFile(message.as_string(unixfrom=True)),
+                save=False
+            )
         msg.mailbox = self
         if 'subject' in message:
-            msg.subject = convert_header_to_unicode(message['subject'])[0:255]
+            msg.subject = (
+                utils.convert_header_to_unicode(message['subject'])[0:255]
+            )
         if 'message-id' in message:
             msg.message_id = message['message-id'][0:255].strip()
         if 'from' in message:
-            msg.from_header = convert_header_to_unicode(message['from'])
+            msg.from_header = utils.convert_header_to_unicode(message['from'])
         if 'to' in message:
-            msg.to_header = convert_header_to_unicode(message['to'])
+            msg.to_header = utils.convert_header_to_unicode(message['to'])
         elif 'Delivered-To' in message:
-            msg.to_header = convert_header_to_unicode(message['Delivered-To'])
+            msg.to_header = utils.convert_header_to_unicode(
+                message['Delivered-To']
+            )
         msg.save()
         message = self._get_dehydrated_message(message, msg)
         msg.set_body(message.as_string())
@@ -557,7 +524,7 @@ class Message(models.Model):
             if self.mailbox.from_email:
                 message.from_email = self.mailbox.from_email
             else:
-                message.from_email = settings.DEFAULT_FROM_EMAIL
+                message.from_email = django_settings.DEFAULT_FROM_EMAIL
         message.extra_headers['Message-ID'] = make_msgid()
         message.extra_headers['Date'] = formatdate()
         message.extra_headers['In-Reply-To'] = self.message_id.strip()
@@ -573,7 +540,7 @@ class Message(models.Model):
         """
         Returns the message body matching content type 'text/plain'.
         """
-        return get_body_from_message(
+        return utils.get_body_from_message(
             self.get_email_object(), 'text', 'plain'
         ).replace('=\n', '').strip()
 
@@ -582,12 +549,14 @@ class Message(models.Model):
         """
         Returns the message body matching content type 'text/html'.
         """
-        return get_body_from_message(
+        return utils.get_body_from_message(
             self.get_email_object(), 'text', 'html'
         ).replace('\n', '').strip()
 
     def _rehydrate(self, msg):
         new = EmailMessage()
+        settings = utils.get_settings()
+
         if msg.is_multipart():
             for header, value in msg.items():
                 new[header] = value
@@ -595,10 +564,10 @@ class Message(models.Model):
                 new.attach(
                     self._rehydrate(part)
                 )
-        elif ATTACHMENT_INTERPOLATION_HEADER in msg.keys():
+        elif settings['attachment_interpolation_header'] in msg.keys():
             try:
                 attachment = MessageAttachment.objects.get(
-                    pk=msg[ATTACHMENT_INTERPOLATION_HEADER]
+                    pk=msg[settings['attachment_interpolation_header']]
                 )
                 for header, value in attachment.items():
                     new[header] = value
@@ -627,9 +596,9 @@ class Message(models.Model):
                     del new['Content-Transfer-Encoding']
                     encode_base64(new)
             except MessageAttachment.DoesNotExist:
-                new[ALTERED_MESSAGE_HEADER] = (
+                new[settings['altered_message_header']] = (
                     'Missing; Attachment %s not found' % (
-                        msg[ATTACHMENT_INTERPOLATION_HEADER]
+                        msg[settings['attachment_interpolation_header']]
                     )
                 )
                 new.set_payload('')
@@ -723,7 +692,7 @@ class MessageAttachment(models.Model):
 
     document = models.FileField(
         _(u'Document'),
-        upload_to=ATTACHMENT_UPLOAD_TO,
+        upload_to=utils.get_attachment_save_path,
     )
 
     def delete(self, *args, **kwargs):
@@ -758,10 +727,11 @@ class MessageAttachment(models.Model):
     def get_filename(self):
         """Returns the original filename of this attachment."""
         file_name = self._get_rehydrated_headers().get_filename()
-        if isinstance(file_name, six.text_type):
-            return file_name
-        elif file_name:
-            return convert_header_to_unicode(file_name)
+        if isinstance(file_name, six.string_types):
+            result = utils.convert_header_to_unicode(file_name)
+            if result is None:
+                return file_name
+            return result
         else:
             return None
 

django_mailbox/runtests.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-import sys
-
-from os.path import dirname, abspath
-
-try:
-    from django import setup
-except ImportError:
-    pass
-from django.conf import settings
-
-if not settings.configured:
-    settings.configure(
-        DATABASES={
-            'default': {
-                'ENGINE': 'django.db.backends.sqlite3',
-            },
-        },
-        INSTALLED_APPS=[
-            'django.contrib.auth',
-            'django.contrib.contenttypes',
-            'django_mailbox',
-        ]
-    )
-
-try:
-    from django.test.runner import DiscoverRunner as TestRunner
-except ImportError:
-    from django.test.simple import DjangoTestSuiteRunner as TestRunner
-
-
-def runtests(*test_args):
-    if not test_args:
-        test_args = ['django_mailbox']
-    parent = dirname(abspath(__file__))
-    sys.path.insert(0, parent)
-    try:
-        # ensure that AppRegistry has loaded
-        setup()
-    except NameError:
-        # This version of Django is too old for an app registry.
-        pass
-    runner = TestRunner(
-        verbosity=1,
-        interactive=False,
-        failfast=False
-    )
-    failures = runner.run_tests(test_args)
-    sys.exit(failures)
-
-
-if __name__ == '__main__':
-    runtests(*sys.argv[1:])

django_mailbox/tests/__init__.py

@@ -1,6 +0,0 @@
-from .test_mailbox import *
-from .test_message_flattening import *
-from .test_process_email import *
-from .test_transports import *
-from .test_integration_imap import *
-from .test_settings import *

django_mailbox/tests/base.py

@@ -7,7 +7,7 @@ import six
 from django.conf import settings
 from django.test import TestCase
 
-from django_mailbox import models
+from django_mailbox import models, utils
 from django_mailbox.models import Mailbox, Message
 
 
@@ -34,9 +34,13 @@ class EmailMessageTestCase(TestCase):
     ]
 
     def setUp(self):
-        self._ALLOWED_MIMETYPES = models.ALLOWED_MIMETYPES
-        self._STRIP_UNALLOWED_MIMETYPES = models.STRIP_UNALLOWED_MIMETYPES
-        self._TEXT_STORED_MIMETYPES = models.TEXT_STORED_MIMETYPES
+        dm_settings = utils.get_settings()
+
+        self._ALLOWED_MIMETYPES = dm_settings['allowed_mimetypes']
+        self._STRIP_UNALLOWED_MIMETYPES = (
+            dm_settings['strip_unallowed_mimetypes']
+        )
+        self._TEXT_STORED_MIMETYPES = dm_settings['text_stored_mimetypes']
 
         self.mailbox = Mailbox.objects.create(from_email='from@example.com')
 

django_mailbox/tests/settings.py

@@ -0,0 +1,11 @@
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+    },
+}
+INSTALLED_APPS = [
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django_mailbox',
+]
+SECRET_KEY = 'beepboop'

django_mailbox/tests/test_message_flattening.py

@@ -1,4 +1,8 @@
-from django_mailbox import models
+import copy
+
+import mock
+
+from django_mailbox import models, utils
 from django_mailbox.models import Message
 from django_mailbox.tests.base import EmailMessageTestCase
 
@@ -74,10 +78,16 @@ class TestMessageFlattening(EmailMessageTestCase):
         expected_email_object = self._get_email_object(
             'message_with_many_multiparts_stripped_html.eml',
         )
-        models.STRIP_UNALLOWED_MIMETYPES = True
-        models.ALLOWED_MIMETYPES = ['text/plain']
+        default_settings = utils.get_settings()
 
-        msg = self.mailbox.process_incoming_message(incoming_email_object)
+        with mock.patch('django_mailbox.utils.get_settings') as get_settings:
+            altered = copy.deepcopy(default_settings)
+            altered['strip_unallowed_mimetypes'] = True
+            altered['allowed_mimetypes'] = ['text/plain']
+
+            get_settings.return_value = altered
+
+            msg = self.mailbox.process_incoming_message(incoming_email_object)
 
         actual_email_object = msg.get_email_object()
 

django_mailbox/tests/test_process_email.py

@@ -41,6 +41,31 @@ class TestProcessEmail(EmailMessageTestCase):
             'Adam Coddington <test@adamcoddington.net>',
         )
 
+    def test_message_with_encoded_attachment_filenames(self):
+        message = self._get_email_object(
+            'message_with_koi8r_filename_attachments.eml'
+        )
+
+        mailbox = Mailbox.objects.create()
+        msg = mailbox.process_incoming_message(message)
+
+        attachments = msg.attachments.order_by('pk').all()
+        self.assertEqual(
+            u'\u041f\u0430\u043a\u0435\u0442 \u043f\u0440\u0435\u0434\u043b'
+            u'\u043e\u0436\u0435\u043d\u0438\u0439 HSE Career Fair 8 \u0430'
+            u'\u043f\u0440\u0435\u043b\u044f 2016.pdf',
+            attachments[0].get_filename()
+        )
+        self.assertEqual(
+            u'\u0412\u0435\u0434\u043e\u043c\u043e\u0441\u0442\u0438.pdf',
+            attachments[1].get_filename()
+        )
+        self.assertEqual(
+            u'\u041f\u0430\u043a\u0435\u0442 \u043f\u0440\u0435\u0434\u043b'
+            u'\u043e\u0436\u0435\u043d\u0438\u0439 2016.pptx',
+            attachments[2].get_filename()
+        )
+
     def test_message_with_attachments(self):
         message = self._get_email_object('message_with_attachment.eml')
 

django_mailbox/tests/test_settings.py

@@ -1,10 +0,0 @@
-from django.test import TestCase
-from django.conf import settings
-from django_mailbox.models import ATTACHMENT_UPLOAD_TO
-
-
-class TestSettings(TestCase):
-    def test_default_attachment_upload_to(self):
-        user_setting = getattr(settings, 'DJANGO_MAILBOX_ATTACHMENT_UPLOAD_TO', False)
-        self.assertFalse(user_setting)
-        self.assertEqual(ATTACHMENT_UPLOAD_TO, 'mailbox_attachments/%Y/%m/%d/')

django_mailbox/utils.py

@@ -1,5 +1,6 @@
 import email.header
 import logging
+import os
 
 import six
 
@@ -9,19 +10,65 @@ from django.conf import settings
 logger = logging.getLogger(__name__)
 
 
-DEFAULT_CHARSET = getattr(
-    settings,
-    'DJANGO_MAILBOX_DEFAULT_CHARSET',
-    'iso8859-1',
-)
+def get_settings():
+    return {
+        'strip_unallowed_mimetypes': getattr(
+            settings,
+            'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES',
+            False
+        ),
+        'allowed_mimetypes': getattr(
+            settings,
+            'DJANGO_MAILBOX_ALLOWED_MIMETYPES',
+            [
+                'text/plain',
+                'text/html'
+            ]
+        ),
+        'text_stored_mimetypes': getattr(
+            settings,
+            'DJANGO_MAILBOX_TEXT_STORED_MIMETYPES',
+            [
+                'text/plain',
+                'text/html'
+            ]
+        ),
+        'altered_message_header': getattr(
+            settings,
+            'DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER',
+            'X-Django-Mailbox-Altered-Message'
+        ),
+        'attachment_interpolation_header': getattr(
+            settings,
+            'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER',
+            'X-Django-Mailbox-Interpolate-Attachment'
+        ),
+        'attachment_upload_to': getattr(
+            settings,
+            'DJANGO_MAILBOX_ATTACHMENT_UPLOAD_TO',
+            'mailbox_attachments/%Y/%m/%d/'
+        ),
+        'store_original_message': getattr(
+            settings,
+            'DJANGO_MAILBOX_STORE_ORIGINAL_MESSAGE',
+            False
+        ),
+        'default_charset': getattr(
+            settings,
+            'DJANGO_MAILBOX_default_charset',
+            'iso8859-1',
+        )
+    }
 
 
 def convert_header_to_unicode(header):
+    default_charset = get_settings()['default_charset']
+
     def _decode(value, encoding):
         if isinstance(value, six.text_type):
             return value
         if not encoding or encoding == 'unknown-8bit':
-            encoding = DEFAULT_CHARSET
+            encoding = default_charset
         return value.decode(encoding, 'replace')
 
     try:
@@ -36,9 +83,9 @@ def convert_header_to_unicode(header):
         logger.exception(
             'Errors encountered decoding header %s into encoding %s.',
             header,
-            DEFAULT_CHARSET,
+            default_charset,
         )
-        return unicode(header, DEFAULT_CHARSET, 'replace')
+        return unicode(header, default_charset, 'replace')
 
 
 def get_body_from_message(message, maintype, subtype):
@@ -78,3 +125,12 @@ def get_body_from_message(message, maintype, subtype):
             body += this_part
 
     return body
+
+
+def get_attachment_save_path(instance, filename):
+    settings = get_settings()
+
+    return os.path.join(
+        settings['attachment_upload_to'],
+        filename,
+    )

setup.cfg

@@ -0,0 +1,4 @@
+[pytest]
+norecursedirs=env docs lib .eggs
+DJANGO_SETTINGS_MODULE=django_mailbox.tests.settings
+addopts = --tb=short -rxs

setup.py

@@ -22,12 +22,9 @@ setup(
     ),
     author='Adam Coddington',
     author_email='me@adamcoddington.net',
-    tests_require=tests_require,
     extras_require={
-        'test': tests_require,
         'gmail-oauth2': gmail_oauth2_require
     },
-    test_suite='django_mailbox.runtests.runtests',
     classifiers=[
         'Framework :: Django',
         'Intended Audience :: Developers',
@@ -44,6 +41,7 @@ setup(
         'Topic :: Communications :: Email :: Email Clients (MUA)',
     ],
     packages=find_packages(),
+    include_package_data = True,
     install_requires=[
         'six>=1.6.1'
     ]

test_requirements.txt

@@ -0,0 +1,3 @@
+pytest==2.9.1
+pytest-django==2.9.1
+mock==2.0.0

tox.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist=py27,py35
+
+[testenv]
+deps=
+    django
+    -r{toxinidir}/test_requirements.txt
+sitepackages=False
+commands=
+    py.test {posargs}