[#130] Fix handling of single-byte filenames in attachment headers; among other unicode handling fixes.

django_mailbox/models.py

@@ -642,8 +642,12 @@ class Message(models.Model):
         if they are encoded as such.
 
         """
+        settings = utils.get_settings()
+
         if self.encoded:
-            return base64.b64decode(self.body.encode('ascii'))
+            return base64.b64decode(
+                self.body.encode(settings['default_charset'], 'replace')
+            )
         return self.body.encode('utf-8')
 
     def set_body(self, body):
@@ -654,10 +658,16 @@ class Message(models.Model):
         no fields existed for storing arbitrary bytes.
 
         """
-        if six.PY3:
+        settings = utils.get_settings()
+
+        if six.PY3 and isinstance(body, six.text_type):
             body = body.encode('utf-8')
+
         self.encoded = True
-        self.body = base64.b64encode(body).decode('ascii')
+        self.body = base64.b64encode(body).decode(
+            settings['default_charset'],
+            'replace',
+        )
 
     def get_email_object(self):
         """Returns an `email.message.Message` instance representing the
@@ -726,18 +736,24 @@ class MessageAttachment(models.Model):
         return super(MessageAttachment, self).delete(*args, **kwargs)
 
     def _get_rehydrated_headers(self):
+        settings = utils.get_settings()
+
         headers = self.headers
         if headers is None:
             return EmailMessage()
-        if sys.version_info < (3, 0):
-            try:
-                headers = headers.encode('utf-8')
-            except UnicodeDecodeError:
-                headers = unicode(headers, 'utf-8').encode('utf-8')
+        if not six.PY3:
+            headers = headers.encode(settings['default_charset'], 'replace')
+
         return email.message_from_string(headers)
 
     def _set_dehydrated_headers(self, email_object):
-        self.headers = email_object.as_string()
+        settings = utils.get_settings()
+
+        headers = email_object.as_string()
+        if isinstance(headers, six.binary_type):
+            headers = headers.decode(settings['default_charset'], 'replace')
+
+        self.headers = headers
 
     def __delitem__(self, name):
         rehydrated = self._get_rehydrated_headers()
@@ -745,7 +761,16 @@ class MessageAttachment(models.Model):
         self._set_dehydrated_headers(rehydrated)
 
     def __setitem__(self, name, value):
+        settings = utils.get_settings()
+
         rehydrated = self._get_rehydrated_headers()
+        if isinstance(value, six.binary_type):
+            # Ensure that no values in the bytestring cannot be encoded
+            # in the default charset
+            value = value.decode(
+                settings['default_charset'],
+                'replace'
+            ).encode(settings['default_charset'], 'replace')
         rehydrated[name] = value
         self._set_dehydrated_headers(rehydrated)
 

django_mailbox/tests/test_process_email.py

@@ -4,11 +4,12 @@ import sys
 import mock
 import six
 
+from django.core.mail import EmailMessage
 from django_mailbox.models import Mailbox, Message
 from django_mailbox.utils import convert_header_to_unicode
 from django_mailbox.tests.base import EmailMessageTestCase
+from django.test.utils import override_settings
 from django.utils.encoding import force_text
-from django.core.mail import EmailMessage
 
 
 __all__ = ['TestProcessEmail']
@@ -88,7 +89,7 @@ class TestProcessEmail(EmailMessageTestCase):
         )
 
     def test_message_with_not_decoded_attachment_header(self):
-        if sys.version_info > (3, 0):
+        if not six.PY3:
             self.skipTest(
                 "This test is only relevant on Python 2.x."
             )
@@ -145,14 +146,17 @@ class TestProcessEmail(EmailMessageTestCase):
             u'odpowied\u017a Burmistrza.jpg'
         )
 
+    @override_settings(
+        DJANGO_MAILBOX_DEFAULT_CHARSET='iso8859-2'
+    )
     def test_message_with_utf_headers_in_attachments(self):
         email_object = self._get_email_object(
-            'message_with_utf8_headers_in_attachment.eml',
+            'message_with_iso8859-2_attachment_filename.eml',
         )
         mailbox = Mailbox.objects.create()
         msg = mailbox.process_incoming_message(email_object)
 
-        expected_count = 2
+        expected_count = 1
         actual_count = msg.attachments.count()
 
         self.assertEqual(
@@ -160,7 +164,13 @@ class TestProcessEmail(EmailMessageTestCase):
             actual_count,
         )
 
-        # TODO: Verify filenames
+        expected_filename = (
+            u'Odp. na wniosek     o udost\u0119pnienie '
+            u'inf. publicznej.jpg'
+        )
+        actual_filename = msg.attachments.all()[0].get_filename()
+
+        self.assertEqual(expected_filename, actual_filename)
 
     def test_message_get_text_body(self):
         message = self._get_email_object('multipart_text.eml')

django_mailbox/utils.py

@@ -56,7 +56,7 @@ def get_settings():
         ),
         'default_charset': getattr(
             settings,
-            'DJANGO_MAILBOX_default_charset',
+            'DJANGO_MAILBOX_DEFAULT_CHARSET',
             'iso8859-1',
         )
     }