From 2df2d8a4fcf8db0a9e46ffcd0060ce748ca1d466 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 5 Mar 2017 21:31:22 -0800 Subject: [PATCH] [#130] Fix handling of single-byte filenames in attachment headers; among other unicode handling fixes. --- django_mailbox/models.py | 43 +++++++++++++++---- ...ge_with_iso8859-2_attachment_filename.eml} | 0 django_mailbox/tests/test_process_email.py | 20 ++++++--- django_mailbox/utils.py | 2 +- 4 files changed, 50 insertions(+), 15 deletions(-) rename django_mailbox/tests/messages/{message_with_utf8_headers_in_attachment.eml => message_with_iso8859-2_attachment_filename.eml} (100%) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 89eb0ad..0315272 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -642,8 +642,12 @@ class Message(models.Model): if they are encoded as such. """ + settings = utils.get_settings() + if self.encoded: - return base64.b64decode(self.body.encode('ascii')) + return base64.b64decode( + self.body.encode(settings['default_charset'], 'replace') + ) return self.body.encode('utf-8') def set_body(self, body): @@ -654,10 +658,16 @@ class Message(models.Model): no fields existed for storing arbitrary bytes. """ - if six.PY3: + settings = utils.get_settings() + + if six.PY3 and isinstance(body, six.text_type): body = body.encode('utf-8') + self.encoded = True - self.body = base64.b64encode(body).decode('ascii') + self.body = base64.b64encode(body).decode( + settings['default_charset'], + 'replace', + ) def get_email_object(self): """Returns an `email.message.Message` instance representing the @@ -726,18 +736,24 @@ class MessageAttachment(models.Model): return super(MessageAttachment, self).delete(*args, **kwargs) def _get_rehydrated_headers(self): + settings = utils.get_settings() + headers = self.headers if headers is None: return EmailMessage() - if sys.version_info < (3, 0): - try: - headers = headers.encode('utf-8') - except UnicodeDecodeError: - headers = unicode(headers, 'utf-8').encode('utf-8') + if not six.PY3: + headers = headers.encode(settings['default_charset'], 'replace') + return email.message_from_string(headers) def _set_dehydrated_headers(self, email_object): - self.headers = email_object.as_string() + settings = utils.get_settings() + + headers = email_object.as_string() + if isinstance(headers, six.binary_type): + headers = headers.decode(settings['default_charset'], 'replace') + + self.headers = headers def __delitem__(self, name): rehydrated = self._get_rehydrated_headers() @@ -745,7 +761,16 @@ class MessageAttachment(models.Model): self._set_dehydrated_headers(rehydrated) def __setitem__(self, name, value): + settings = utils.get_settings() + rehydrated = self._get_rehydrated_headers() + if isinstance(value, six.binary_type): + # Ensure that no values in the bytestring cannot be encoded + # in the default charset + value = value.decode( + settings['default_charset'], + 'replace' + ).encode(settings['default_charset'], 'replace') rehydrated[name] = value self._set_dehydrated_headers(rehydrated) diff --git a/django_mailbox/tests/messages/message_with_utf8_headers_in_attachment.eml b/django_mailbox/tests/messages/message_with_iso8859-2_attachment_filename.eml similarity index 100% rename from django_mailbox/tests/messages/message_with_utf8_headers_in_attachment.eml rename to django_mailbox/tests/messages/message_with_iso8859-2_attachment_filename.eml diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 707fdff..839d45f 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -4,11 +4,12 @@ import sys import mock import six +from django.core.mail import EmailMessage from django_mailbox.models import Mailbox, Message from django_mailbox.utils import convert_header_to_unicode from django_mailbox.tests.base import EmailMessageTestCase +from django.test.utils import override_settings from django.utils.encoding import force_text -from django.core.mail import EmailMessage __all__ = ['TestProcessEmail'] @@ -88,7 +89,7 @@ class TestProcessEmail(EmailMessageTestCase): ) def test_message_with_not_decoded_attachment_header(self): - if sys.version_info > (3, 0): + if not six.PY3: self.skipTest( "This test is only relevant on Python 2.x." ) @@ -145,14 +146,17 @@ class TestProcessEmail(EmailMessageTestCase): u'odpowied\u017a Burmistrza.jpg' ) + @override_settings( + DJANGO_MAILBOX_DEFAULT_CHARSET='iso8859-2' + ) def test_message_with_utf_headers_in_attachments(self): email_object = self._get_email_object( - 'message_with_utf8_headers_in_attachment.eml', + 'message_with_iso8859-2_attachment_filename.eml', ) mailbox = Mailbox.objects.create() msg = mailbox.process_incoming_message(email_object) - expected_count = 2 + expected_count = 1 actual_count = msg.attachments.count() self.assertEqual( @@ -160,7 +164,13 @@ class TestProcessEmail(EmailMessageTestCase): actual_count, ) - # TODO: Verify filenames + expected_filename = ( + u'Odp. na wniosek o udost\u0119pnienie ' + u'inf. publicznej.jpg' + ) + actual_filename = msg.attachments.all()[0].get_filename() + + self.assertEqual(expected_filename, actual_filename) def test_message_get_text_body(self): message = self._get_email_object('multipart_text.eml') diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index 3159c2c..ccca6ea 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -56,7 +56,7 @@ def get_settings(): ), 'default_charset': getattr( settings, - 'DJANGO_MAILBOX_default_charset', + 'DJANGO_MAILBOX_DEFAULT_CHARSET', 'iso8859-1', ) }