Always decode headers into unicode objects before assigning to model; fixes #12.

django_mailbox/admin.py

@@ -5,7 +5,7 @@ from django.contrib import admin
 
 from django_mailbox.models import MessageAttachment, Message, Mailbox
 from django_mailbox.signals import message_received
-from django_mailbox.utils import decode_header
+from django_mailbox.utils import convert_header_to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ class MessageAdmin(admin.ModelAdmin):
         return msg.attachments.count()
 
     def subject(self, msg):
-        return decode_header(msg.subject)
+        return convert_header_to_unicode(msg.subject)
 
     inlines = [
         MessageAttachmentInline,

django_mailbox/models.py

@@ -1,6 +1,5 @@
 import base64
 import email
-from email.header import decode_header
 from email.message import Message as EmailMessage
 from email.utils import formatdate, parseaddr
 from email.encoders import encode_base64
@@ -26,6 +25,8 @@ from django_mailbox.transports import Pop3Transport, ImapTransport,\
 from django_mailbox.signals import message_received
 import six
 
+from .utils import convert_header_to_unicode
+
 
 STRIP_UNALLOWED_MIMETYPES = getattr(
     settings,
@@ -256,13 +257,13 @@ class Mailbox(models.Model):
         msg = Message()
         msg.mailbox = self
         if 'subject' in message:
-            msg.subject = message['subject'][0:255]
+            msg.subject = convert_header_to_unicode(message['subject'][0:255])
         if 'message-id' in message:
             msg.message_id = message['message-id'][0:255]
         if 'from' in message:
-            msg.from_header = message['from']
+            msg.from_header = convert_header_to_unicode(message['from'])
         if 'to' in message:
-            msg.to_header = message['to']
+            msg.to_header = convert_header_to_unicode(message['to'])
         msg.save()
         message = self._get_dehydrated_message(message, msg)
         msg.set_body(message.as_string())
@@ -557,10 +558,7 @@ class MessageAttachment(models.Model):
     def get_filename(self):
         file_name = self._get_rehydrated_headers().get_filename()
         if file_name:
-            encoded_subject, encoding = decode_header(file_name)[0]
-            if encoding:
-                encoded_subject = encoded_subject.decode(encoding)
-            return encoded_subject
+            return convert_header_to_unicode(file_name)
         else:
             return None
 

django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml

@@ -21,7 +21,7 @@ Received: from [144.206.32.69] by e.mail.ru with HTTP;
     Mon, 21 Apr 2014 17:01:45 +0400
 From: =?UTF-8?B?dGVzdCB0ZXN0?= <mr.test32@mail.ru>
 To: mr.test32@yandex.ru
-Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ!
+Subject: Узнай как зарабатывать от 1000$ в неделю!
 
 Content-Type: text/html;
     charset="iso-8859-1"

django_mailbox/tests/test_process_email.py

@@ -174,3 +174,23 @@ class TestProcessEmail(EmailMessageTestCase):
             actual_body,
             expected_body,
         )
+
+    def test_message_with_single_byte_subject_encoding(self):
+        email_object = self._get_email_object(
+            'message_with_single_byte_extended_subject_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(email_object)
+
+        expected_subject = (
+            u'\xd3\xe7\xed\xe0\xe9 \xea\xe0\xea \xe7\xe0\xf0'
+            u'\xe0\xe1\xe0\xf2\xfb\xe2\xe0\xf2\xfc \xee\xf2 1000$ '
+            u'\xe2 \xed\xe5\xe4\xe5\xeb\xfe!'
+        )
+        actual_subject = msg.subject
+        self.assertEqual(actual_subject, expected_subject)
+
+        expected_from = u'test test<mr.test32@mail.ru>'
+        actual_from = msg.from_header
+
+        self.assertEqual(expected_from, actual_from)

django_mailbox/utils.py

@@ -10,16 +10,19 @@ logger = logging.getLogger(__name__)
 DEFAULT_CHARSET = getattr(
     settings,
     'DJANGO_MAILBOX_DEFAULT_CHARSET',
-    'ascii',
+    'iso8859-1',
 )
 
 
-def decode_header(header):
+def convert_header_to_unicode(header):
     try:
         return ''.join(
             [
-                unicode(t[0], t[1] or DEFAULT_CHARSET)
-                for t in email.header.decode_header(header)
+                (
+                    bytestr.decode(encoding, 'REPLACE')
+                    if encoding
+                    else bytestr.decode(DEFAULT_CHARSET, 'REPLACE')
+                ) for bytestr, encoding in email.header.decode_header(header)
             ]
         )
     except UnicodeDecodeError: