Merge branch 'master' of github.com:coddingtonbear/django-mailbox

django_mailbox/admin.py

@@ -5,7 +5,7 @@ from django.contrib import admin
 
 from django_mailbox.models import MessageAttachment, Message, Mailbox
 from django_mailbox.signals import message_received
-from django_mailbox.utils import decode_header
+from django_mailbox.utils import convert_header_to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ class MessageAdmin(admin.ModelAdmin):
         return msg.attachments.count()
 
     def subject(self, msg):
-        return decode_header(msg.subject)
+        return convert_header_to_unicode(msg.subject)
 
     inlines = [
         MessageAttachmentInline,

django_mailbox/models.py

@@ -1,6 +1,5 @@
 import base64
 import email
-from email.header import decode_header
 from email.message import Message as EmailMessage
 from email.utils import formatdate, parseaddr
 from email.encoders import encode_base64
@@ -21,6 +20,8 @@ from django_mailbox.signals import message_received
 import six
 from six.moves.urllib.parse import unquote, urlparse
 
+from .utils import convert_header_to_unicode
+
 
 STRIP_UNALLOWED_MIMETYPES = getattr(
     settings,
@@ -266,13 +267,13 @@ class Mailbox(models.Model):
         msg = Message()
         msg.mailbox = self
         if 'subject' in message:
-            msg.subject = message['subject'][0:255]
+            msg.subject = convert_header_to_unicode(message['subject'])[0:255]
         if 'message-id' in message:
             msg.message_id = message['message-id'][0:255]
         if 'from' in message:
-            msg.from_header = message['from']
+            msg.from_header = convert_header_to_unicode(message['from'])
         if 'to' in message:
-            msg.to_header = message['to']
+            msg.to_header = convert_header_to_unicode(message['to'])
         msg.save()
         message = self._get_dehydrated_message(message, msg)
         msg.set_body(message.as_string())
@@ -567,10 +568,7 @@ class MessageAttachment(models.Model):
     def get_filename(self):
         file_name = self._get_rehydrated_headers().get_filename()
         if file_name:
-            encoded_subject, encoding = decode_header(file_name)[0]
-            if encoding:
-                encoded_subject = encoded_subject.decode(encoding)
-            return encoded_subject
+            return convert_header_to_unicode(file_name)
         else:
             return None
 

django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml

@@ -0,0 +1,67 @@
+Received: from mxfront40.mail.yandex.net ([127.0.0.1])
+    by mxfront40.mail.yandex.net with LMTP id 1kYaj2Z0
+    for <mr.test32@yandex.ru>; Mon, 21 Apr 2014 17:01:46 +0400
+Received: from f284.i.mail.ru (f284.i.mail.ru [217.69.138.184])
+    by mxfront40.mail.yandex.net (nwsmtp/Yandex) with ESMTPS id 5ybtvC8EjR-1k5qERXS;
+    Mon, 21 Apr 2014 17:01:46 +0400
+    (using TLSv1 with cipher AES256-SHA (256/256 bits))
+    (Client certificate not present)
+X-Yandex-Front: mxfront40.mail.yandex.net
+X-Yandex-TimeMark: 1398085306
+X-Yandex-Uniq: 048ae9ee-4930-499b-803d-4e7c7dd64da5
+Authentication-Results: mxfront40.mail.yandex.net; spf=pass (mxfront40.mail.yandex.net: domain of mail.ru designates 217.69.138.184 as permitted sender) smtp.mail=mr.test32@mail.ru; dkim=pass header.i=@mail.ru
+X-Yandex-Spam: 1
+DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=mail.ru; s=mail2;
+    h=Content-Type:Message-ID:Reply-To:Date:Mime-Version:Subject:To:From; bh=oJMg2fPrmKvsyMXib8I9eF7JpnqtQmXIK5Ve6pwFXSI=;
+    b=VG3S4nAtfFv+yQMf1Ubtb4y29mTOQL73foRmudvkAdAaPhRjFb+84OspikmmFKbK+jRfanFWkXjQWa9B72uK7d8F4NsO3e9a+MffXcMtsz9AgmFe72Af7sQrXiqkwmUGeG/TQtPjR5E/gi3IHFrHFQD+apER59c8AlqEoQLvrRg=;
+Received: from mail by f284.i.mail.ru with local (envelope-from <mr.test32@mail.ru>)
+    id 1WcDr0-0006oa-11
+    for mr.test32@yandex.ru; Mon, 21 Apr 2014 17:01:46 +0400
+Received: from [144.206.32.69] by e.mail.ru with HTTP;
+    Mon, 21 Apr 2014 17:01:45 +0400
+From: =?UTF-8?B?dGVzdCB0ZXN0?= <mr.test32@mail.ru>
+To: mr.test32@yandex.ru
+Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ!
+
+Content-Type: text/html;
+    charset="iso-8859-1"
+Content-Transfer-Encoding: 7bit
+X-Spam_score: 7.1
+X-Spam_score_int: 71
+X-Spam_bar: +++++++
+X-Spam_report: Spam detection software, running on the system , has
+ identified this incoming email as possible spam.  The original message
+ has been attached to this so you can view it (if it isn't spam) or label
+ similar future email.  If you have any questions, see
+ The administrator of that system for details.
+ 
+ Content preview:  [...] 
+ 
+ Content analysis details:   (7.1 points, 2.0 required)
+ 
+  pts rule name              description
+ ---- ---------------------- --------------------------------------------------
+  1.3 RCVD_IN_BL_SPAMCOP_NET RBL: Received via a relay in bl.spamcop.net
+               [Blocked - see <http://www.spamcop.net/bl.shtml?201.216.250.71>]
+  1.4 RCVD_IN_BRBL_LASTEXT   RBL: RCVD_IN_BRBL_LASTEXT
+                             [201.216.250.71 listed in bb.barracudacentral.org]
+  0.0 HTML_EXTRA_CLOSE       BODY: HTML contains far too many close tags
+  0.0 HTML_MESSAGE           BODY: HTML included in message
+  0.8 BAYES_50               BODY: Bayes spam probability is 40 to 60%
+                             [score: 0.5000]
+  0.7 MIME_HTML_ONLY         BODY: Message only has text/html MIME parts
+  1.2 HTML_IMAGE_ONLY_04     BODY: HTML: images with 0-400 bytes of words
+  1.5 SUBJ_ILLEGAL_CHARS     Subject: has too many raw illegal characters
+  0.0 SUBJECT_NEEDS_ENCODING SUBJECT_NEEDS_ENCODING
+  0.0 HTML_SHORT_LINK_IMG_1  HTML is very short with a linked image
+  0.0 T_REMOTE_IMAGE         Message contains an external image
+
+<html>
+<body>
+<tbody>
+</p>
+<a href="http://nemetica.pl/b.php"><img
+src="http://silamisli.yomu.ru/b.php?jpg"></a>
+</p>
+</body>
+</html>

django_mailbox/tests/test_process_email.py

@@ -1,4 +1,5 @@
 import os.path
+import sys
 
 import six
 
@@ -174,3 +175,29 @@ class TestProcessEmail(EmailMessageTestCase):
             actual_body,
             expected_body,
         )
+
+    def test_message_with_single_byte_subject_encoding(self):
+        email_object = self._get_email_object(
+            'message_with_single_byte_extended_subject_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(email_object)
+
+        expected_subject = six.u(
+            '\u00D3\u00E7\u00ED\u00E0\u00E9 \u00EA\u00E0\u00EA '
+            '\u00E7\u00E0\u00F0\u00E0\u00E1\u00E0\u00F2\u00FB\u00E2'
+            '\u00E0\u00F2\u00FC \u00EE\u00F2 1000$ \u00E2 '
+            '\u00ED\u00E5\u00E4\u00E5\u00EB\u00FE!'
+        )
+        actual_subject = msg.subject
+        self.assertEqual(actual_subject, expected_subject)
+
+        if sys.version_info >= (3, 3):
+            # There were various bugfixes in Py3k's email module,
+            # this is apparently one of them.
+            expected_from = six.u('test test <mr.test32@mail.ru>')
+        else:
+            expected_from = six.u('test test<mr.test32@mail.ru>')
+        actual_from = msg.from_header
+
+        self.assertEqual(expected_from, actual_from)

django_mailbox/utils.py

@@ -1,6 +1,8 @@
 import email.header
 import logging
 
+import six
+
 from django.conf import settings
 
 
@@ -10,16 +12,24 @@ logger = logging.getLogger(__name__)
 DEFAULT_CHARSET = getattr(
     settings,
     'DJANGO_MAILBOX_DEFAULT_CHARSET',
-    'ascii',
+    'iso8859-1',
 )
 
 
-def decode_header(header):
+def convert_header_to_unicode(header):
+    def _decode(value, encoding):
+        if isinstance(value, six.text_type):
+            return value
+        if not encoding or encoding == 'unknown-8bit':
+            encoding = DEFAULT_CHARSET
+        return value.decode(encoding, 'REPLACE')
+
     try:
         return ''.join(
             [
-                unicode(t[0], t[1] or DEFAULT_CHARSET)
-                for t in email.header.decode_header(header)
+                (
+                    _decode(bytestr, encoding)
+                ) for bytestr, encoding in email.header.decode_header(header)
             ]
         )
     except UnicodeDecodeError: