diff --git a/django_mailbox/admin.py b/django_mailbox/admin.py index ea989f1..69df8dc 100644 --- a/django_mailbox/admin.py +++ b/django_mailbox/admin.py @@ -5,7 +5,7 @@ from django.contrib import admin from django_mailbox.models import MessageAttachment, Message, Mailbox from django_mailbox.signals import message_received -from django_mailbox.utils import decode_header +from django_mailbox.utils import convert_header_to_unicode logger = logging.getLogger(__name__) @@ -51,7 +51,7 @@ class MessageAdmin(admin.ModelAdmin): return msg.attachments.count() def subject(self, msg): - return decode_header(msg.subject) + return convert_header_to_unicode(msg.subject) inlines = [ MessageAttachmentInline, diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 61e4260..28fafa7 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -1,6 +1,5 @@ import base64 import email -from email.header import decode_header from email.message import Message as EmailMessage from email.utils import formatdate, parseaddr from email.encoders import encode_base64 @@ -21,6 +20,8 @@ from django_mailbox.signals import message_received import six from six.moves.urllib.parse import unquote, urlparse +from .utils import convert_header_to_unicode + STRIP_UNALLOWED_MIMETYPES = getattr( settings, @@ -266,13 +267,13 @@ class Mailbox(models.Model): msg = Message() msg.mailbox = self if 'subject' in message: - msg.subject = message['subject'][0:255] + msg.subject = convert_header_to_unicode(message['subject'])[0:255] if 'message-id' in message: msg.message_id = message['message-id'][0:255] if 'from' in message: - msg.from_header = message['from'] + msg.from_header = convert_header_to_unicode(message['from']) if 'to' in message: - msg.to_header = message['to'] + msg.to_header = convert_header_to_unicode(message['to']) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) @@ -567,10 +568,7 @@ class MessageAttachment(models.Model): def get_filename(self): file_name = self._get_rehydrated_headers().get_filename() if file_name: - encoded_subject, encoding = decode_header(file_name)[0] - if encoding: - encoded_subject = encoded_subject.decode(encoding) - return encoded_subject + return convert_header_to_unicode(file_name) else: return None diff --git a/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml new file mode 100644 index 0000000..e6c96d8 --- /dev/null +++ b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml @@ -0,0 +1,67 @@ +Received: from mxfront40.mail.yandex.net ([127.0.0.1]) + by mxfront40.mail.yandex.net with LMTP id 1kYaj2Z0 + for ; Mon, 21 Apr 2014 17:01:46 +0400 +Received: from f284.i.mail.ru (f284.i.mail.ru [217.69.138.184]) + by mxfront40.mail.yandex.net (nwsmtp/Yandex) with ESMTPS id 5ybtvC8EjR-1k5qERXS; + Mon, 21 Apr 2014 17:01:46 +0400 + (using TLSv1 with cipher AES256-SHA (256/256 bits)) + (Client certificate not present) +X-Yandex-Front: mxfront40.mail.yandex.net +X-Yandex-TimeMark: 1398085306 +X-Yandex-Uniq: 048ae9ee-4930-499b-803d-4e7c7dd64da5 +Authentication-Results: mxfront40.mail.yandex.net; spf=pass (mxfront40.mail.yandex.net: domain of mail.ru designates 217.69.138.184 as permitted sender) smtp.mail=mr.test32@mail.ru; dkim=pass header.i=@mail.ru +X-Yandex-Spam: 1 +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=mail.ru; s=mail2; + h=Content-Type:Message-ID:Reply-To:Date:Mime-Version:Subject:To:From; bh=oJMg2fPrmKvsyMXib8I9eF7JpnqtQmXIK5Ve6pwFXSI=; + b=VG3S4nAtfFv+yQMf1Ubtb4y29mTOQL73foRmudvkAdAaPhRjFb+84OspikmmFKbK+jRfanFWkXjQWa9B72uK7d8F4NsO3e9a+MffXcMtsz9AgmFe72Af7sQrXiqkwmUGeG/TQtPjR5E/gi3IHFrHFQD+apER59c8AlqEoQLvrRg=; +Received: from mail by f284.i.mail.ru with local (envelope-from ) + id 1WcDr0-0006oa-11 + for mr.test32@yandex.ru; Mon, 21 Apr 2014 17:01:46 +0400 +Received: from [144.206.32.69] by e.mail.ru with HTTP; + Mon, 21 Apr 2014 17:01:45 +0400 +From: =?UTF-8?B?dGVzdCB0ZXN0?= +To: mr.test32@yandex.ru +Subject: Узнай как зарабатывать от 1000$ в неделю! + +Content-Type: text/html; + charset="iso-8859-1" +Content-Transfer-Encoding: 7bit +X-Spam_score: 7.1 +X-Spam_score_int: 71 +X-Spam_bar: +++++++ +X-Spam_report: Spam detection software, running on the system , has + identified this incoming email as possible spam. The original message + has been attached to this so you can view it (if it isn't spam) or label + similar future email. If you have any questions, see + The administrator of that system for details. + + Content preview: [...] + + Content analysis details: (7.1 points, 2.0 required) + + pts rule name description + ---- ---------------------- -------------------------------------------------- + 1.3 RCVD_IN_BL_SPAMCOP_NET RBL: Received via a relay in bl.spamcop.net + [Blocked - see ] + 1.4 RCVD_IN_BRBL_LASTEXT RBL: RCVD_IN_BRBL_LASTEXT + [201.216.250.71 listed in bb.barracudacentral.org] + 0.0 HTML_EXTRA_CLOSE BODY: HTML contains far too many close tags + 0.0 HTML_MESSAGE BODY: HTML included in message + 0.8 BAYES_50 BODY: Bayes spam probability is 40 to 60% + [score: 0.5000] + 0.7 MIME_HTML_ONLY BODY: Message only has text/html MIME parts + 1.2 HTML_IMAGE_ONLY_04 BODY: HTML: images with 0-400 bytes of words + 1.5 SUBJ_ILLEGAL_CHARS Subject: has too many raw illegal characters + 0.0 SUBJECT_NEEDS_ENCODING SUBJECT_NEEDS_ENCODING + 0.0 HTML_SHORT_LINK_IMG_1 HTML is very short with a linked image + 0.0 T_REMOTE_IMAGE Message contains an external image + + + + +

+ +

+ + diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 0dea582..b63b44d 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -1,4 +1,5 @@ import os.path +import sys import six @@ -174,3 +175,29 @@ class TestProcessEmail(EmailMessageTestCase): actual_body, expected_body, ) + + def test_message_with_single_byte_subject_encoding(self): + email_object = self._get_email_object( + 'message_with_single_byte_extended_subject_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(email_object) + + expected_subject = six.u( + '\u00D3\u00E7\u00ED\u00E0\u00E9 \u00EA\u00E0\u00EA ' + '\u00E7\u00E0\u00F0\u00E0\u00E1\u00E0\u00F2\u00FB\u00E2' + '\u00E0\u00F2\u00FC \u00EE\u00F2 1000$ \u00E2 ' + '\u00ED\u00E5\u00E4\u00E5\u00EB\u00FE!' + ) + actual_subject = msg.subject + self.assertEqual(actual_subject, expected_subject) + + if sys.version_info >= (3, 3): + # There were various bugfixes in Py3k's email module, + # this is apparently one of them. + expected_from = six.u('test test ') + else: + expected_from = six.u('test test') + actual_from = msg.from_header + + self.assertEqual(expected_from, actual_from) diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index b977ce6..7d884eb 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -1,6 +1,8 @@ import email.header import logging +import six + from django.conf import settings @@ -10,16 +12,24 @@ logger = logging.getLogger(__name__) DEFAULT_CHARSET = getattr( settings, 'DJANGO_MAILBOX_DEFAULT_CHARSET', - 'ascii', + 'iso8859-1', ) -def decode_header(header): +def convert_header_to_unicode(header): + def _decode(value, encoding): + if isinstance(value, six.text_type): + return value + if not encoding or encoding == 'unknown-8bit': + encoding = DEFAULT_CHARSET + return value.decode(encoding, 'REPLACE') + try: return ''.join( [ - unicode(t[0], t[1] or DEFAULT_CHARSET) - for t in email.header.decode_header(header) + ( + _decode(bytestr, encoding) + ) for bytestr, encoding in email.header.decode_header(header) ] ) except UnicodeDecodeError: diff --git a/setup.py b/setup.py index 11a180f..bbc0076 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ tests_require = [ setup( name='django-mailbox', - version='3.0.3', + version='3.1.1', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from '