From d3a766ffd63da6372ea9b5beb408df266eceba9d Mon Sep 17 00:00:00 2001 From: dmippoltiov Date: Mon, 21 Apr 2014 17:40:40 +0400 Subject: [PATCH 1/7] Create test_UnicodeDecodeError.eml this message raise error on get_mail: Traceback: File "/usr/local/lib/python2.7/site-packages/django/core/handlers/base.py" in get_response 114. response = wrapped_callback(request, *callback_args, **callback_kwargs) File "/usr/local/lib/python2.7/site-packages/django/views/generic/base.py" in view 69. return self.dispatch(request, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/django/views/generic/base.py" in dispatch 87. return handler(request, *args, **kwargs) File "/home/leha/django-site/proj/views.py" in get 140. messages = mailbox.get_new_mail() File "/usr/local/lib/python2.7/site-packages/django_mailbox/models.py" in get_new_mail 285. msg = self.process_incoming_message(message) File "/usr/local/lib/python2.7/site-packages/django_mailbox/models.py" in process_incoming_message 183. msg = self._process_message(message) File "/usr/local/lib/python2.7/site-packages/django_mailbox/models.py" in _process_message 266. msg.save() File "/usr/local/lib/python2.7/site-packages/django/db/models/base.py" in save 545. force_update=force_update, update_fields=update_fields) File "/usr/local/lib/python2.7/site-packages/django/db/models/base.py" in save_base 573. updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields) File "/usr/local/lib/python2.7/site-packages/django/db/models/base.py" in _save_table 654. result = self._do_insert(cls._base_manager, using, fields, update_pk, raw) File "/usr/local/lib/python2.7/site-packages/django/db/models/base.py" in _do_insert 687. using=using, raw=raw) File "/usr/local/lib/python2.7/site-packages/django/db/models/manager.py" in _insert 232. return insert_query(self.model, objs, fields, **kwargs) File "/usr/local/lib/python2.7/site-packages/django/db/models/query.py" in insert_query 1511. return query.get_compiler(using=using).execute_sql(return_id) File "/usr/local/lib/python2.7/site-packages/django/db/models/sql/compiler.py" in execute_sql 898. cursor.execute(sql, params) File "/usr/local/lib/python2.7/site-packages/django/db/backends/util.py" in execute 73. sql = self.db.ops.last_executed_query(self.cursor, sql, params) File "/usr/local/lib/python2.7/site-packages/django/db/backends/postgresql_psycopg2/operations.py" in last_executed_query 214. return cursor.query.decode('utf-8') File "/usr/local/lib/python2.7/encodings/utf_8.py" in decode 16. return codecs.utf_8_decode(input, errors, True) Exception Type: UnicodeDecodeError at /proj/new_mail/ Exception Value: 'utf8' codec can't decode byte 0xd3 in position 187: invalid continuation byte --- .../messages/test_UnicodeDecodeError.eml | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 django_mailbox/tests/messages/test_UnicodeDecodeError.eml diff --git a/django_mailbox/tests/messages/test_UnicodeDecodeError.eml b/django_mailbox/tests/messages/test_UnicodeDecodeError.eml new file mode 100644 index 0000000..f699a20 --- /dev/null +++ b/django_mailbox/tests/messages/test_UnicodeDecodeError.eml @@ -0,0 +1,67 @@ +Received: from mxfront40.mail.yandex.net ([127.0.0.1]) + by mxfront40.mail.yandex.net with LMTP id 1kYaj2Z0 + for ; Mon, 21 Apr 2014 17:01:46 +0400 +Received: from f284.i.mail.ru (f284.i.mail.ru [217.69.138.184]) + by mxfront40.mail.yandex.net (nwsmtp/Yandex) with ESMTPS id 5ybtvC8EjR-1k5qERXS; + Mon, 21 Apr 2014 17:01:46 +0400 + (using TLSv1 with cipher AES256-SHA (256/256 bits)) + (Client certificate not present) +X-Yandex-Front: mxfront40.mail.yandex.net +X-Yandex-TimeMark: 1398085306 +X-Yandex-Uniq: 048ae9ee-4930-499b-803d-4e7c7dd64da5 +Authentication-Results: mxfront40.mail.yandex.net; spf=pass (mxfront40.mail.yandex.net: domain of mail.ru designates 217.69.138.184 as permitted sender) smtp.mail=mr.test32@mail.ru; dkim=pass header.i=@mail.ru +X-Yandex-Spam: 1 +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=mail.ru; s=mail2; + h=Content-Type:Message-ID:Reply-To:Date:Mime-Version:Subject:To:From; bh=oJMg2fPrmKvsyMXib8I9eF7JpnqtQmXIK5Ve6pwFXSI=; + b=VG3S4nAtfFv+yQMf1Ubtb4y29mTOQL73foRmudvkAdAaPhRjFb+84OspikmmFKbK+jRfanFWkXjQWa9B72uK7d8F4NsO3e9a+MffXcMtsz9AgmFe72Af7sQrXiqkwmUGeG/TQtPjR5E/gi3IHFrHFQD+apER59c8AlqEoQLvrRg=; +Received: from mail by f284.i.mail.ru with local (envelope-from ) + id 1WcDr0-0006oa-11 + for mr.test32@yandex.ru; Mon, 21 Apr 2014 17:01:46 +0400 +Received: from [144.206.32.69] by e.mail.ru with HTTP; + Mon, 21 Apr 2014 17:01:45 +0400 +From: =?UTF-8?B?dGVzdCB0ZXN0?= +To: mr.test32@yandex.ru +Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ! + +Content-Type: text/html; + charset="iso-8859-1" +Content-Transfer-Encoding: 7bit +X-Spam_score: 7.1 +X-Spam_score_int: 71 +X-Spam_bar: +++++++ +X-Spam_report: Spam detection software, running on the system , has + identified this incoming email as possible spam. The original message + has been attached to this so you can view it (if it isn't spam) or label + similar future email. If you have any questions, see + The administrator of that system for details. + + Content preview: [...] + + Content analysis details: (7.1 points, 2.0 required) + + pts rule name description + ---- ---------------------- -------------------------------------------------- + 1.3 RCVD_IN_BL_SPAMCOP_NET RBL: Received via a relay in bl.spamcop.net + [Blocked - see ] + 1.4 RCVD_IN_BRBL_LASTEXT RBL: RCVD_IN_BRBL_LASTEXT + [201.216.250.71 listed in bb.barracudacentral.org] + 0.0 HTML_EXTRA_CLOSE BODY: HTML contains far too many close tags + 0.0 HTML_MESSAGE BODY: HTML included in message + 0.8 BAYES_50 BODY: Bayes spam probability is 40 to 60% + [score: 0.5000] + 0.7 MIME_HTML_ONLY BODY: Message only has text/html MIME parts + 1.2 HTML_IMAGE_ONLY_04 BODY: HTML: images with 0-400 bytes of words + 1.5 SUBJ_ILLEGAL_CHARS Subject: has too many raw illegal characters + 0.0 SUBJECT_NEEDS_ENCODING SUBJECT_NEEDS_ENCODING + 0.0 HTML_SHORT_LINK_IMG_1 HTML is very short with a linked image + 0.0 T_REMOTE_IMAGE Message contains an external image + + + + +

+ +

+ + From 79e189fb2283f301eac1c3d37f39221db3936a81 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 15:49:49 -0700 Subject: [PATCH 2/7] Always decode headers into unicode objects before assigning to model; fixes #12. --- django_mailbox/admin.py | 4 ++-- django_mailbox/models.py | 14 ++++++------- ...single_byte_extended_subject_encoding.eml} | 2 +- django_mailbox/tests/test_process_email.py | 20 +++++++++++++++++++ django_mailbox/utils.py | 11 ++++++---- 5 files changed, 36 insertions(+), 15 deletions(-) rename django_mailbox/tests/messages/{test_UnicodeDecodeError.eml => message_with_single_byte_extended_subject_encoding.eml} (97%) diff --git a/django_mailbox/admin.py b/django_mailbox/admin.py index ea989f1..69df8dc 100644 --- a/django_mailbox/admin.py +++ b/django_mailbox/admin.py @@ -5,7 +5,7 @@ from django.contrib import admin from django_mailbox.models import MessageAttachment, Message, Mailbox from django_mailbox.signals import message_received -from django_mailbox.utils import decode_header +from django_mailbox.utils import convert_header_to_unicode logger = logging.getLogger(__name__) @@ -51,7 +51,7 @@ class MessageAdmin(admin.ModelAdmin): return msg.attachments.count() def subject(self, msg): - return decode_header(msg.subject) + return convert_header_to_unicode(msg.subject) inlines = [ MessageAttachmentInline, diff --git a/django_mailbox/models.py b/django_mailbox/models.py index e604caa..0d9c709 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -1,6 +1,5 @@ import base64 import email -from email.header import decode_header from email.message import Message as EmailMessage from email.utils import formatdate, parseaddr from email.encoders import encode_base64 @@ -26,6 +25,8 @@ from django_mailbox.transports import Pop3Transport, ImapTransport,\ from django_mailbox.signals import message_received import six +from .utils import convert_header_to_unicode + STRIP_UNALLOWED_MIMETYPES = getattr( settings, @@ -256,13 +257,13 @@ class Mailbox(models.Model): msg = Message() msg.mailbox = self if 'subject' in message: - msg.subject = message['subject'][0:255] + msg.subject = convert_header_to_unicode(message['subject'][0:255]) if 'message-id' in message: msg.message_id = message['message-id'][0:255] if 'from' in message: - msg.from_header = message['from'] + msg.from_header = convert_header_to_unicode(message['from']) if 'to' in message: - msg.to_header = message['to'] + msg.to_header = convert_header_to_unicode(message['to']) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) @@ -557,10 +558,7 @@ class MessageAttachment(models.Model): def get_filename(self): file_name = self._get_rehydrated_headers().get_filename() if file_name: - encoded_subject, encoding = decode_header(file_name)[0] - if encoding: - encoded_subject = encoded_subject.decode(encoding) - return encoded_subject + return convert_header_to_unicode(file_name) else: return None diff --git a/django_mailbox/tests/messages/test_UnicodeDecodeError.eml b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml similarity index 97% rename from django_mailbox/tests/messages/test_UnicodeDecodeError.eml rename to django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml index f699a20..e6c96d8 100644 --- a/django_mailbox/tests/messages/test_UnicodeDecodeError.eml +++ b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml @@ -21,7 +21,7 @@ Received: from [144.206.32.69] by e.mail.ru with HTTP; Mon, 21 Apr 2014 17:01:45 +0400 From: =?UTF-8?B?dGVzdCB0ZXN0?= To: mr.test32@yandex.ru -Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ! +Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ! Content-Type: text/html; charset="iso-8859-1" diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 0dea582..2d4b3a0 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -174,3 +174,23 @@ class TestProcessEmail(EmailMessageTestCase): actual_body, expected_body, ) + + def test_message_with_single_byte_subject_encoding(self): + email_object = self._get_email_object( + 'message_with_single_byte_extended_subject_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(email_object) + + expected_subject = ( + u'\xd3\xe7\xed\xe0\xe9 \xea\xe0\xea \xe7\xe0\xf0' + u'\xe0\xe1\xe0\xf2\xfb\xe2\xe0\xf2\xfc \xee\xf2 1000$ ' + u'\xe2 \xed\xe5\xe4\xe5\xeb\xfe!' + ) + actual_subject = msg.subject + self.assertEqual(actual_subject, expected_subject) + + expected_from = u'test test' + actual_from = msg.from_header + + self.assertEqual(expected_from, actual_from) diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index b977ce6..fb25c55 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -10,16 +10,19 @@ logger = logging.getLogger(__name__) DEFAULT_CHARSET = getattr( settings, 'DJANGO_MAILBOX_DEFAULT_CHARSET', - 'ascii', + 'iso8859-1', ) -def decode_header(header): +def convert_header_to_unicode(header): try: return ''.join( [ - unicode(t[0], t[1] or DEFAULT_CHARSET) - for t in email.header.decode_header(header) + ( + bytestr.decode(encoding, 'REPLACE') + if encoding + else bytestr.decode(DEFAULT_CHARSET, 'REPLACE') + ) for bytestr, encoding in email.header.decode_header(header) ] ) except UnicodeDecodeError: From 79493cb1bc0112dd629bf9a941918a817815f055 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 15:50:46 -0700 Subject: [PATCH 3/7] Bumping version number to 3.1. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a7557e6..decf8cb 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ tests_require = [ setup( name='django-mailbox', - version='3.0.3', + version='3.1', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From d871d4144816e35bd25ee353fa5133346cbce210 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 16:06:29 -0700 Subject: [PATCH 4/7] Fix test for Py3 support. --- django_mailbox/tests/test_process_email.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 2d4b3a0..77b89ba 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -182,10 +182,11 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) - expected_subject = ( - u'\xd3\xe7\xed\xe0\xe9 \xea\xe0\xea \xe7\xe0\xf0' - u'\xe0\xe1\xe0\xf2\xfb\xe2\xe0\xf2\xfc \xee\xf2 1000$ ' - u'\xe2 \xed\xe5\xe4\xe5\xeb\xfe!' + expected_subject = six.u( + '\u00D3\u00E7\u00ED\u00E0\u00E9 \u00EA\u00E0\u00EA ' + '\u00E7\u00E0\u00F0\u00E0\u00E1\u00E0\u00F2\u00FB\u00E2' + '\u00E0\u00F2\u00FC \u00EE\u00F2 1000$ \u00E2 ' + '\u00ED\u00E5\u00E4\u00E5\u00EB\u00FE!' ) actual_subject = msg.subject self.assertEqual(actual_subject, expected_subject) From ef42c319e9596ec37bf9a0554665de71627b7765 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 16:24:16 -0700 Subject: [PATCH 5/7] Updating yet another string for python3. --- django_mailbox/tests/test_process_email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 77b89ba..d3f15bc 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -191,7 +191,7 @@ class TestProcessEmail(EmailMessageTestCase): actual_subject = msg.subject self.assertEqual(actual_subject, expected_subject) - expected_from = u'test test' + expected_from = six.u('test test') actual_from = msg.from_header self.assertEqual(expected_from, actual_from) From a2cf80ef99fa3e4b63b9d7b5ce02aee0564254e0 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 17:06:17 -0700 Subject: [PATCH 6/7] Fixing header decoding such that it works properly under both Python 3.x and 2.x. --- django_mailbox/models.py | 2 +- django_mailbox/tests/test_process_email.py | 7 ++++++- django_mailbox/utils.py | 13 ++++++++++--- setup.py | 2 +- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 0d9c709..f6e938b 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -257,7 +257,7 @@ class Mailbox(models.Model): msg = Message() msg.mailbox = self if 'subject' in message: - msg.subject = convert_header_to_unicode(message['subject'][0:255]) + msg.subject = convert_header_to_unicode(message['subject'])[0:255] if 'message-id' in message: msg.message_id = message['message-id'][0:255] if 'from' in message: diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index d3f15bc..1f2213c 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -191,7 +191,12 @@ class TestProcessEmail(EmailMessageTestCase): actual_subject = msg.subject self.assertEqual(actual_subject, expected_subject) - expected_from = six.u('test test') + if six.PY3: + # There were various bugfixes in Py3k's email module, + # this is apparently one of them. + expected_from = six.u('test test ') + else: + expected_from = six.u('test test') actual_from = msg.from_header self.assertEqual(expected_from, actual_from) diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index fb25c55..7d884eb 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -1,6 +1,8 @@ import email.header import logging +import six + from django.conf import settings @@ -15,13 +17,18 @@ DEFAULT_CHARSET = getattr( def convert_header_to_unicode(header): + def _decode(value, encoding): + if isinstance(value, six.text_type): + return value + if not encoding or encoding == 'unknown-8bit': + encoding = DEFAULT_CHARSET + return value.decode(encoding, 'REPLACE') + try: return ''.join( [ ( - bytestr.decode(encoding, 'REPLACE') - if encoding - else bytestr.decode(DEFAULT_CHARSET, 'REPLACE') + _decode(bytestr, encoding) ) for bytestr, encoding in email.header.decode_header(header) ] ) diff --git a/setup.py b/setup.py index decf8cb..81b7b61 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ tests_require = [ setup( name='django-mailbox', - version='3.1', + version='3.1.1', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 0f0628a7425fa57019e7ba1d8caba49ba7e08458 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 17:13:41 -0700 Subject: [PATCH 7/7] Only expect additional space in Python 3.3+. --- django_mailbox/tests/test_process_email.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 1f2213c..b63b44d 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -1,4 +1,5 @@ import os.path +import sys import six @@ -191,7 +192,7 @@ class TestProcessEmail(EmailMessageTestCase): actual_subject = msg.subject self.assertEqual(actual_subject, expected_subject) - if six.PY3: + if sys.version_info >= (3, 3): # There were various bugfixes in Py3k's email module, # this is apparently one of them. expected_from = six.u('test test ')