From e4a8a05f3d21c8381f14acb1fdf18576d45bb4d4 Mon Sep 17 00:00:00 2001 From: Lars Kruse Date: Tue, 9 May 2017 23:28:19 +0200 Subject: [PATCH 1/6] Fix handling of missing 'content-transfer-encoding' header Closes: #136 Python's email module raises a KeyError if the header 'content-transfer-encoding' is missing: https://bugs.python.org/issue27321 --- django_mailbox/models.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 89eb0ad..db88535 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -225,6 +225,8 @@ class Mailbox(models.Model): def process_incoming_message(self, message): """Process a message incoming to this mailbox.""" msg = self._process_message(message) + if msg is None: + return None msg.outgoing = False msg.save() @@ -235,6 +237,8 @@ class Mailbox(models.Model): def record_outgoing_message(self, message): """Record an outgoing message associated with this mailbox.""" msg = self._process_message(message) + if msg is None: + return None msg.outgoing = True msg.save() return msg @@ -363,7 +367,14 @@ class Mailbox(models.Model): ) msg.save() message = self._get_dehydrated_message(message, msg) - msg.set_body(message.as_string()) + try: + body = message.as_string() + except KeyError as exc: + # email.message.replace_header may raise 'KeyError' if the header + # 'content-transfer-encoding' is missing + logger.warning("Failed to parse message: %s", exc,) + return None + msg.set_body(body) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( @@ -382,7 +393,8 @@ class Mailbox(models.Model): return new_mail for message in connection.get_message(condition): msg = self.process_incoming_message(message) - new_mail.append(msg) + if not msg is None: + new_mail.append(msg) self.last_polling = now() if django.VERSION >= (1, 5): # Django 1.5 introduces update_fields self.save(update_fields=['last_polling']) From 234ed009096fd3c4d0e8445e221fb3909a647a3c Mon Sep 17 00:00:00 2001 From: Lars Kruse Date: Tue, 9 May 2017 23:59:42 +0200 Subject: [PATCH 2/6] Add flake8 style check --- .flake8 | 2 ++ django_mailbox/admin.py | 3 +++ .../management/commands/rebuildmessageattachments.py | 8 ++++---- django_mailbox/models.py | 2 +- django_mailbox/tests/base.py | 2 +- django_mailbox/tests/test_process_email.py | 10 ++++++---- django_mailbox/tests/test_transports.py | 8 ++++++-- django_mailbox/transports/__init__.py | 2 ++ django_mailbox/transports/base.py | 4 ++-- django_mailbox/utils.py | 2 +- 10 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..1f20aa3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +exclude = django_mailbox/migrations, django_mailbox/south_migrations diff --git a/django_mailbox/admin.py b/django_mailbox/admin.py index 6bfebd4..25eaa71 100644 --- a/django_mailbox/admin.py +++ b/django_mailbox/admin.py @@ -23,6 +23,8 @@ def get_new_mail(mailbox_admin, request, queryset): for mailbox in queryset.all(): logger.debug('Receiving mail for %s' % mailbox) mailbox.get_new_mail() + + get_new_mail.short_description = 'Get new mail' @@ -103,6 +105,7 @@ class MessageAdmin(admin.ModelAdmin): ) actions = [resend_message_received_signal] + if getattr(settings, 'DJANGO_MAILBOX_ADMIN_ENABLED', True): admin.site.register(Message, MessageAdmin) admin.site.register(MessageAttachment, MessageAttachmentAdmin) diff --git a/django_mailbox/management/commands/rebuildmessageattachments.py b/django_mailbox/management/commands/rebuildmessageattachments.py index 3bd2eb3..6bfc24f 100644 --- a/django_mailbox/management/commands/rebuildmessageattachments.py +++ b/django_mailbox/management/commands/rebuildmessageattachments.py @@ -23,7 +23,7 @@ class Command(BaseCommand): """ def handle(self, *args, **options): - ATTACHMENT_HASH_MAP = {} + attachment_hash_map = {} attachments_without_messages = MessageAttachment.objects.filter( message=None @@ -38,7 +38,7 @@ class Command(BaseCommand): md5 = hashlib.md5() for chunk in attachment.document.file.chunks(): md5.update(chunk) - ATTACHMENT_HASH_MAP[md5.hexdigest()] = attachment.pk + attachment_hash_map[md5.hexdigest()] = attachment.pk for message_record in Message.objects.all().order_by('id'): message = email.message_from_string(message_record.body) @@ -51,9 +51,9 @@ class Command(BaseCommand): md5 = hashlib.md5() md5.update(part.get_payload(decode=True)) digest = md5.hexdigest() - if digest in ATTACHMENT_HASH_MAP: + if digest in attachment_hash_map: attachment = MessageAttachment.objects.get( - pk=ATTACHMENT_HASH_MAP[digest] + pk=attachment_hash_map[digest] ) attachment.message = message_record attachment.save() diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 89eb0ad..2e6a90c 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -733,7 +733,7 @@ class MessageAttachment(models.Model): try: headers = headers.encode('utf-8') except UnicodeDecodeError: - headers = unicode(headers, 'utf-8').encode('utf-8') + headers = headers.decode('utf-8').encode('utf-8') return email.message_from_string(headers) def _set_dehydrated_headers(self, email_object): diff --git a/django_mailbox/tests/base.py b/django_mailbox/tests/base.py index 0f5516c..5553aa1 100644 --- a/django_mailbox/tests/base.py +++ b/django_mailbox/tests/base.py @@ -171,7 +171,7 @@ class EmailMessageTestCase(TestCase): ) ) - def assertEqual(self, left, right): + def assertEqual(self, left, right): # noqa: N802 if not isinstance(left, email.message.Message): return super(EmailMessageTestCase, self).assertEqual(left, right) return self.compare_email_objects(left, right) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index f265bdc..de123c2 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -213,7 +213,7 @@ class TestProcessEmail(EmailMessageTestCase): success = True try: - msg = self.mailbox.process_incoming_message(email_object) + self.mailbox.process_incoming_message(email_object) except ValueError: success = False @@ -232,7 +232,8 @@ class TestProcessEmail(EmailMessageTestCase): # this is the code of _process_message() msg = Message() # if STORE_ORIGINAL_MESSAGE: - # msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message), save=False) + # msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message), + # save=False) msg.mailbox = self.mailbox if 'subject' in message: msg.subject = convert_header_to_unicode(message['subject'])[0:255] @@ -246,7 +247,7 @@ class TestProcessEmail(EmailMessageTestCase): msg.to_header = convert_header_to_unicode(message['Delivered-To']) msg.save() - #here the message is ok + # here the message is ok str_msg = message.as_string() message = self.mailbox._get_dehydrated_message(message, msg) try: @@ -358,7 +359,8 @@ class TestProcessEmail(EmailMessageTestCase): self.assertEqual(expected_from, actual_from) def test_message_reply(self): - email_object = EmailMessage('Test subject', # subject + email_object = EmailMessage( + 'Test subject', # subject 'Test body', # body 'username@example.com', # from ['mr.test32@mail.ru'], # to diff --git a/django_mailbox/tests/test_transports.py b/django_mailbox/tests/test_transports.py index aaa0d0d..1d3c68f 100644 --- a/django_mailbox/tests/test_transports.py +++ b/django_mailbox/tests/test_transports.py @@ -59,6 +59,7 @@ class IMAPTestCase(EmailMessageTestCase): return FAKE_UID_FETCH_SIZES if arg2 == '(RFC822)': return FAKE_UID_FETCH_MSG + def imap_server_list_method(pattern=None): return FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS @@ -88,6 +89,7 @@ class TestImapTransport(IMAPTestCase): expected_message = self._get_email_object('generic_message.eml') self.assertEqual(expected_message, actual_message) + class TestImapArchivedTransport(TestImapTransport): def setUp(self): super(TestImapArchivedTransport, self).setUp() @@ -100,6 +102,7 @@ class TestImapArchivedTransport(TestImapTransport): ) self.transport.server = self.imap_server + class TestMaxSizeImapTransport(TestImapTransport): @override_settings(DJANGO_MAILBOX_MAX_MESSAGE_SIZE=5807) @@ -115,7 +118,9 @@ class TestMaxSizeImapTransport(TestImapTransport): def test_size_limit(self): all_message_ids = self.transport._get_all_message_ids() - small_message_ids = self.transport._get_small_message_ids(all_message_ids) + small_message_ids = self.transport._get_small_message_ids( + all_message_ids, + ) self.assertEqual(len(small_message_ids), 1) def test_get_email_message(self): @@ -163,4 +168,3 @@ class TestPop3Transport(EmailMessageTestCase): expected_message = self._get_email_object('generic_message.eml') self.assertEqual(expected_message, actual_message) - diff --git a/django_mailbox/transports/__init__.py b/django_mailbox/transports/__init__.py index 03b31f0..7aeef95 100644 --- a/django_mailbox/transports/__init__.py +++ b/django_mailbox/transports/__init__.py @@ -1,3 +1,5 @@ +# all imports below are only used by external modules +# flake8: noqa from django_mailbox.transports.imap import ImapTransport from django_mailbox.transports.pop3 import Pop3Transport from django_mailbox.transports.maildir import MaildirTransport diff --git a/django_mailbox/transports/base.py b/django_mailbox/transports/base.py index 753df52..d65ad35 100644 --- a/django_mailbox/transports/base.py +++ b/django_mailbox/transports/base.py @@ -4,9 +4,9 @@ import six # Do *not* remove this, we need to use this in subclasses of EmailTransport if six.PY3: - from email.errors import MessageParseError + from email.errors import MessageParseError # noqa: F401 else: - from email.Errors import MessageParseError + from email.Errors import MessageParseError # noqa: F401 class EmailTransport(object): diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index 3159c2c..17a4b37 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -89,7 +89,7 @@ def convert_header_to_unicode(header): header, default_charset, ) - return unicode(header, default_charset, 'replace') + return header.decode(default_charset, 'replace') def get_body_from_message(message, maintype, subtype): From 9df9f0aee1ef770815f8b2e44f9ce95b29d074e3 Mon Sep 17 00:00:00 2001 From: Lars Kruse Date: Wed, 10 May 2017 00:07:05 +0200 Subject: [PATCH 3/6] Run 'flake8' tests via tox --- tox.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8acbaf8..4255817 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py34,py35 +envlist=py27,py34,py35,flake8 [testenv] deps= @@ -8,3 +8,7 @@ deps= sitepackages=False commands= py.test {posargs} + +[testenv:flake8] +deps=flake8 +commands=flake8 django_mailbox From 21036d73c439835b6589010896160a512736a621 Mon Sep 17 00:00:00 2001 From: Adam Dobrawy Date: Thu, 22 Jun 2017 02:52:58 +0200 Subject: [PATCH 4/6] Add missing classifiers in setup.py --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.py b/setup.py index e429a1c..f6d6feb 100755 --- a/setup.py +++ b/setup.py @@ -31,6 +31,14 @@ setup( 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', + 'Framework :: Django', + 'Framework :: Django :: 1.4', + 'Framework :: Django :: 1.5', + 'Framework :: Django :: 1.6', + 'Framework :: Django :: 1.7', + 'Framework :: Django :: 1.8', + 'Framework :: Django :: 1.9', + 'Framework :: Django :: 1.10', 'Programming Language :: Python', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', From 8f63751f133fca96a204cd014d4e0560a857ebe0 Mon Sep 17 00:00:00 2001 From: Ivlev Denis Date: Sat, 8 Jul 2017 15:01:55 +0300 Subject: [PATCH 5/6] Add python3.6 & django1.11+ to tests --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5ae7dfa..9312422 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - '2.7' - '3.4' - '3.5' +- '3.6' env: matrix: - DJANGO='>=1.4,<1.5' @@ -10,6 +11,7 @@ env: - DJANGO='>=1.8,<1.9' - DJANGO='>=1.9,<1.10' - DJANGO='>=1.10,<1.11' + - DJANGO='>=1.11,<2.0' global: - secure: aswHU7pQroGM+GHoYlhXzzk2FYfqxXJORjqXPsbgoHAIu4Bssaj8+YAzIcdy3j9kSt4I8VBpjnn8H/wzQXki75JBZOosQrIeMK018+uR+ZMONBLqDYW/N7EJHLgZt9QXxQNKeZygrD4GN/Dc4gLHGvPQC/RfPuuHcnF0Liaahoo= - secure: RZ6M6984P885GRoyx9n/WCCWGoFAzYpS8sZkXu3e/HK9oPXfaM2IEHjkq03jIC/FcWn/QMtFjOUBqQU94rnqdivFdFkeZHk1WUQgC0hztH3Qhh9zu2PNIrYUDpVD5dJqBpprWbSwFP8yNsJlP9A2RUubTlZblKHuaBhhiuNN+kU= @@ -23,10 +25,14 @@ matrix: python: '3.4' - env: DJANGO='>=1.4,<1.5' python: '3.5' + - env: DJANGO='>=1.4,<1.5' + python: '3.6' - env: DJANGO='>=1.7,<1.8' python: '2.6' - env: DJANGO='>=1.7,<1.8' python: '3.5' + - env: DJANGO='>=1.7,<1.8' + python: '3.6' - env: DJANGO='>=1.8,<1.9' python: '2.6' - env: DJANGO='>=1.9,<1.10' From fc9a1aaa11fb45ba1acf898a351455beb13e5066 Mon Sep 17 00:00:00 2001 From: Adam Dobrawy Date: Sat, 8 Jul 2017 14:15:13 +0200 Subject: [PATCH 6/6] Update django and python version in classifiers --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index f6d6feb..e445b4a 100755 --- a/setup.py +++ b/setup.py @@ -39,10 +39,14 @@ setup( 'Framework :: Django :: 1.8', 'Framework :: Django :: 1.9', 'Framework :: Django :: 1.10', + 'Framework :: Django :: 1.11', 'Programming Language :: Python', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Communications :: Email', 'Topic :: Communications :: Email :: Post-Office', 'Topic :: Communications :: Email :: Post-Office :: IMAP',