From 79e189fb2283f301eac1c3d37f39221db3936a81 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 15:49:49 -0700 Subject: [PATCH 01/86] Always decode headers into unicode objects before assigning to model; fixes #12. --- django_mailbox/admin.py | 4 ++-- django_mailbox/models.py | 14 ++++++------- ...single_byte_extended_subject_encoding.eml} | 2 +- django_mailbox/tests/test_process_email.py | 20 +++++++++++++++++++ django_mailbox/utils.py | 11 ++++++---- 5 files changed, 36 insertions(+), 15 deletions(-) rename django_mailbox/tests/messages/{test_UnicodeDecodeError.eml => message_with_single_byte_extended_subject_encoding.eml} (97%) diff --git a/django_mailbox/admin.py b/django_mailbox/admin.py index ea989f1..69df8dc 100644 --- a/django_mailbox/admin.py +++ b/django_mailbox/admin.py @@ -5,7 +5,7 @@ from django.contrib import admin from django_mailbox.models import MessageAttachment, Message, Mailbox from django_mailbox.signals import message_received -from django_mailbox.utils import decode_header +from django_mailbox.utils import convert_header_to_unicode logger = logging.getLogger(__name__) @@ -51,7 +51,7 @@ class MessageAdmin(admin.ModelAdmin): return msg.attachments.count() def subject(self, msg): - return decode_header(msg.subject) + return convert_header_to_unicode(msg.subject) inlines = [ MessageAttachmentInline, diff --git a/django_mailbox/models.py b/django_mailbox/models.py index e604caa..0d9c709 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -1,6 +1,5 @@ import base64 import email -from email.header import decode_header from email.message import Message as EmailMessage from email.utils import formatdate, parseaddr from email.encoders import encode_base64 @@ -26,6 +25,8 @@ from django_mailbox.transports import Pop3Transport, ImapTransport,\ from django_mailbox.signals import message_received import six +from .utils import convert_header_to_unicode + STRIP_UNALLOWED_MIMETYPES = getattr( settings, @@ -256,13 +257,13 @@ class Mailbox(models.Model): msg = Message() msg.mailbox = self if 'subject' in message: - msg.subject = message['subject'][0:255] + msg.subject = convert_header_to_unicode(message['subject'][0:255]) if 'message-id' in message: msg.message_id = message['message-id'][0:255] if 'from' in message: - msg.from_header = message['from'] + msg.from_header = convert_header_to_unicode(message['from']) if 'to' in message: - msg.to_header = message['to'] + msg.to_header = convert_header_to_unicode(message['to']) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) @@ -557,10 +558,7 @@ class MessageAttachment(models.Model): def get_filename(self): file_name = self._get_rehydrated_headers().get_filename() if file_name: - encoded_subject, encoding = decode_header(file_name)[0] - if encoding: - encoded_subject = encoded_subject.decode(encoding) - return encoded_subject + return convert_header_to_unicode(file_name) else: return None diff --git a/django_mailbox/tests/messages/test_UnicodeDecodeError.eml b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml similarity index 97% rename from django_mailbox/tests/messages/test_UnicodeDecodeError.eml rename to django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml index f699a20..e6c96d8 100644 --- a/django_mailbox/tests/messages/test_UnicodeDecodeError.eml +++ b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml @@ -21,7 +21,7 @@ Received: from [144.206.32.69] by e.mail.ru with HTTP; Mon, 21 Apr 2014 17:01:45 +0400 From: =?UTF-8?B?dGVzdCB0ZXN0?= To: mr.test32@yandex.ru -Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ! +Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ! Content-Type: text/html; charset="iso-8859-1" diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 0dea582..2d4b3a0 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -174,3 +174,23 @@ class TestProcessEmail(EmailMessageTestCase): actual_body, expected_body, ) + + def test_message_with_single_byte_subject_encoding(self): + email_object = self._get_email_object( + 'message_with_single_byte_extended_subject_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(email_object) + + expected_subject = ( + u'\xd3\xe7\xed\xe0\xe9 \xea\xe0\xea \xe7\xe0\xf0' + u'\xe0\xe1\xe0\xf2\xfb\xe2\xe0\xf2\xfc \xee\xf2 1000$ ' + u'\xe2 \xed\xe5\xe4\xe5\xeb\xfe!' + ) + actual_subject = msg.subject + self.assertEqual(actual_subject, expected_subject) + + expected_from = u'test test' + actual_from = msg.from_header + + self.assertEqual(expected_from, actual_from) diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index b977ce6..fb25c55 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -10,16 +10,19 @@ logger = logging.getLogger(__name__) DEFAULT_CHARSET = getattr( settings, 'DJANGO_MAILBOX_DEFAULT_CHARSET', - 'ascii', + 'iso8859-1', ) -def decode_header(header): +def convert_header_to_unicode(header): try: return ''.join( [ - unicode(t[0], t[1] or DEFAULT_CHARSET) - for t in email.header.decode_header(header) + ( + bytestr.decode(encoding, 'REPLACE') + if encoding + else bytestr.decode(DEFAULT_CHARSET, 'REPLACE') + ) for bytestr, encoding in email.header.decode_header(header) ] ) except UnicodeDecodeError: From 79493cb1bc0112dd629bf9a941918a817815f055 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 15:50:46 -0700 Subject: [PATCH 02/86] Bumping version number to 3.1. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a7557e6..decf8cb 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ tests_require = [ setup( name='django-mailbox', - version='3.0.3', + version='3.1', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From d871d4144816e35bd25ee353fa5133346cbce210 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 16:06:29 -0700 Subject: [PATCH 03/86] Fix test for Py3 support. --- django_mailbox/tests/test_process_email.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 2d4b3a0..77b89ba 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -182,10 +182,11 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) - expected_subject = ( - u'\xd3\xe7\xed\xe0\xe9 \xea\xe0\xea \xe7\xe0\xf0' - u'\xe0\xe1\xe0\xf2\xfb\xe2\xe0\xf2\xfc \xee\xf2 1000$ ' - u'\xe2 \xed\xe5\xe4\xe5\xeb\xfe!' + expected_subject = six.u( + '\u00D3\u00E7\u00ED\u00E0\u00E9 \u00EA\u00E0\u00EA ' + '\u00E7\u00E0\u00F0\u00E0\u00E1\u00E0\u00F2\u00FB\u00E2' + '\u00E0\u00F2\u00FC \u00EE\u00F2 1000$ \u00E2 ' + '\u00ED\u00E5\u00E4\u00E5\u00EB\u00FE!' ) actual_subject = msg.subject self.assertEqual(actual_subject, expected_subject) From ef42c319e9596ec37bf9a0554665de71627b7765 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 16:24:16 -0700 Subject: [PATCH 04/86] Updating yet another string for python3. --- django_mailbox/tests/test_process_email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 77b89ba..d3f15bc 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -191,7 +191,7 @@ class TestProcessEmail(EmailMessageTestCase): actual_subject = msg.subject self.assertEqual(actual_subject, expected_subject) - expected_from = u'test test' + expected_from = six.u('test test') actual_from = msg.from_header self.assertEqual(expected_from, actual_from) From a2cf80ef99fa3e4b63b9d7b5ce02aee0564254e0 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 17:06:17 -0700 Subject: [PATCH 05/86] Fixing header decoding such that it works properly under both Python 3.x and 2.x. --- django_mailbox/models.py | 2 +- django_mailbox/tests/test_process_email.py | 7 ++++++- django_mailbox/utils.py | 13 ++++++++++--- setup.py | 2 +- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 0d9c709..f6e938b 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -257,7 +257,7 @@ class Mailbox(models.Model): msg = Message() msg.mailbox = self if 'subject' in message: - msg.subject = convert_header_to_unicode(message['subject'][0:255]) + msg.subject = convert_header_to_unicode(message['subject'])[0:255] if 'message-id' in message: msg.message_id = message['message-id'][0:255] if 'from' in message: diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index d3f15bc..1f2213c 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -191,7 +191,12 @@ class TestProcessEmail(EmailMessageTestCase): actual_subject = msg.subject self.assertEqual(actual_subject, expected_subject) - expected_from = six.u('test test') + if six.PY3: + # There were various bugfixes in Py3k's email module, + # this is apparently one of them. + expected_from = six.u('test test ') + else: + expected_from = six.u('test test') actual_from = msg.from_header self.assertEqual(expected_from, actual_from) diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index fb25c55..7d884eb 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -1,6 +1,8 @@ import email.header import logging +import six + from django.conf import settings @@ -15,13 +17,18 @@ DEFAULT_CHARSET = getattr( def convert_header_to_unicode(header): + def _decode(value, encoding): + if isinstance(value, six.text_type): + return value + if not encoding or encoding == 'unknown-8bit': + encoding = DEFAULT_CHARSET + return value.decode(encoding, 'REPLACE') + try: return ''.join( [ ( - bytestr.decode(encoding, 'REPLACE') - if encoding - else bytestr.decode(DEFAULT_CHARSET, 'REPLACE') + _decode(bytestr, encoding) ) for bytestr, encoding in email.header.decode_header(header) ] ) diff --git a/setup.py b/setup.py index decf8cb..81b7b61 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ tests_require = [ setup( name='django-mailbox', - version='3.1', + version='3.1.1', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 0f0628a7425fa57019e7ba1d8caba49ba7e08458 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 22 Apr 2014 17:13:41 -0700 Subject: [PATCH 06/86] Only expect additional space in Python 3.3+. --- django_mailbox/tests/test_process_email.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 1f2213c..b63b44d 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -1,4 +1,5 @@ import os.path +import sys import six @@ -191,7 +192,7 @@ class TestProcessEmail(EmailMessageTestCase): actual_subject = msg.subject self.assertEqual(actual_subject, expected_subject) - if six.PY3: + if sys.version_info >= (3, 3): # There were various bugfixes in Py3k's email module, # this is apparently one of them. expected_from = six.u('test test ') From ff68e564d5a0d0f58b3750f40f8548fa7fe33db4 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Thu, 24 Apr 2014 06:32:55 -0700 Subject: [PATCH 07/86] Ignore virtualenv. --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 0d20b64..d3e19c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ *.pyc +bin/* +lib/* +src/* +include/* +.Python From 002e33bd95e78aedd7e60a9239ee799a7410cc64 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Thu, 24 Apr 2014 15:28:54 -0700 Subject: [PATCH 08/86] Refactoring imports for Python3 support. Fixes #13. * Refactoring unicode payload handling of mis-encoded payloads. * Uses six.moves.urllib to find proper urllib versions. --- .travis.yml | 5 +-- django_mailbox/models.py | 36 ++++++++++++------- ..._invalid_content_for_declared_encoding.eml | 2 +- setup.py | 2 +- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e88b63..5cf4dc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,10 @@ python: - "2.6" - "2.7" - "3.2" + - "3.3" env: - - DJANGO=1.5.5 - - DJANGO=1.6.1 + - DJANGO=1.5.6 + - DJANGO=1.6.3 install: - pip install -q Django==$DJANGO --use-mirrors - pip install -q -e . --use-mirrors diff --git a/django_mailbox/models.py b/django_mailbox/models.py index e604caa..61e4260 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -4,18 +4,12 @@ from email.header import decode_header from email.message import Message as EmailMessage from email.utils import formatdate, parseaddr from email.encoders import encode_base64 -import urllib import mimetypes import os.path from quopri import encode as encode_quopri import sys import uuid -try: - import urllib.parse as urlparse -except ImportError: - import urlparse - from django.conf import settings from django.core.mail.message import make_msgid from django.core.files.base import ContentFile @@ -25,6 +19,7 @@ from django_mailbox.transports import Pop3Transport, ImapTransport,\ MMDFTransport from django_mailbox.signals import message_received import six +from six.moves.urllib.parse import unquote, urlparse STRIP_UNALLOWED_MIMETYPES = getattr( @@ -117,7 +112,7 @@ class Mailbox(models.Model): @property def _protocol_info(self): - return urlparse.urlparse(self.uri) + return urlparse(self.uri) @property def _domain(self): @@ -129,11 +124,11 @@ class Mailbox(models.Model): @property def username(self): - return urllib.unquote(self._protocol_info.username) + return unquote(self._protocol_info.username) @property def password(self): - return urllib.unquote(self._protocol_info.password) + return unquote(self._protocol_info.password) @property def location(self): @@ -249,6 +244,21 @@ class Mailbox(models.Model): placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk) new = placeholder else: + content_charset = msg.get_content_charset() + if not content_charset: + content_charset = 'ascii' + try: + # Make sure that the payload can be properly decoded in the + # defined charset, if it can't, let's mash some things + # inside the payload :-\ + msg.get_payload(decode=True).decode(content_charset) + except UnicodeDecodeError: + msg.set_payload( + msg.get_payload(decode=True).decode( + content_charset, + 'ignore' + ) + ) new = msg return new @@ -495,7 +505,7 @@ class Message(models.Model): return self.body.encode('utf-8') def set_body(self, body): - if sys.version_info >= (3, 0): + if six.PY3: body = body.encode('utf-8') self.encoded = True self.body = base64.b64encode(body).decode('ascii') @@ -503,10 +513,10 @@ class Message(models.Model): def get_email_object(self): """ Returns an `email.message.Message` instance for this message.""" body = self.get_body() - if sys.version_info < (3, 0): - flat = email.message_from_string(body) - else: + if six.PY3: flat = email.message_from_bytes(body) + else: + flat = email.message_from_string(body) return self._rehydrate(flat) def delete(self, *args, **kwargs): diff --git a/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml b/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml index 8860e9d..ea0d874 100644 --- a/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml +++ b/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml @@ -1,5 +1,5 @@ Date: Fri, 30 Jan 2012 11:30:01 PST -Subject: Tjest +Subject: Tjest Wrong From: "Somebody" To: idontknow@somewhere.com MIME-Version: 1.0 diff --git a/setup.py b/setup.py index a7557e6..11a180f 100755 --- a/setup.py +++ b/setup.py @@ -41,6 +41,6 @@ setup( 'django_mailbox.tests', ], install_requires=[ - 'six' + 'six>=1.6.1' ] ) From 4002d5c222ef2e6077c855063a7842af8b7323ea Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Thu, 24 Apr 2014 15:49:49 -0700 Subject: [PATCH 09/86] Bumping version number. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bbc0076..12aa67f 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ tests_require = [ setup( name='django-mailbox', - version='3.1.1', + version='3.1.2', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 3b2d60a903caa645d6a7d255f9f3b166cb7bd345 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 25 Apr 2014 16:37:51 -0700 Subject: [PATCH 10/86] Actually fixing Python 3 support; verified both IMAP and POP3. Fixes #13 (seriously this time). --- .gitignore | 2 + django_mailbox/tests/__init__.py | 1 + django_mailbox/tests/base.py | 17 +++-- django_mailbox/tests/test_transports.py | 87 +++++++++++++++++++++++++ django_mailbox/transports/base.py | 19 ++++++ django_mailbox/transports/generic.py | 5 +- django_mailbox/transports/imap.py | 9 +-- django_mailbox/transports/pop3.py | 20 ++++-- setup.py | 1 + 9 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 django_mailbox/tests/test_transports.py create mode 100644 django_mailbox/transports/base.py diff --git a/.gitignore b/.gitignore index d3e19c6..db89701 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,7 @@ bin/* lib/* src/* +dist/* include/* .Python +*egg* diff --git a/django_mailbox/tests/__init__.py b/django_mailbox/tests/__init__.py index 42d5abe..b83434c 100644 --- a/django_mailbox/tests/__init__.py +++ b/django_mailbox/tests/__init__.py @@ -1,3 +1,4 @@ from .test_mailbox import * from .test_message_flattening import * from .test_process_email import * +from .test_transports import * diff --git a/django_mailbox/tests/base.py b/django_mailbox/tests/base.py index b062797..a559b9e 100644 --- a/django_mailbox/tests/base.py +++ b/django_mailbox/tests/base.py @@ -1,8 +1,9 @@ import email import os.path +import six + from django.test import TestCase -import sys from django_mailbox import models from django_mailbox.models import Mailbox, Message @@ -22,7 +23,7 @@ class EmailMessageTestCase(TestCase): self.mailbox = Mailbox.objects.create() super(EmailMessageTestCase, self).setUp() - def _get_email_object(self, name): + def _get_email_as_text(self, name): with open( os.path.join( os.path.dirname(__file__), @@ -31,10 +32,14 @@ class EmailMessageTestCase(TestCase): ), 'rb' ) as f: - if sys.version_info < (3, 0): - return email.message_from_string(f.read()) - else: - return email.message_from_bytes(f.read()) + return f.read() + + def _get_email_object(self, name): + copy = self._get_email_as_text(name) + if six.PY3: + return email.message_from_bytes(copy) + else: + return email.message_from_string(copy) def _headers_identical(self, left, right, header=None): """ Check if headers are (close enough to) identical. diff --git a/django_mailbox/tests/test_transports.py b/django_mailbox/tests/test_transports.py new file mode 100644 index 0000000..b731748 --- /dev/null +++ b/django_mailbox/tests/test_transports.py @@ -0,0 +1,87 @@ +import mock +import six + +from django_mailbox.tests.base import EmailMessageTestCase +from django_mailbox.transports import ImapTransport, Pop3Transport + + +class TestImapTransport(EmailMessageTestCase): + def setUp(self): + self.arbitrary_hostname = 'one.two.three' + self.arbitrary_port = 100 + self.ssl = False + self.transport = ImapTransport( + self.arbitrary_hostname, + self.arbitrary_port, + self.ssl + ) + self.transport.server = None + super(TestImapTransport, self).setUp() + + def test_get_email_message(self): + with mock.patch.object(self.transport, 'server') as server: + server.search.return_value = ( + 'OK', + [ + 'One', # This is totally arbitrary + ] + ) + server.fetch.return_value = ( + 'OK', + [ + [ + '1 (RFC822 {8190}', # Wat? + self._get_email_as_text('generic_message.eml') + ], + ')', + ] + ) + + actual_messages = list(self.transport.get_message()) + + self.assertEqual(len(actual_messages), 1) + + actual_message = actual_messages[0] + expected_message = self._get_email_object('generic_message.eml') + + self.assertEqual(expected_message, actual_message) + + + +class TestPop3Transport(EmailMessageTestCase): + def setUp(self): + self.arbitrary_hostname = 'one.two.three' + self.arbitrary_port = 100 + self.ssl = False + self.transport = Pop3Transport( + self.arbitrary_hostname, + self.arbitrary_port, + self.ssl + ) + self.transport.server = None + super(TestPop3Transport, self).setUp() + + def test_get_email_message(self): + with mock.patch.object(self.transport, 'server') as server: + # Consider this value arbitrary, the second parameter + # should have one entry per message in the inbox + server.list.return_value = [None, ['some_msg']] + server.retr.return_value = [ + '+OK message follows', + [ + line.encode('ascii') + for line in self._get_email_as_text( + 'generic_message.eml' + ).decode('ascii').split('\n') + ], + 10018, # Some arbitrary size, ideally matching the above + ] + + actual_messages = list(self.transport.get_message()) + + self.assertEqual(len(actual_messages), 1) + + actual_message = actual_messages[0] + expected_message = self._get_email_object('generic_message.eml') + + self.assertEqual(expected_message, actual_message) diff --git a/django_mailbox/transports/base.py b/django_mailbox/transports/base.py new file mode 100644 index 0000000..753df52 --- /dev/null +++ b/django_mailbox/transports/base.py @@ -0,0 +1,19 @@ +import email + +import six + +# Do *not* remove this, we need to use this in subclasses of EmailTransport +if six.PY3: + from email.errors import MessageParseError +else: + from email.Errors import MessageParseError + + +class EmailTransport(object): + def get_email_from_bytes(self, contents): + if six.PY3: + message = email.message_from_bytes(contents) + else: + message = email.message_from_string(contents) + + return message diff --git a/django_mailbox/transports/generic.py b/django_mailbox/transports/generic.py index 059d078..5a9562d 100644 --- a/django_mailbox/transports/generic.py +++ b/django_mailbox/transports/generic.py @@ -1,4 +1,7 @@ -class GenericFileMailbox(object): +from .base import EmailTransport + + +class GenericFileMailbox(EmailTransport): _variant = None _path = None diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index 2ba3db2..ff3074c 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -1,8 +1,9 @@ -import email from imaplib import IMAP4, IMAP4_SSL +from .base import EmailTransport, MessageParseError -class ImapTransport(object): + +class ImapTransport(EmailTransport): def __init__(self, hostname, port=None, ssl=False): self.hostname = hostname self.port = port @@ -29,9 +30,9 @@ class ImapTransport(object): for key in inbox[0].split(): try: typ, msg_contents = self.server.fetch(key, '(RFC822)') - message = email.message_from_string(msg_contents[0][1]) + message = self.get_email_from_bytes(msg_contents[0][1]) yield message - except email.Errors.MessageParseError: + except MessageParseError: continue self.server.store(key, "+FLAGS", "\\Deleted") self.server.expunge() diff --git a/django_mailbox/transports/pop3.py b/django_mailbox/transports/pop3.py index 652fbe7..8520557 100644 --- a/django_mailbox/transports/pop3.py +++ b/django_mailbox/transports/pop3.py @@ -1,8 +1,11 @@ -import email +import six + from poplib import POP3, POP3_SSL +from .base import EmailTransport, MessageParseError -class Pop3Transport(object): + +class Pop3Transport(EmailTransport): def __init__(self, hostname, port=None, ssl=False): self.hostname = hostname self.port = port @@ -20,14 +23,21 @@ class Pop3Transport(object): self.server.user(username) self.server.pass_(password) + def get_message_body(self, message_lines): + if six.PY3: + return six.binary_type('\r\n', 'ascii').join(message_lines) + return '\r\n'.join(message_lines) + def get_message(self): message_count = len(self.server.list()[1]) for i in range(message_count): try: - msg_contents = "\r\n".join(self.server.retr(i + 1)[1]) - message = email.message_from_string(msg_contents) + msg_contents = self.get_message_body( + self.server.retr(i + 1)[1] + ) + message = self.get_email_from_bytes(msg_contents) yield message - except email.Errors.MessageParseError: + except MessageParseError: continue self.server.dele(i + 1) self.server.quit() diff --git a/setup.py b/setup.py index 12aa67f..836014e 100755 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ from setuptools import setup tests_require = [ 'django', + 'mock', ] setup( From 872f4786c790dfa8344665336590fdd890dff775 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 25 Apr 2014 16:50:44 -0700 Subject: [PATCH 11/86] Actually fixing Python 3 support; thanks for pointing out that it was broken, @greendee! --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 836014e..7b082c5 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ tests_require = [ setup( name='django-mailbox', - version='3.1.2', + version='3.2', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 0e0c6cf170150b6784b2bf1a48c3452684b33aa5 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 23 May 2014 09:17:14 +0100 Subject: [PATCH 12/86] Added email archiving option to IMAP transport --- django_mailbox/models.py | 7 ++++++- django_mailbox/tests/test_transports.py | 17 ++++++++++++++-- django_mailbox/transports/imap.py | 12 ++++++++++- docs/topics/mailbox_types.rst | 27 ++++++++++++++++--------- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 28fafa7..761f76b 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -146,6 +146,10 @@ class Mailbox(models.Model): def use_ssl(self): return '+ssl' in self._protocol_info.scheme.lower() + @property + def archive(self): + return self._protocol_info.query + def get_connection(self): if not self.uri: return None @@ -153,7 +157,8 @@ class Mailbox(models.Model): conn = ImapTransport( self.location, port=self.port if self.port else None, - ssl=self.use_ssl + ssl=self.use_ssl, + archive=self.archive ) conn.connect(self.username, self.password) elif self.type == 'pop3': diff --git a/django_mailbox/tests/test_transports.py b/django_mailbox/tests/test_transports.py index b731748..132d3f7 100644 --- a/django_mailbox/tests/test_transports.py +++ b/django_mailbox/tests/test_transports.py @@ -10,10 +10,12 @@ class TestImapTransport(EmailMessageTestCase): self.arbitrary_hostname = 'one.two.three' self.arbitrary_port = 100 self.ssl = False + self.archive = 'Archive' self.transport = ImapTransport( self.arbitrary_hostname, self.arbitrary_port, - self.ssl + self.ssl, + self.archive ) self.transport.server = None super(TestImapTransport, self).setUp() @@ -36,7 +38,18 @@ class TestImapTransport(EmailMessageTestCase): ')', ] ) - + server.list.return_value = ( + 'OK', + [ + '(\\HasNoChildren) "/" "Archive"' + ] + ) + server.copy.return_value = ( + 'OK', + [ + '[COPYUID 1 2 2] (Success)' + ] + ) actual_messages = list(self.transport.get_message()) self.assertEqual(len(actual_messages), 1) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index ff3074c..01325f8 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -4,9 +4,10 @@ from .base import EmailTransport, MessageParseError class ImapTransport(EmailTransport): - def __init__(self, hostname, port=None, ssl=False): + def __init__(self, hostname, port=None, ssl=False, archive=''): self.hostname = hostname self.port = port + self.archive = archive if ssl: self.transport = IMAP4_SSL if not self.port: @@ -27,6 +28,11 @@ class ImapTransport(EmailTransport): if not inbox[0]: return + if self.archive: + typ, folders = self.server.list(pattern=self.archive) + if folders[0] == None: + self.archive = False + for key in inbox[0].split(): try: typ, msg_contents = self.server.fetch(key, '(RFC822)') @@ -34,6 +40,10 @@ class ImapTransport(EmailTransport): yield message except MessageParseError: continue + + if self.archive: + self.server.copy(key, self.archive) + self.server.store(key, "+FLAGS", "\\Deleted") self.server.expunge() return diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index 0e60bae..ee53616 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -7,24 +7,25 @@ POP3 and IMAP as well as local file-based mailboxes. .. table:: 'Protocol' Options - ============ ============== =============================================== + ============ ============== =============================================================== Mailbox Type 'Protocol':// Notes - ============ ============== =============================================== + ============ ============== =============================================================== POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` - IMAP ``imap://`` Can also specify SSL with ``imap+ssl://`` + IMAP ``imap://`` Can also specify SSL and archive with ``imap+ssl://?myarchive`` Maildir ``maildir://`` Mbox ``mbox://`` Babyl ``babyl://`` MH ``mh://`` MMDF ``mmdf://`` Piped Mail *empty* See :ref:`receiving-mail-from-exim4-or-postfix` - ============ ============== =============================================== + ============ ============== =============================================================== .. warning:: - This will delete any messages it can find in the inbox you specify; + This will delete or move any messages it can find in the inbox you specify; do not use an e-mail inbox that you would like to share between - applications. + applications. For the IMAP protocol you can specify an archive folder where + emails are moved to. POP3 and IMAP Mailboxes @@ -44,11 +45,17 @@ Also, if your username or password include any non-ascii characters, they should be URL-encoded (for example, if your username includes an ``@``, it should be changed to ``%40`` in your URI). -If you have an account named ``youremailaddress@gmail.com`` with a password -of ``1234`` on GMail, which uses a POP3 server of 'pop.gmail.com' and requires -SSL, you would enter the following as your URI:: +If you are using an IMAP Mailbox, you can archive all messages before they +are deleted from the inbox. To archive emails, add the archive folder +name as a queryparemeter to the uri (for example, if your mailbox has a +folder called myarchive, add ``?myarchive`` to the uri). - pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com +If you have an account named ``youremailaddress@gmail.com`` with a password +of ``1234`` on GMail, which uses a IMAP server of 'imap.gmail.com', requires +SSL and you want to archive all emails, you would enter the following as +your URI:: + + pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com?[GMAIL]/All Mail Local File-based Mailboxes From 02567acdc21847bd661e9b8cb090983d9430ec4a Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 25 May 2014 12:35:20 -0700 Subject: [PATCH 13/86] Minor alterations to @yellowcap's pull request submission: * Using a query parameter argument (rather than the entire query string) to specify the folder into which e-mail messages should be archived. * Duplicating the IMAP tests; this could probably be a bit more simply done, but we'll at least be verifying both archived and non-archived use cases. * Simplifying some aspects of the documentation to include references to this new feature. --- django_mailbox/models.py | 11 +++++-- django_mailbox/tests/test_transports.py | 43 ++++++++++++++++++++++++- docs/topics/mailbox_types.rst | 34 ++++++++++--------- 3 files changed, 70 insertions(+), 18 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 761f76b..f4b5dc6 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -18,7 +18,7 @@ from django_mailbox.transports import Pop3Transport, ImapTransport,\ MMDFTransport from django_mailbox.signals import message_received import six -from six.moves.urllib.parse import unquote, urlparse +from six.moves.urllib.parse import parse_qs, unquote, urlparse from .utils import convert_header_to_unicode @@ -115,6 +115,10 @@ class Mailbox(models.Model): def _protocol_info(self): return urlparse(self.uri) + @property + def _query_string(self): + return parse_qs(self._protocol_info.query) + @property def _domain(self): return self._protocol_info.hostname @@ -148,7 +152,10 @@ class Mailbox(models.Model): @property def archive(self): - return self._protocol_info.query + archive_folder = self._query_string.get('archive', None) + if not archive_folder: + return None + return archive_folder[0] def get_connection(self): if not self.uri: diff --git a/django_mailbox/tests/test_transports.py b/django_mailbox/tests/test_transports.py index 132d3f7..7f044d4 100644 --- a/django_mailbox/tests/test_transports.py +++ b/django_mailbox/tests/test_transports.py @@ -6,6 +6,47 @@ from django_mailbox.transports import ImapTransport, Pop3Transport class TestImapTransport(EmailMessageTestCase): + def setUp(self): + self.arbitrary_hostname = 'one.two.three' + self.arbitrary_port = 100 + self.ssl = False + self.transport = ImapTransport( + self.arbitrary_hostname, + self.arbitrary_port, + self.ssl + ) + self.transport.server = None + super(TestImapTransport, self).setUp() + + def test_get_email_message(self): + with mock.patch.object(self.transport, 'server') as server: + server.search.return_value = ( + 'OK', + [ + 'One', # This is totally arbitrary + ] + ) + server.fetch.return_value = ( + 'OK', + [ + [ + '1 (RFC822 {8190}', # Wat? + self._get_email_as_text('generic_message.eml') + ], + ')', + ] + ) + actual_messages = list(self.transport.get_message()) + + self.assertEqual(len(actual_messages), 1) + + actual_message = actual_messages[0] + expected_message = self._get_email_object('generic_message.eml') + + self.assertEqual(expected_message, actual_message) + + +class TestImapArchivedTransport(EmailMessageTestCase): def setUp(self): self.arbitrary_hostname = 'one.two.three' self.arbitrary_port = 100 @@ -18,7 +59,7 @@ class TestImapTransport(EmailMessageTestCase): self.archive ) self.transport.server = None - super(TestImapTransport, self).setUp() + super(TestImapArchivedTransport, self).setUp() def test_get_email_message(self): with mock.patch.object(self.transport, 'server') as server: diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index ee53616..fce8461 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -7,25 +7,26 @@ POP3 and IMAP as well as local file-based mailboxes. .. table:: 'Protocol' Options - ============ ============== =============================================================== + ============ ============== ================================================================================================================================================================= Mailbox Type 'Protocol':// Notes - ============ ============== =============================================================== + ============ ============== ================================================================================================================================================================= POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` - IMAP ``imap://`` Can also specify SSL and archive with ``imap+ssl://?myarchive`` + IMAP ``imap://`` Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI. Maildir ``maildir://`` Mbox ``mbox://`` Babyl ``babyl://`` MH ``mh://`` MMDF ``mmdf://`` Piped Mail *empty* See :ref:`receiving-mail-from-exim4-or-postfix` - ============ ============== =============================================================== + ============ ============== ================================================================================================================================================================= + .. warning:: - This will delete or move any messages it can find in the inbox you specify; + Unless you are using IMAP's 'Archive' feature, + this will delete any messages it can find in the inbox you specify; do not use an e-mail inbox that you would like to share between - applications. For the IMAP protocol you can specify an archive folder where - emails are moved to. + applications. POP3 and IMAP Mailboxes @@ -40,22 +41,25 @@ Basic IMAP Example: ``imap://username:password@server`` Basic POP3 Example: ``pop3://username:password@server`` Most mailboxes these days are SSL-enabled; -if yours is, add ``+ssl`` to your URI. +if yours is, add ``+ssl`` to the protocol section of your URI. Also, if your username or password include any non-ascii characters, they should be URL-encoded (for example, if your username includes an ``@``, it should be changed to ``%40`` in your URI). If you are using an IMAP Mailbox, you can archive all messages before they are deleted from the inbox. To archive emails, add the archive folder -name as a queryparemeter to the uri (for example, if your mailbox has a -folder called myarchive, add ``?myarchive`` to the uri). +name as a query paremeter to the uri. For example, if your mailbox has a +folder named ``myarchivefolder`` that you would like to copy messages to +after processing, add ``?archive=myarchivefolder`` to the end of the URI. -If you have an account named ``youremailaddress@gmail.com`` with a password -of ``1234`` on GMail, which uses a IMAP server of 'imap.gmail.com', requires -SSL and you want to archive all emails, you would enter the following as -your URI:: +For a verbose example, if you have an account named +``youremailaddress@gmail.com`` with a password +of ``1234`` on GMail, which uses a IMAP server of ``imap.gmail.com`` (requiring +SSL) and you would like to archive all processed emails +into a folder named ``Archived``, you +would enter the following as your URI:: - pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com?[GMAIL]/All Mail + pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com?archive=Archived Local File-based Mailboxes From bf2f0c89d45f8930d181c4d264df66842a07badc Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 25 May 2014 12:37:46 -0700 Subject: [PATCH 14/86] Bumping version number to 3.3. --- docs/conf.py | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 1ed33fb..aa866ac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ master_doc = 'index' # General information about the project. project = u'django-mailbox' -copyright = u'2013, Adam Coddington' +copyright = u'2014, Adam Coddington' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '3.0' +version = '3.3' # The full version, including alpha/beta/rc tags. -release = '3.0' +release = '3.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 7b082c5..c080a93 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ tests_require = [ setup( name='django-mailbox', - version='3.2', + version='3.3', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 0aa757955d631df9fb8e6cbe3e372dcae56e2255 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 25 May 2014 12:57:38 -0700 Subject: [PATCH 15/86] Create archive folder if it does not exist. --- django_mailbox/transports/imap.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index 01325f8..078392e 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -30,8 +30,9 @@ class ImapTransport(EmailTransport): if self.archive: typ, folders = self.server.list(pattern=self.archive) - if folders[0] == None: - self.archive = False + if folders[0] is None: + # If the archive folder does not exist, create it + self.server.create(self.archive) for key in inbox[0].split(): try: From 6336e910bd241196e9827c1d0bda1cc528821d09 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 26 May 2014 10:25:55 -0700 Subject: [PATCH 16/86] Fixing a documentation glitch; thanks for catching this, @yellowcap. --- docs/topics/mailbox_types.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index fce8461..014960d 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -59,7 +59,7 @@ SSL) and you would like to archive all processed emails into a folder named ``Archived``, you would enter the following as your URI:: - pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com?archive=Archived + imap+ssl://youremailaddress%40gmail.com:1234@imap.gmail.com?archive=Archived Local File-based Mailboxes From cc4e72305d1f441ec45a72453c11110c605efad3 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Wed, 28 May 2014 20:10:45 -0600 Subject: [PATCH 17/86] Adding gmail oauth2 authentication python-social-auth is an optional dependency --- django_mailbox/google_utils.py | 126 ++++++++++++++++++++++++++ django_mailbox/transports/__init__.py | 1 + django_mailbox/transports/gmail.py | 49 ++++++++++ setup.py | 9 +- 4 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 django_mailbox/google_utils.py create mode 100644 django_mailbox/transports/gmail.py diff --git a/django_mailbox/google_utils.py b/django_mailbox/google_utils.py new file mode 100644 index 0000000..9900b70 --- /dev/null +++ b/django_mailbox/google_utils.py @@ -0,0 +1,126 @@ +from social.apps.django_app.default.models import UserSocialAuth +import requests +from django.conf import settings + + +class AccessTokenNotFound(Exception): + pass + + +class RefreshTokenNotFound(Exception): + pass + + +def get_google_consumer_key(): + return settings.SOCIAL_AUTH_GOOGLE_OAUTH2_KEY + + +def get_google_consumer_secret(): + return settings.SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET + + +def get_google_access_token(email): + # TODO: This should be cacheable + try: + me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") + return me.extra_data['access_token'] + except (UserSocialAuth.DoesNotExist, KeyError): + raise AccessTokenNotFound + + +def update_google_extra_data(email, extra_data): + try: + me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") + me.extra_data = extra_data + me.save() + except (UserSocialAuth.DoesNotExist, KeyError): + raise AccessTokenNotFound + + +def get_google_refresh_token(email): + try: + me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") + return me.extra_data['refresh_token'] + except (UserSocialAuth.DoesNotExist, KeyError): + raise RefreshTokenNotFound + + +def google_api_get(email, url): + headers = dict( + Authorization="Bearer %s" % get_google_access_token(email), + ) + r = requests.get(url, headers=headers) + print "I got a %s" % r.status_code + if r.status_code == 401: + # Go use the refresh token + refresh_authorization(email) + r = requests.get(url, headers=headers) + print "I got a %s" % r.status_code + if r.status_code == 200: + try: + return r.json() + except ValueError: + print "returning text" + return r.text + + +def google_api_post(email, url, post_data, authorized=True): + # TODO: Make this a lot less ugly. especially the 401 handling + headers = dict() + if authorized is True: + headers.update(dict( + Authorization="Bearer %s" % get_google_access_token(email), + )) + r = requests.post(url, headers=headers, data=post_data) + if r.status_code == 401: + refresh_authorization(email) + r = requests.post(url, headers=headers, data=post_data) + if r.status_code == 200: + try: + return r.json() + except ValueError: + print "returning text" + return r.text + + +def refresh_authorization(email): + refresh_token = get_google_refresh_token(email) + post_data = dict( + refresh_token=refresh_token, + client_id=get_google_consumer_key(), + client_secret=get_google_consumer_secret(), + grant_type='refresh_token', + ) + results = google_api_post( + email, + "https://accounts.google.com/o/oauth2/token?access_type=offline", + post_data, + authorized=False) + results.update({'refresh_token': refresh_token}) + update_google_extra_data(email, results) + + +def fetch_user_info(email): + result = google_api_get( + email, + "https://www.googleapis.com/oauth2/v1/userinfo?alt=json" + ) + return result + + +def fetch_google_contacts(email, limit=10000): + result = google_api_get( + email, + "https://www.google.com/m8/feeds/contacts/default/full?v=3.0&alt=json&max-results=%s" % limit + ) + entries = result['feed']['entry'] + valid_entries = [x for x in entries if u'gd$email' in x.keys() and u'gd$name' in x.keys()] + contacts = [] + for each in valid_entries: + try: + name = each[u'gd$name'][u'gd$fullName'][u'$t'] + except KeyError: + name = None + for each_email in each[u'gd$email']: + contacts.append(dict(name=name, email=each_email[u'address'])) + return contacts diff --git a/django_mailbox/transports/__init__.py b/django_mailbox/transports/__init__.py index fc17082..03b31f0 100644 --- a/django_mailbox/transports/__init__.py +++ b/django_mailbox/transports/__init__.py @@ -5,3 +5,4 @@ from django_mailbox.transports.mbox import MboxTransport from django_mailbox.transports.babyl import BabylTransport from django_mailbox.transports.mh import MHTransport from django_mailbox.transports.mmdf import MMDFTransport +from django_mailbox.transports.gmail import GmailImapTransport diff --git a/django_mailbox/transports/gmail.py b/django_mailbox/transports/gmail.py new file mode 100644 index 0000000..223ee51 --- /dev/null +++ b/django_mailbox/transports/gmail.py @@ -0,0 +1,49 @@ +from django_mailbox.transports.imap import ImapTransport + +class GmailImapTransport(ImapTransport): + + def connect(self, username, password): + # Try to use oauth2 first. It's much safer + try: + self._connect_oauth(username) + except (TypeError, ValueError), e: + print " Couldn't do oauth2", e + self.server = self.transport(self.hostname, self.port) + typ, msg = self.server.login(username, password) + self.server.select() + + def _connect_oauth(self, username): + # username should be an email address that has already been authorized + # for gmail access + try: + from django_mailbox.google_utils import ( + get_google_access_token, + fetch_user_info, + AccessTokenNotFound, + ) + except ImportError: + raise ValueError( + "Install python-social-auth to use oauth2 auth for gmail" + ) + + access_token = None + while access_token is None: + try: + access_token = get_google_access_token(username) + google_email_address = fetch_user_info(username)['email'] + except TypeError: + # This means that the google process took too long + # Trying again is the right thing to do + pass + except AccessTokenNotFound: + raise ValueError( + "No Token available in python-social-auth for %s" % username + ) + + auth_string = 'user=%s\1auth=Bearer %s\1\1' % ( + google_email_address, + access_token + ) + self.server = self.transport(self.hostname, self.port) + self.server.authenticate('XOAUTH2', lambda x: auth_string) + self.server.select() diff --git a/setup.py b/setup.py index c080a93..1f3800b 100755 --- a/setup.py +++ b/setup.py @@ -5,6 +5,10 @@ tests_require = [ 'mock', ] +gmail_oauth2_require = [ + 'python-social-auth', +] + setup( name='django-mailbox', version='3.3', @@ -16,7 +20,10 @@ setup( author='Adam Coddington', author_email='me@adamcoddington.net', tests_require=tests_require, - extras_require={'test': tests_require}, + extras_require={ + 'test': tests_require, + 'gmail-oauth2': gmail_oauth2_require + }, test_suite='django_mailbox.runtests.runtests', classifiers=[ 'Framework :: Django', From 29d36a493a3e20bcbf4deb849d5adb27f44fa6c5 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Wed, 28 May 2014 20:39:13 -0600 Subject: [PATCH 18/86] Adding ability to limit messages by max size Adding a new setting for maximum message size Adding method to limit a set of message uids by size Switching to IMAP uids so that we can trust that subsequent commands will use the same ids --- django_mailbox/transports/imap.py | 59 +++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index 078392e..570788a 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -2,12 +2,20 @@ from imaplib import IMAP4, IMAP4_SSL from .base import EmailTransport, MessageParseError +from django.conf import settings + +MAX_MESSAGE_SIZE = getattr( + settings, + 'DJANGO_MAILBOX_MAX_MESSAGE_SIZE', + False +) class ImapTransport(EmailTransport): def __init__(self, hostname, port=None, ssl=False, archive=''): self.hostname = hostname self.port = port self.archive = archive + self.MAX_MSG_SIZE = MAX_MESSAGE_SIZE if ssl: self.transport = IMAP4_SSL if not self.port: @@ -22,10 +30,47 @@ class ImapTransport(EmailTransport): typ, msg = self.server.login(username, password) self.server.select() - def get_message(self): - typ, inbox = self.server.search(None, 'ALL') - if not inbox[0]: + def _get_all_message_ids(self): + # Fetch all the message uids + response, message_ids = self.server.uid('search', None, 'ALL', ) + return message_ids[0].split(' ') + + def _get_small_message_ids(self, message_ids): + # Using existing message uids, get the sizes and + # return only those that are under the size + # limit + safe_message_ids = [] + + status, data = self.server.uid( + 'fetch', + ','.join(message_ids), + '(BODY.PEEK[HEADER] RFC822.SIZE BODYSTRUCTURE)' + ) + + for each_msg in data: + if isinstance(each_msg, tuple): + try: + metadata, structure = each_msg[0].split(' BODYSTRUCTURE ') + uid = metadata.split('(')[1].split(' ')[1] + size = metadata.split('(')[1].split(' ')[3] + if int(size) <= int(self.MAX_MSG_SIZE): + safe_message_ids.append(uid) + except ValueError, e: + print "ValueError: %s working on %s" % (e, each_msg[0]) + print each_msg + pass + return safe_message_ids + + def get_message(self): + message_ids = self._get_all_message_ids() + + # Limit the uids to the small ones if we care about that + if self.MAX_MSG_SIZE: + message_ids = self._get_small_message_ids(message_ids) + + + if not message_ids: return if self.archive: @@ -34,17 +79,17 @@ class ImapTransport(EmailTransport): # If the archive folder does not exist, create it self.server.create(self.archive) - for key in inbox[0].split(): + for uid in message_ids: try: - typ, msg_contents = self.server.fetch(key, '(RFC822)') + typ, msg_contents = self.server.uid('fetch', uid, '(RFC822)') message = self.get_email_from_bytes(msg_contents[0][1]) yield message except MessageParseError: continue if self.archive: - self.server.copy(key, self.archive) + self.server.uid('copy', uid, self.archive) - self.server.store(key, "+FLAGS", "\\Deleted") + self.server.uid('store', uid, "+FLAGS", "\\Deleted") self.server.expunge() return From 58550c76c933b797e0487192d98ded1d7d583cb4 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Wed, 28 May 2014 20:50:48 -0600 Subject: [PATCH 19/86] Teaching people how to use the settings --- docs/topics/settings.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6023046..9f4dfe7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -58,3 +58,10 @@ Settings a message instance's ``get_email_object`` method being called. Value of this field is the primary key of the ``django_mailbox.MessageAttachment`` instance currently storing this payload component's contents. + +* ``DJANGO_MAILBOX_MAX_MESSAGE_SIZE`` + * Default: ``False`` + * Type: ``integer`` + * If this is set, it will be read as a number of + bytes. Any messages above that size will not be + downloaded. ``2000000`` is 2 Megabytes. From b2abec1d401787c841b968ad364ed3b797e20c57 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Wed, 28 May 2014 20:53:44 -0600 Subject: [PATCH 20/86] fixing typo in the docs --- docs/topics/settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9f4dfe7..290e884 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -60,6 +60,7 @@ Settings instance currently storing this payload component's contents. * ``DJANGO_MAILBOX_MAX_MESSAGE_SIZE`` + * Default: ``False`` * Type: ``integer`` * If this is set, it will be read as a number of From 677e420dec863dfa5e3a49d6129fb64254e63132 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Wed, 28 May 2014 21:24:46 -0600 Subject: [PATCH 21/86] Adding gmail to the mailbox types and adding the type to models.py --- django_mailbox/models.py | 10 +++++++++- docs/topics/mailbox_types.rst | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index f4b5dc6..4e3fef4 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -15,7 +15,7 @@ from django.core.files.base import ContentFile from django.db import models from django_mailbox.transports import Pop3Transport, ImapTransport,\ MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ - MMDFTransport + MMDFTransport, GmailImapTransport from django_mailbox.signals import message_received import six from six.moves.urllib.parse import parse_qs, unquote, urlparse @@ -168,6 +168,14 @@ class Mailbox(models.Model): archive=self.archive ) conn.connect(self.username, self.password) + elif self.type == 'gmail': + conn = GmailImapTransport( + self.location, + port=self.port if self.port else None, + ssl=True, + archive=self.archive + ) + conn.connect(self.username, self.password) elif self.type == 'pop3': conn = Pop3Transport( self.location, diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index 014960d..eb01ad0 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -61,6 +61,20 @@ would enter the following as your URI:: imap+ssl://youremailaddress%40gmail.com:1234@imap.gmail.com?archive=Archived +Gmail IMAP with Oauth2 authentication +------------------------------------- + +Gmail supports using oauth2 for authentication_ which is more secure. +To handle the handshake and storing the credentials, use python-social-auth_. + +.. _authentication: https://developers.google.com/gmail/xoauth2_protocol +.. _python-social-auth: http://psa.matiasaguirre.net/ + +The Gmail Mailbox is also a regular IMAP mailbox, but the password will be ignored if oauth2 succeeds. It can fall back to password as needed. +Build your URI accordingly:: + + gmail+ssl://youremailaddress%40gmail.com:oauth2@imap.gmail.com?archive=Archived + Local File-based Mailboxes -------------------------- From d6f3ff2c5c4375343139570c5d9b0ed77b163158 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Wed, 28 May 2014 21:30:04 -0600 Subject: [PATCH 22/86] Adding a missed edit --- docs/topics/mailbox_types.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index eb01ad0..de0616f 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -12,6 +12,7 @@ POP3 and IMAP as well as local file-based mailboxes. ============ ============== ================================================================================================================================================================= POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` IMAP ``imap://`` Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI. + Gmail IMAP ``gmail+ssl://`` Password is ignored if oauth2 succeeds, but can be used as a fallback Maildir ``maildir://`` Mbox ``mbox://`` Babyl ``babyl://`` From 7e4f8cd6cd4f31e0c3d44b7cd03c468e3e0d754f Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Wed, 28 May 2014 21:32:25 -0600 Subject: [PATCH 23/86] reformatting --- docs/topics/mailbox_types.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index de0616f..7ec15f3 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -7,19 +7,19 @@ POP3 and IMAP as well as local file-based mailboxes. .. table:: 'Protocol' Options - ============ ============== ================================================================================================================================================================= - Mailbox Type 'Protocol':// Notes - ============ ============== ================================================================================================================================================================= - POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` - IMAP ``imap://`` Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI. + ============ ================ ================================================================================================================================================================= + Mailbox Type 'Protocol':// Notes + ============ ================ ================================================================================================================================================================= + POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` + IMAP ``imap://`` Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI. Gmail IMAP ``gmail+ssl://`` Password is ignored if oauth2 succeeds, but can be used as a fallback Maildir ``maildir://`` Mbox ``mbox://`` Babyl ``babyl://`` MH ``mh://`` MMDF ``mmdf://`` - Piped Mail *empty* See :ref:`receiving-mail-from-exim4-or-postfix` - ============ ============== ================================================================================================================================================================= + Piped Mail *empty* See :ref:`receiving-mail-from-exim4-or-postfix` + ============ ================ ================================================================================================================================================================= .. warning:: From 857643e97828e9187aa399298064ba5236a698b1 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Wed, 28 May 2014 21:33:38 -0600 Subject: [PATCH 24/86] Update mailbox_types.rst --- docs/topics/mailbox_types.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index 7ec15f3..bb8122c 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -12,7 +12,7 @@ POP3 and IMAP as well as local file-based mailboxes. ============ ================ ================================================================================================================================================================= POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` IMAP ``imap://`` Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI. - Gmail IMAP ``gmail+ssl://`` Password is ignored if oauth2 succeeds, but can be used as a fallback + Gmail IMAP ``gmail+ssl://`` Password is only used as a fallback if oauth2 fails Maildir ``maildir://`` Mbox ``mbox://`` Babyl ``babyl://`` From ea793545d9245ca1ed73e5e51bb05931104cee97 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Thu, 29 May 2014 07:31:14 -0600 Subject: [PATCH 25/86] Update gmail.py --- django_mailbox/transports/gmail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_mailbox/transports/gmail.py b/django_mailbox/transports/gmail.py index 223ee51..e477293 100644 --- a/django_mailbox/transports/gmail.py +++ b/django_mailbox/transports/gmail.py @@ -6,8 +6,8 @@ class GmailImapTransport(ImapTransport): # Try to use oauth2 first. It's much safer try: self._connect_oauth(username) - except (TypeError, ValueError), e: - print " Couldn't do oauth2", e + except (TypeError, ValueError) as e: + print " Couldn't do oauth2 because %s" % e self.server = self.transport(self.hostname, self.port) typ, msg = self.server.login(username, password) self.server.select() From 05c8d3ece81411951bbed3300738032885898ec1 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Thu, 29 May 2014 16:52:08 -0700 Subject: [PATCH 26/86] Minor cleanup & a Python 3k fix. --- django_mailbox/google_utils.py | 8 ++++++-- django_mailbox/transports/gmail.py | 12 ++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/django_mailbox/google_utils.py b/django_mailbox/google_utils.py index 9900b70..a668ff6 100644 --- a/django_mailbox/google_utils.py +++ b/django_mailbox/google_utils.py @@ -111,10 +111,14 @@ def fetch_user_info(email): def fetch_google_contacts(email, limit=10000): result = google_api_get( email, - "https://www.google.com/m8/feeds/contacts/default/full?v=3.0&alt=json&max-results=%s" % limit + "https://www.google.com/m8/feeds/contacts/default/full" + "?v=3.0&alt=json&max-results=%s" % limit ) entries = result['feed']['entry'] - valid_entries = [x for x in entries if u'gd$email' in x.keys() and u'gd$name' in x.keys()] + valid_entries = [ + x for x in entries + if u'gd$email' in x.keys() and u'gd$name' in x.keys() + ] contacts = [] for each in valid_entries: try: diff --git a/django_mailbox/transports/gmail.py b/django_mailbox/transports/gmail.py index e477293..0922f9a 100644 --- a/django_mailbox/transports/gmail.py +++ b/django_mailbox/transports/gmail.py @@ -1,5 +1,11 @@ +import logging + from django_mailbox.transports.imap import ImapTransport + +logger = logging.getLogger(__name__) + + class GmailImapTransport(ImapTransport): def connect(self, username, password): @@ -7,7 +13,7 @@ class GmailImapTransport(ImapTransport): try: self._connect_oauth(username) except (TypeError, ValueError) as e: - print " Couldn't do oauth2 because %s" % e + logger.warning("Couldn't do oauth2 because %s" % e) self.server = self.transport(self.hostname, self.port) typ, msg = self.server.login(username, password) self.server.select() @@ -37,7 +43,9 @@ class GmailImapTransport(ImapTransport): pass except AccessTokenNotFound: raise ValueError( - "No Token available in python-social-auth for %s" % username + "No Token available in python-social-auth for %s" % ( + username + ) ) auth_string = 'user=%s\1auth=Bearer %s\1\1' % ( From ec18a01880819d0aa2096a6085723b4eef31b59a Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Thu, 29 May 2014 23:41:11 -0600 Subject: [PATCH 27/86] Refactored the imap transport based on the tests Testing the switch to uid-based IMAP calls required better testing The better testing revealed some code that wasn't so pretty I fixed the tests and the code --- django_mailbox/tests/base.py | 10 ++ django_mailbox/tests/test_transports.py | 127 +++++++++++------------- django_mailbox/transports/imap.py | 39 ++++---- 3 files changed, 88 insertions(+), 88 deletions(-) diff --git a/django_mailbox/tests/base.py b/django_mailbox/tests/base.py index a559b9e..72c3249 100644 --- a/django_mailbox/tests/base.py +++ b/django_mailbox/tests/base.py @@ -8,6 +8,16 @@ from django.test import TestCase from django_mailbox import models from django_mailbox.models import Mailbox, Message +def get_email_as_text(name): + with open( + os.path.join( + os.path.dirname(__file__), + 'messages', + name, + ), + 'rb' + ) as f: + return f.read() class EmailMessageTestCase(TestCase): ALLOWED_EXTRA_HEADERS = [ diff --git a/django_mailbox/tests/test_transports.py b/django_mailbox/tests/test_transports.py index 7f044d4..b701eae 100644 --- a/django_mailbox/tests/test_transports.py +++ b/django_mailbox/tests/test_transports.py @@ -1,12 +1,42 @@ import mock -import six -from django_mailbox.tests.base import EmailMessageTestCase +from django.test.utils import override_settings + +from django_mailbox.tests.base import EmailMessageTestCase, get_email_as_text from django_mailbox.transports import ImapTransport, Pop3Transport +FAKE_UID_SEARCH_ANSWER = ('OK', ['18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44']) +FAKE_UID_FETCH_SIZES = ('OK', ['1 (UID 18 RFC822.SIZE 58070000000)', '2 (UID 19 RFC822.SIZE 2593)']) +FAKE_UID_FETCH_MSG = ('OK', [('1 (UID 18 RFC822 {5807}', get_email_as_text('generic_message.eml') ),]) +FAKE_UID_COPY_MSG = ('OK', ['[COPYUID 1 2 2] (Success)']) +FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS = ('OK', ['(\\HasNoChildren \\All) "/" "[Gmail]/All Mail"']) -class TestImapTransport(EmailMessageTestCase): +class IMAPTestCase(EmailMessageTestCase): def setUp(self): + def imap_server_uid_method(*args): + cmd = args[0] + arg2 = args[2] + if cmd == 'search': + return FAKE_UID_SEARCH_ANSWER + if cmd == 'copy': + return FAKE_UID_COPY_MSG + if cmd == 'fetch': + if arg2 == '(RFC822.SIZE)': + return FAKE_UID_FETCH_SIZES + if arg2 == '(RFC822)': + return FAKE_UID_FETCH_MSG + def imap_server_list_method(pattern=None): + return FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS + + self.imap_server = mock.Mock() + self.imap_server.uid = imap_server_uid_method + self.imap_server.list = imap_server_list_method + super(IMAPTestCase, self).setUp() + + +class TestImapTransport(IMAPTestCase): + def setUp(self): + super(TestImapTransport, self).setUp() self.arbitrary_hostname = 'one.two.three' self.arbitrary_port = 100 self.ssl = False @@ -15,42 +45,18 @@ class TestImapTransport(EmailMessageTestCase): self.arbitrary_port, self.ssl ) - self.transport.server = None - super(TestImapTransport, self).setUp() + self.transport.server = self.imap_server def test_get_email_message(self): - with mock.patch.object(self.transport, 'server') as server: - server.search.return_value = ( - 'OK', - [ - 'One', # This is totally arbitrary - ] - ) - server.fetch.return_value = ( - 'OK', - [ - [ - '1 (RFC822 {8190}', # Wat? - self._get_email_as_text('generic_message.eml') - ], - ')', - ] - ) - actual_messages = list(self.transport.get_message()) - - self.assertEqual(len(actual_messages), 1) - + actual_messages = list(self.transport.get_message()) + self.assertEqual(len(actual_messages), 27) actual_message = actual_messages[0] expected_message = self._get_email_object('generic_message.eml') - self.assertEqual(expected_message, actual_message) - -class TestImapArchivedTransport(EmailMessageTestCase): +class TestImapArchivedTransport(TestImapTransport): def setUp(self): - self.arbitrary_hostname = 'one.two.three' - self.arbitrary_port = 100 - self.ssl = False + super(TestImapArchivedTransport, self).setUp() self.archive = 'Archive' self.transport = ImapTransport( self.arbitrary_hostname, @@ -58,50 +64,34 @@ class TestImapArchivedTransport(EmailMessageTestCase): self.ssl, self.archive ) - self.transport.server = None - super(TestImapArchivedTransport, self).setUp() + self.transport.server = self.imap_server + +class TestMaxSizeImapTransport(TestImapTransport): + + @override_settings(DJANGO_MAILBOX_MAX_MESSAGE_SIZE=5807) + def setUp(self): + super(TestMaxSizeImapTransport, self).setUp() + + self.transport = ImapTransport( + self.arbitrary_hostname, + self.arbitrary_port, + self.ssl, + ) + self.transport.server = self.imap_server + + def test_size_limit(self): + all_message_ids = self.transport._get_all_message_ids() + small_message_ids = self.transport._get_small_message_ids(all_message_ids) + self.assertEqual(len(small_message_ids), 1) def test_get_email_message(self): - with mock.patch.object(self.transport, 'server') as server: - server.search.return_value = ( - 'OK', - [ - 'One', # This is totally arbitrary - ] - ) - server.fetch.return_value = ( - 'OK', - [ - [ - '1 (RFC822 {8190}', # Wat? - self._get_email_as_text('generic_message.eml') - ], - ')', - ] - ) - server.list.return_value = ( - 'OK', - [ - '(\\HasNoChildren) "/" "Archive"' - ] - ) - server.copy.return_value = ( - 'OK', - [ - '[COPYUID 1 2 2] (Success)' - ] - ) - actual_messages = list(self.transport.get_message()) - + actual_messages = list(self.transport.get_message()) self.assertEqual(len(actual_messages), 1) - actual_message = actual_messages[0] expected_message = self._get_email_object('generic_message.eml') - self.assertEqual(expected_message, actual_message) - class TestPop3Transport(EmailMessageTestCase): def setUp(self): self.arbitrary_hostname = 'one.two.three' @@ -139,3 +129,4 @@ class TestPop3Transport(EmailMessageTestCase): expected_message = self._get_email_object('generic_message.eml') self.assertEqual(expected_message, actual_message) + diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index 570788a..c40a647 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -1,17 +1,21 @@ +import logging + +logger = logging.getLogger(__name__) + from imaplib import IMAP4, IMAP4_SSL from .base import EmailTransport, MessageParseError from django.conf import settings -MAX_MESSAGE_SIZE = getattr( - settings, - 'DJANGO_MAILBOX_MAX_MESSAGE_SIZE', - False -) class ImapTransport(EmailTransport): def __init__(self, hostname, port=None, ssl=False, archive=''): + MAX_MESSAGE_SIZE = getattr( + settings, + 'DJANGO_MAILBOX_MAX_MESSAGE_SIZE', + False + ) self.hostname = hostname self.port = port self.archive = archive @@ -30,10 +34,9 @@ class ImapTransport(EmailTransport): typ, msg = self.server.login(username, password) self.server.select() - def _get_all_message_ids(self): # Fetch all the message uids - response, message_ids = self.server.uid('search', None, 'ALL', ) + response, message_ids = self.server.uid('search', None, 'ALL') return message_ids[0].split(' ') def _get_small_message_ids(self, message_ids): @@ -45,21 +48,18 @@ class ImapTransport(EmailTransport): status, data = self.server.uid( 'fetch', ','.join(message_ids), - '(BODY.PEEK[HEADER] RFC822.SIZE BODYSTRUCTURE)' + '(RFC822.SIZE)' ) for each_msg in data: - if isinstance(each_msg, tuple): - try: - metadata, structure = each_msg[0].split(' BODYSTRUCTURE ') - uid = metadata.split('(')[1].split(' ')[1] - size = metadata.split('(')[1].split(' ')[3] - if int(size) <= int(self.MAX_MSG_SIZE): - safe_message_ids.append(uid) - except ValueError, e: - print "ValueError: %s working on %s" % (e, each_msg[0]) - print each_msg - pass + try: + uid = each_msg.split(' ')[2] + size = each_msg.split(' ')[4].rstrip(')') + if int(size) <= int(self.MAX_MSG_SIZE): + safe_message_ids.append(uid) + except ValueError, e: + logger.warning("ValueError: %s working on %s" % (e, each_msg[0])) + pass return safe_message_ids def get_message(self): @@ -69,7 +69,6 @@ class ImapTransport(EmailTransport): if self.MAX_MSG_SIZE: message_ids = self._get_small_message_ids(message_ids) - if not message_ids: return From 1efa1140a689edd50aa843f6e69be87fe5b96523 Mon Sep 17 00:00:00 2001 From: Alex Lovell-Troy Date: Thu, 29 May 2014 23:47:48 -0600 Subject: [PATCH 28/86] I'll get the 2to3 changes right someday --- django_mailbox/transports/imap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index c40a647..b6255b1 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -57,7 +57,7 @@ class ImapTransport(EmailTransport): size = each_msg.split(' ')[4].rstrip(')') if int(size) <= int(self.MAX_MSG_SIZE): safe_message_ids.append(uid) - except ValueError, e: + except ValueError as e: logger.warning("ValueError: %s working on %s" % (e, each_msg[0])) pass return safe_message_ids From 61b1e55c5036c76c36cbbef087d261469c5dbbda Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 30 May 2014 16:32:57 -0700 Subject: [PATCH 29/86] Minor stylistic changes. --- django_mailbox/transports/imap.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index b6255b1..044838b 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -11,7 +11,7 @@ from django.conf import settings class ImapTransport(EmailTransport): def __init__(self, hostname, port=None, ssl=False, archive=''): - MAX_MESSAGE_SIZE = getattr( + self.max_message_size = getattr( settings, 'DJANGO_MAILBOX_MAX_MESSAGE_SIZE', False @@ -19,7 +19,6 @@ class ImapTransport(EmailTransport): self.hostname = hostname self.port = port self.archive = archive - self.MAX_MSG_SIZE = MAX_MESSAGE_SIZE if ssl: self.transport = IMAP4_SSL if not self.port: @@ -55,10 +54,12 @@ class ImapTransport(EmailTransport): try: uid = each_msg.split(' ')[2] size = each_msg.split(' ')[4].rstrip(')') - if int(size) <= int(self.MAX_MSG_SIZE): + if int(size) <= int(self.max_message_size): safe_message_ids.append(uid) except ValueError as e: - logger.warning("ValueError: %s working on %s" % (e, each_msg[0])) + logger.warning( + "ValueError: %s working on %s" % (e, each_msg[0]) + ) pass return safe_message_ids @@ -66,7 +67,7 @@ class ImapTransport(EmailTransport): message_ids = self._get_all_message_ids() # Limit the uids to the small ones if we care about that - if self.MAX_MSG_SIZE: + if self.max_message_size: message_ids = self._get_small_message_ids(message_ids) if not message_ids: From 7a19c5db2515591233c5c28e7d11e55ddf13431b Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sat, 31 May 2014 19:04:24 -0700 Subject: [PATCH 30/86] Minor documentation tweaks. --- docs/topics/mailbox_types.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index bb8122c..e3c9244 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -12,7 +12,7 @@ POP3 and IMAP as well as local file-based mailboxes. ============ ================ ================================================================================================================================================================= POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` IMAP ``imap://`` Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI. - Gmail IMAP ``gmail+ssl://`` Password is only used as a fallback if oauth2 fails + Gmail IMAP ``gmail+ssl://`` Uses OAuth authentication for Gmail's IMAP transport. See :ref:`gmail-oauth` for details. Maildir ``maildir://`` Mbox ``mbox://`` Babyl ``babyl://`` @@ -62,16 +62,21 @@ would enter the following as your URI:: imap+ssl://youremailaddress%40gmail.com:1234@imap.gmail.com?archive=Archived +.. _gmail-oauth: + Gmail IMAP with Oauth2 authentication ------------------------------------- -Gmail supports using oauth2 for authentication_ which is more secure. +For added security, Gmail supports using OAuth2 for authentication_. To handle the handshake and storing the credentials, use python-social-auth_. .. _authentication: https://developers.google.com/gmail/xoauth2_protocol .. _python-social-auth: http://psa.matiasaguirre.net/ -The Gmail Mailbox is also a regular IMAP mailbox, but the password will be ignored if oauth2 succeeds. It can fall back to password as needed. +The Gmail Mailbox is also a regular IMAP mailbox, +but the password you specify will be ignored if OAuth2 authentication succeeds. +It will fall back to use your specified password as needed. + Build your URI accordingly:: gmail+ssl://youremailaddress%40gmail.com:oauth2@imap.gmail.com?archive=Archived From d1485d9cd5bd55a0832536c067c298c5ebbb3be7 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 1 Jun 2014 14:24:34 -0700 Subject: [PATCH 31/86] Replacing print statements with logger.info --- django_mailbox/google_utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/django_mailbox/google_utils.py b/django_mailbox/google_utils.py index a668ff6..4d6b773 100644 --- a/django_mailbox/google_utils.py +++ b/django_mailbox/google_utils.py @@ -1,6 +1,11 @@ -from social.apps.django_app.default.models import UserSocialAuth -import requests +import logging + from django.conf import settings +import requests +from social.apps.django_app.default.models import UserSocialAuth + + +logger = logging.getLogger(__name__) class AccessTokenNotFound(Exception): @@ -50,17 +55,16 @@ def google_api_get(email, url): Authorization="Bearer %s" % get_google_access_token(email), ) r = requests.get(url, headers=headers) - print "I got a %s" % r.status_code + logger.info("I got a %s", r.status_code) if r.status_code == 401: # Go use the refresh token refresh_authorization(email) r = requests.get(url, headers=headers) - print "I got a %s" % r.status_code + logger.info("I got a %s", r.status_code) if r.status_code == 200: try: return r.json() except ValueError: - print "returning text" return r.text @@ -79,7 +83,6 @@ def google_api_post(email, url, post_data, authorized=True): try: return r.json() except ValueError: - print "returning text" return r.text From 8af7c676fb8a3d8377f74150b69aadc60593164e Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 1 Jun 2014 14:26:19 -0700 Subject: [PATCH 32/86] Remoing unaccessed 'fetch_contacts' function. --- django_mailbox/google_utils.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/django_mailbox/google_utils.py b/django_mailbox/google_utils.py index 4d6b773..b834f59 100644 --- a/django_mailbox/google_utils.py +++ b/django_mailbox/google_utils.py @@ -109,25 +109,3 @@ def fetch_user_info(email): "https://www.googleapis.com/oauth2/v1/userinfo?alt=json" ) return result - - -def fetch_google_contacts(email, limit=10000): - result = google_api_get( - email, - "https://www.google.com/m8/feeds/contacts/default/full" - "?v=3.0&alt=json&max-results=%s" % limit - ) - entries = result['feed']['entry'] - valid_entries = [ - x for x in entries - if u'gd$email' in x.keys() and u'gd$name' in x.keys() - ] - contacts = [] - for each in valid_entries: - try: - name = each[u'gd$name'][u'gd$fullName'][u'$t'] - except KeyError: - name = None - for each_email in each[u'gd$email']: - contacts.append(dict(name=name, email=each_email[u'address'])) - return contacts From 652ae282ba28070221d2d9384049284fab239e1d Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 1 Jun 2014 14:27:03 -0700 Subject: [PATCH 33/86] Bumping version number to 3.4. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f3800b..447350a 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='3.3', + version='3.4', url='http://github.com/latestrevision/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From db2193d2be7615985cd7fba2504b0e422ec018c2 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 2 Jun 2014 16:13:49 -0700 Subject: [PATCH 34/86] Updating readme. --- readme.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/readme.rst b/readme.rst index 8f23136..b5ac337 100644 --- a/readme.rst +++ b/readme.rst @@ -1,12 +1,10 @@ .. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master :target: https://travis-ci.org/coddingtonbear/django-mailbox -How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, -or had to otherwise construct an application driven by e-mail? -One too many times, I'm sure. +Easily consume POP3, IMAP, or a local mailbox for incoming content. -This small Django application will allow you to specify mailboxes that you would like consumed for incoming content; -the e-mail will be stored, and you can process it at will (or, if you're in a hurry, by subscribing to a signal). +This Django application will allow you to specify mailboxes that you would like consumed for incoming content; +the e-mail will be stored in the database, and you can process it at will (or, if you're in a hurry, by subscribing to a signal). - Documentation for django-mailbox is available on `ReadTheDocs `_. From 8741648666836ddc134fe91e09d1cb68258fa4cf Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 3 Jun 2014 15:09:44 -0700 Subject: [PATCH 35/86] Minor readme rewording. --- readme.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.rst b/readme.rst index b5ac337..9aa09bc 100644 --- a/readme.rst +++ b/readme.rst @@ -1,7 +1,7 @@ .. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master :target: https://travis-ci.org/coddingtonbear/django-mailbox -Easily consume POP3, IMAP, or a local mailbox for incoming content. +Easily pull messages from POP3, IMAP, or local mailboxes into Django models. This Django application will allow you to specify mailboxes that you would like consumed for incoming content; the e-mail will be stored in the database, and you can process it at will (or, if you're in a hurry, by subscribing to a signal). From 3a2e05b01f0bef6d7eda4af35266025721d6a006 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 3 Jun 2014 19:51:34 -0700 Subject: [PATCH 36/86] Updating documentation to explain exim4 configurations in greater detail. --- docs/topics/polling.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/topics/polling.rst b/docs/topics/polling.rst index 7b52d47..0134104 100644 --- a/docs/topics/polling.rst +++ b/docs/topics/polling.rst @@ -58,8 +58,8 @@ start by adding a new router configuration to your Exim4 configuration like:: django_mailbox: debug_print = 'R: django_mailbox for $localpart@$domain' driver = accept - domains = +local_domains transport = send_to_django_mailbox + domains = mydomain.com local_parts = emailusernameone : emailusernametwo Make sure that the e-mail addresses you would like handled by Django Mailbox @@ -72,6 +72,14 @@ For example, if one of the e-mail addresses targeted at this machine is ``jane@example.com``, the contents of ``local_parts`` would be, simply ``jane``. +.. note:: + + If you would like messages addressed to *any* account *@mydomain.com* + to be delivered to django_mailbox, simply omit the above ``local_parts`` + setting. In the same vein, if you would like messages addressed to + any domain or any local domains, you can omit the ``domains`` setting + or set it to ``+local_domains`` respectively. + Next, a new transport configuration to your Exim4 configuration:: send_to_django_mailbox: From 83a6249c8517339d37a0c16e59c7d4db1380bbef Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Thu, 31 Jul 2014 20:30:42 -0700 Subject: [PATCH 37/86] s/latestrevision/coddingtonbear/g --- docs/index.rst | 4 ++-- docs/topics/installation.rst | 4 ++-- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4d5c0b4..fbb2493 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,8 +6,8 @@ Django-mailbox ============== -.. image:: https://travis-ci.org/latestrevision/django-mailbox.png?branch=master - :target: https://travis-ci.org/latestrevision/django-mailbox +.. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master + :target: https://travis-ci.org/coddingtonbear/django-mailbox How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, or had to otherwise construct an application driven by e-mail? diff --git a/docs/topics/installation.rst b/docs/topics/installation.rst index 6286070..1784560 100644 --- a/docs/topics/installation.rst +++ b/docs/topics/installation.rst @@ -6,9 +6,9 @@ You can either install from pip:: pip install django-mailbox -*or* checkout and install the source from the `github repository `_:: +*or* checkout and install the source from the `github repository `_:: - git clone https://github.com/latestrevision/django-mailbox.git + git clone https://github.com/coddingtonbear/django-mailbox.git cd django-mailbox python setup.py install diff --git a/setup.py b/setup.py index 447350a..e5bb0c4 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', version='3.4', - url='http://github.com/latestrevision/django-mailbox/', + url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' 'Postfix or Exim4 into your Django application automatically.' From 9706fda4e304cbbae5e1ac386aa71ac8c1ff2f4c Mon Sep 17 00:00:00 2001 From: fsboehme Date: Wed, 13 Aug 2014 16:45:41 -0700 Subject: [PATCH 38/86] Update installation.rst added db migration step and mailbox setup to installation notes --- docs/topics/installation.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/installation.rst b/docs/topics/installation.rst index 1784560..e8343b9 100644 --- a/docs/topics/installation.rst +++ b/docs/topics/installation.rst @@ -15,3 +15,8 @@ You can either install from pip:: After you have installed the package, you should add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in your project's ``settings.py`` file. +Run ``python manage.py migrate django_mailbox`` to create the required database tables. + +Head to your admin and create a mailbox to consume (see next docs page). + +Test your setup by selecting 'Get new mail' from the action dropdown in the Mailbox change list. From 24dd3be36a127ba09283e9b94e5ca75f30be8af4 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Wed, 13 Aug 2014 17:40:03 -0700 Subject: [PATCH 39/86] Minor alterations and clarifications to @fsboehme's fantastic addition. --- docs/topics/installation.rst | 37 +++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/docs/topics/installation.rst b/docs/topics/installation.rst index e8343b9..bb5cc5a 100644 --- a/docs/topics/installation.rst +++ b/docs/topics/installation.rst @@ -2,21 +2,36 @@ Installation ============ -You can either install from pip:: +1. You can either install from pip:: - pip install django-mailbox + pip install django-mailbox + + *or* checkout and install the source from the `github repository `_:: + + git clone https://github.com/coddingtonbear/django-mailbox.git + cd django-mailbox + python setup.py install -*or* checkout and install the source from the `github repository `_:: +2. After you have installed the package, + add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in + your project's ``settings.py`` file. - git clone https://github.com/coddingtonbear/django-mailbox.git - cd django-mailbox - python setup.py install +3. From your project folder, run ``python manage.py migrate django_mailbox`` to + create the required database tables. -After you have installed the package, -you should add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in your project's ``settings.py`` file. +4. Head to your project's Django Admin and create a mailbox to consume. -Run ``python manage.py migrate django_mailbox`` to create the required database tables. -Head to your admin and create a mailbox to consume (see next docs page). +.. note:: + + Once you have entered a mailbox to consume, you can easily verify that you + have properly configured your mailbox by either: + + * From the Django Admin, using the 'Get New Mail' action from the action + dropdown on the Mailbox changelist + (http://yourproject.com/admin/django_mailbox/mailbox/). + * *Or* from a shell opened to your project's directory, using the + ``getmail`` management command by running:: + + python manage.py getmail -Test your setup by selecting 'Get new mail' from the action dropdown in the Mailbox change list. From 3ffbcea3bf34e0afdc00e2ba922aded595a60c79 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Wed, 13 Aug 2014 17:40:26 -0700 Subject: [PATCH 40/86] This probably should have been using explicit references anyway. --- docs/index.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index fbb2493..5076fb7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,9 +20,13 @@ Contents: .. toctree:: :maxdepth: 3 - :glob: - topics/* + topics/installation + topics/mailbox_types + topics/polling + topics/signal + topics/settings + topics/message-storage Indices and tables From 7474cf01181c161a56542b445fb102b83b130809 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Thu, 14 Aug 2014 21:38:52 -0700 Subject: [PATCH 41/86] Prevent ImapTransport from raising exception when no messages are available. Fixes #24. Release 3.4.1. --- django_mailbox/transports/imap.py | 8 +++++++- setup.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index 044838b..62ebf39 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -36,7 +36,13 @@ class ImapTransport(EmailTransport): def _get_all_message_ids(self): # Fetch all the message uids response, message_ids = self.server.uid('search', None, 'ALL') - return message_ids[0].split(' ') + message_id_string = message_ids[0].strip() + # Usually `message_id_string` will be a list of space-separated + # ids; we must make sure that it isn't an empty string before + # splitting into individual UIDs. + if message_id_string: + return message_id_string.split(' ') + return [] def _get_small_message_ids(self, message_ids): # Using existing message uids, get the sizes and diff --git a/setup.py b/setup.py index e5bb0c4..fe50575 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='3.4', + version='3.4.1', url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 09718a1901ee299b069091a8b47295d405eff42c Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 15 Aug 2014 21:00:12 -0700 Subject: [PATCH 42/86] Adding share/ to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index db89701..8fdf2f0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ bin/* lib/* src/* dist/* +share/* include/* .Python *egg* From 6b29b1bc86bb8f319f7e3e480a8cc885e41283d2 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 15 Aug 2014 21:02:53 -0700 Subject: [PATCH 43/86] Moving 'message storage details' page into appendix --- .gitignore | 1 + docs/index.rst | 2 +- docs/topics/appendix.rst | 8 ++++++++ docs/topics/{ => appendix}/message-storage.rst | 0 4 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 docs/topics/appendix.rst rename docs/topics/{ => appendix}/message-storage.rst (100%) diff --git a/.gitignore b/.gitignore index 8fdf2f0..72dc87e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ lib/* src/* dist/* share/* +docs/_build/* include/* .Python *egg* diff --git a/docs/index.rst b/docs/index.rst index 5076fb7..0c063fd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,7 +26,7 @@ Contents: topics/polling topics/signal topics/settings - topics/message-storage + topics/appendix Indices and tables diff --git a/docs/topics/appendix.rst b/docs/topics/appendix.rst new file mode 100644 index 0000000..c11f726 --- /dev/null +++ b/docs/topics/appendix.rst @@ -0,0 +1,8 @@ +Appendix +======== + +.. toctree:: + :maxdepth: 3 + :glob: + + appendix/* diff --git a/docs/topics/message-storage.rst b/docs/topics/appendix/message-storage.rst similarity index 100% rename from docs/topics/message-storage.rst rename to docs/topics/appendix/message-storage.rst From 1503bcaeee619273dbb33ef572d2ba1e875a4206 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 17 Aug 2014 18:32:25 -0700 Subject: [PATCH 44/86] Moving settings into appendix, too. --- docs/index.rst | 1 - docs/topics/{ => appendix}/settings.rst | 0 2 files changed, 1 deletion(-) rename docs/topics/{ => appendix}/settings.rst (100%) diff --git a/docs/index.rst b/docs/index.rst index 0c063fd..0ecaaa4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,7 +25,6 @@ Contents: topics/mailbox_types topics/polling topics/signal - topics/settings topics/appendix diff --git a/docs/topics/settings.rst b/docs/topics/appendix/settings.rst similarity index 100% rename from docs/topics/settings.rst rename to docs/topics/appendix/settings.rst From b4c20bdf2efb9c962304b845f1ee21fcfd574226 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 22 Aug 2014 19:25:14 -0700 Subject: [PATCH 45/86] Reordering imports to match style guide. --- django_mailbox/transports/imap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index 62ebf39..e60fd11 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -1,12 +1,12 @@ +from imaplib import IMAP4, IMAP4_SSL import logging -logger = logging.getLogger(__name__) - -from imaplib import IMAP4, IMAP4_SSL +from django.conf import settings from .base import EmailTransport, MessageParseError -from django.conf import settings + +logger = logging.getLogger(__name__) class ImapTransport(EmailTransport): From dad65d802c7eab86c88dca3ef466dc4c43daffc0 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 22 Aug 2014 19:27:36 -0700 Subject: [PATCH 46/86] Wrap added \Deleted flag in parentheses. Fixes #23. Apparently the IMAP specification expects that the +FLAGS argument be surrounded by parentheses; although some e-mail backends (like Gmail) will accept the +FLAGS argument without surrounding the \Deleted flag in parentheses, many (reasonably) follow the specification much more closely. This problem was first noted in report #23. Searching for other examples of mail deletion, I've found several Stack Overflow articles in which people are attempting the same task, and each either uses `imap.store` without surrounding '\Deleted' in parentheses, or uses `imap.uid` *while* surrounding '\Deleted' in parentheses: * http://stackoverflow.com/questions/1777264/using-python-imaplib-to-delete-an-email-from-gmail * http://stackoverflow.com/questions/3180891/imap-deleting-messages --- django_mailbox/transports/imap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index e60fd11..90435eb 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -96,6 +96,6 @@ class ImapTransport(EmailTransport): if self.archive: self.server.uid('copy', uid, self.archive) - self.server.uid('store', uid, "+FLAGS", "\\Deleted") + self.server.uid('store', uid, "+FLAGS", "(\\Deleted)") self.server.expunge() return From 591f1545c87c4893e5555be2978a1d9cb2c2423d Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 22 Aug 2014 19:46:39 -0700 Subject: [PATCH 47/86] Decode from ASCII with replacement if errors are encountered while gathering text. Fixes #20. --- django_mailbox/models.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 4e3fef4..b126b74 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -3,6 +3,7 @@ import email from email.message import Message as EmailMessage from email.utils import formatdate, parseaddr from email.encoders import encode_base64 +import logging import mimetypes import os.path from quopri import encode as encode_quopri @@ -23,6 +24,9 @@ from six.moves.urllib.parse import parse_qs, unquote, urlparse from .utils import convert_header_to_unicode +logger = logging.getLogger(__name__) + + STRIP_UNALLOWED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES', @@ -447,7 +451,7 @@ class Message(models.Model): def get_text_body(self): def get_body_from_message(message): - body = '' + body = six.text_type('') for part in message.walk(): if ( part.get_content_maintype() == 'text' @@ -458,7 +462,19 @@ class Message(models.Model): if charset: this_part = this_part.decode(charset, 'replace') - body += this_part + try: + body += this_part + except ValueError: + # Since it did not declare a charset, and we *should* + # be 7-bit clean right now, let's assume it is ASCII. + body += this_part.decode('ascii', 'replace') + logger.warning( + 'Error encountered while decoding text ' + 'payload from an incorrectly-constructed e-mail; ' + 'payload was converted to ASCII with ' + 'replacement, but some data may not be ' + 'represented as the sender intended.' + ) return body return get_body_from_message( From 992375fc6bee16a48c83559feac8c084b62807f4 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 22 Aug 2014 19:55:18 -0700 Subject: [PATCH 48/86] Version 3.4.2a1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fe50575..fa10ddc 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='3.4.1', + version='3.4.2a1', url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From db6190a295383247c0762d1ff2a82a9af3d8d2cd Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 22 Aug 2014 20:02:58 -0700 Subject: [PATCH 49/86] Cleaning up management commands for PEP-8 and consistency. --- django_mailbox/management/commands/getmail.py | 25 ++++++++++++++----- .../commands/processincomingmessage.py | 11 +++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/django_mailbox/management/commands/getmail.py b/django_mailbox/management/commands/getmail.py index 6041782..31b2dbd 100644 --- a/django_mailbox/management/commands/getmail.py +++ b/django_mailbox/management/commands/getmail.py @@ -1,17 +1,30 @@ +import logging + from django.core.management.base import BaseCommand from django_mailbox.models import Mailbox + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + class Command(BaseCommand): def handle(self, *args, **options): mailboxes = Mailbox.active_mailboxes.all() if args: - mailboxes = mailboxes.filter(name = ' '.join(args)) + mailboxes = mailboxes.filter( + name=' '.join(args) + ) for mailbox in mailboxes: - self.stdout.write('Gathering messages for %s\n' % mailbox.name) + logger.info( + 'Gathering messages for %s', + mailbox.name + ) messages = mailbox.get_new_mail() for message in messages: - self.stdout.write('Received %s (from %s)\n' % ( - message.subject, - message.from_address - )) + logger.info( + 'Received %s (from %s)', + message.subject, + message.from_address + ) diff --git a/django_mailbox/management/commands/processincomingmessage.py b/django_mailbox/management/commands/processincomingmessage.py index 8d269b8..2fd9cba 100644 --- a/django_mailbox/management/commands/processincomingmessage.py +++ b/django_mailbox/management/commands/processincomingmessage.py @@ -7,9 +7,11 @@ from django.core.management.base import BaseCommand from django_mailbox.models import Mailbox + logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) + class Command(BaseCommand): args = "<[Mailbox Name (optional)]>" command = "Receive incoming mail via stdin" @@ -22,14 +24,17 @@ class Command(BaseCommand): else: mailbox = self.get_mailbox_for_message(message) mailbox.process_incoming_message(message) - logger.info("Message received from %s" % message['from']) + logger.info( + "Message received from %s", + message['from'] + ) else: logger.warning("Message not processable.") def get_mailbox_by_name(self, name): mailbox, created = Mailbox.objects.get_or_create( - name=name, - ) + name=name, + ) return mailbox def get_mailbox_for_message(self, message): From 4c9402edd03cde3a11bc5754ac176736de1c7915 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 22 Aug 2014 20:04:53 -0700 Subject: [PATCH 50/86] Version 3.4.2a2. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fa10ddc..178e3fd 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='3.4.2a1', + version='3.4.2a2', url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 342c6af252f1c558343347a30a586c7c2bb223cf Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 22 Aug 2014 20:08:33 -0700 Subject: [PATCH 51/86] Version 3.4.2. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 178e3fd..3cbfc2e 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='3.4.2a2', + version='3.4.2', url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 4f400fab48d32663ae853eb367e989e9675b1ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ariel=20Gerardo=20R=C3=ADos?= Date: Fri, 29 Aug 2014 15:39:49 -0300 Subject: [PATCH 52/86] Add some spaces between constants and imports. --- django_mailbox/models.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index b126b74..c666046 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -1,27 +1,35 @@ -import base64 -import email +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Models declaration for application ``django_mailbox``. +""" + +from email.encoders import encode_base64 from email.message import Message as EmailMessage from email.utils import formatdate, parseaddr -from email.encoders import encode_base64 +from quopri import encode as encode_quopri +import base64 +import email import logging import mimetypes import os.path -from quopri import encode as encode_quopri import sys import uuid - -from django.conf import settings -from django.core.mail.message import make_msgid -from django.core.files.base import ContentFile -from django.db import models -from django_mailbox.transports import Pop3Transport, ImapTransport,\ - MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ - MMDFTransport, GmailImapTransport -from django_mailbox.signals import message_received import six from six.moves.urllib.parse import parse_qs, unquote, urlparse +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.mail.message import make_msgid +from django.db import models +from django.utils.translation import ugettext as _ + from .utils import convert_header_to_unicode +from django_mailbox.signals import message_received +from django_mailbox.transports import Pop3Transport, ImapTransport, \ + MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ + MMDFTransport, GmailImapTransport logger = logging.getLogger(__name__) @@ -32,6 +40,7 @@ STRIP_UNALLOWED_MIMETYPES = getattr( 'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES', False ) + ALLOWED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_ALLOWED_MIMETYPES', @@ -40,6 +49,7 @@ ALLOWED_MIMETYPES = getattr( 'text/html' ] ) + TEXT_STORED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_TEXT_STORED_MIMETYPES', @@ -48,11 +58,13 @@ TEXT_STORED_MIMETYPES = getattr( 'text/html' ] ) + ALTERED_MESSAGE_HEADER = getattr( settings, 'DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER', 'X-Django-Mailbox-Altered-Message' ) + ATTACHMENT_INTERPOLATION_HEADER = getattr( settings, 'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER', From 305d1c5c38f1ae0ec9ec492ee70bba5ab2547e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ariel=20Gerardo=20R=C3=ADos?= Date: Fri, 29 Aug 2014 15:40:22 -0300 Subject: [PATCH 53/86] Add translation support to field names and help texts. --- django_mailbox/models.py | 80 ++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index c666046..836c60f 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -16,6 +16,7 @@ import mimetypes import os.path import sys import uuid + import six from six.moves.urllib.parse import parse_qs, unquote, urlparse @@ -80,10 +81,15 @@ class ActiveMailboxManager(models.Manager): class Mailbox(models.Model): - name = models.CharField(max_length=255) - uri = models.CharField( + name = models.CharField( + _(u'Name'), max_length=255, - help_text=( + ) + + uri = models.CharField( + _(u'URI'), + max_length=255, + help_text=(_( "Example: imap+ssl://myusername:mypassword@someserver
" "
" "Internet transports include 'imap' and 'pop3'; " @@ -92,14 +98,16 @@ class Mailbox(models.Model): "
" "Be sure to urlencode your username and password should they " "contain illegal characters (like @, :, etc)." - ), + )), blank=True, null=True, default=None, ) + from_email = models.CharField( + _(u'From email'), max_length=255, - help_text=( + help_text=(_( "Example: MailBot <mailbot@yourdomain.com>
" "'From' header to set for outgoing email.
" "
" @@ -107,19 +115,21 @@ class Mailbox(models.Model): "setting is unnecessary.
" "If you send e-mail without setting this, your 'From' header will'" "be set to match the setting `DEFAULT_FROM_EMAIL`." - ), + )), blank=True, null=True, default=None, ) + active = models.BooleanField( - help_text=( + _(u'Active'), + help_text=(_( "Check this e-mail inbox for new e-mail messages during polling " "cycles. This checkbox does not have an effect upon whether " "mail is collected here when this mailbox receives mail from a " "pipe, and does not affect whether e-mail messages can be " "dispatched from this mailbox. " - ), + )), blank=True, default=True, ) @@ -362,34 +372,62 @@ class UnreadMessageManager(models.Manager): class Message(models.Model): - mailbox = models.ForeignKey(Mailbox, related_name='messages') - subject = models.CharField(max_length=255) - message_id = models.CharField(max_length=255) + mailbox = models.ForeignKey( + Mailbox, + related_name='messages', + verbose_name=_(u'Mailbox'), + ) + + subject = models.CharField( + _(u'Subject'), + max_length=255 + ) + + message_id = models.CharField( + _(u'Message ID'), + max_length=255 + ) + in_reply_to = models.ForeignKey( 'django_mailbox.Message', related_name='replies', blank=True, null=True, + verbose_name=_(u'In reply to'), ) + from_header = models.CharField( + _('From header'), max_length=255, ) - to_header = models.TextField() + + to_header = models.TextField( + _(u'To header'), + ) + outgoing = models.BooleanField( + _(u'Outgoing'), default=False, blank=True, ) - body = models.TextField() + body = models.TextField( + _(u'Body'), + ) + encoded = models.BooleanField( + _(u'Encoded'), default=False, - help_text='True if the e-mail body is Base64 encoded' + help_text=_('True if the e-mail body is Base64 encoded'), ) processed = models.DateTimeField( + _('Processed'), auto_now_add=True ) + read = models.DateTimeField( + _(u'Read'), default=None, blank=True, null=True, @@ -584,9 +622,19 @@ class MessageAttachment(models.Model): related_name='attachments', null=True, blank=True, + verbose_name=_('Message'), + ) + + headers = models.TextField( + _(u'Headers'), + null=True, + blank=True, + ) + + document = models.FileField( + _(u'Document'), + upload_to='mailbox_attachments/%Y/%m/%d/' ) - headers = models.TextField(null=True, blank=True) - document = models.FileField(upload_to='mailbox_attachments/%Y/%m/%d/') def delete(self, *args, **kwargs): self.document.delete() From ce7e17b50ccf370b6aab88fc4ce3fcee70fb3ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ariel=20Gerardo=20R=C3=ADos?= Date: Fri, 29 Aug 2014 17:24:38 -0300 Subject: [PATCH 54/86] Add file header and dummy HTML method. --- django_mailbox/admin.py | 12 ++++++++++++ django_mailbox/models.py | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/django_mailbox/admin.py b/django_mailbox/admin.py index 69df8dc..aa77b57 100644 --- a/django_mailbox/admin.py +++ b/django_mailbox/admin.py @@ -1,3 +1,11 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Model configuration in application ``django_mailbox`` for administration +console. +""" + import logging from django.conf import settings @@ -7,6 +15,7 @@ from django_mailbox.models import MessageAttachment, Message, Mailbox from django_mailbox.signals import message_received from django_mailbox.utils import convert_header_to_unicode + logger = logging.getLogger(__name__) @@ -21,6 +30,8 @@ def resend_message_received_signal(message_admin, request, queryset): for message in queryset.all(): logger.debug('Resending \'message_received\' signal for %s' % message) message_received.send(sender=message_admin, message=message) + + resend_message_received_signal.short_description = ( 'Re-send message received signal' ) @@ -79,6 +90,7 @@ class MessageAdmin(admin.ModelAdmin): ) readonly_fields = ( 'text', + 'html', ) actions = [resend_message_received_signal] diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 836c60f..3d33b71 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -499,6 +499,10 @@ class Message(models.Model): def text(self): return self.get_text_body() + @property + def html(self): + return "" + def get_text_body(self): def get_body_from_message(message): body = six.text_type('') From 6d73301f0d82bc8371eb51dba006b7aba5f15b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ariel=20Gerardo=20R=C3=ADos?= Date: Mon, 1 Sep 2014 15:30:30 -0300 Subject: [PATCH 55/86] Add property to fetch HTML body message. --- django_mailbox/models.py | 46 +++++++++++----------------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 3d33b71..67e44d6 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -26,7 +26,7 @@ from django.core.mail.message import make_msgid from django.db import models from django.utils.translation import ugettext as _ -from .utils import convert_header_to_unicode +from .utils import convert_header_to_unicode, get_body_from_message from django_mailbox.signals import message_received from django_mailbox.transports import Pop3Transport, ImapTransport, \ MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ @@ -497,43 +497,21 @@ class Message(models.Model): @property def text(self): - return self.get_text_body() + """ + Returns the message body matching content type 'text/plain'. + """ + return get_body_from_message( + self.get_email_object(), 'text', 'plain' + ).replace('=\n', '').strip() @property def html(self): - return "" - - def get_text_body(self): - def get_body_from_message(message): - body = six.text_type('') - for part in message.walk(): - if ( - part.get_content_maintype() == 'text' - and part.get_content_subtype() == 'plain' - ): - charset = part.get_content_charset() - this_part = part.get_payload(decode=True) - if charset: - this_part = this_part.decode(charset, 'replace') - - try: - body += this_part - except ValueError: - # Since it did not declare a charset, and we *should* - # be 7-bit clean right now, let's assume it is ASCII. - body += this_part.decode('ascii', 'replace') - logger.warning( - 'Error encountered while decoding text ' - 'payload from an incorrectly-constructed e-mail; ' - 'payload was converted to ASCII with ' - 'replacement, but some data may not be ' - 'represented as the sender intended.' - ) - return body - + """ + Returns the message body matching content type 'text/html'. + """ return get_body_from_message( - self.get_email_object() - ).replace('=\n', '').strip() + self.get_email_object(), 'text', 'html' + ).replace('\n', '').strip() def _rehydrate(self, msg): new = EmailMessage() From c1bfc9b3a8772418982faac0a682fdb1cd2c1322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ariel=20Gerardo=20R=C3=ADos?= Date: Mon, 1 Sep 2014 15:31:13 -0300 Subject: [PATCH 56/86] Extracts inner method to process multiple content types. --- django_mailbox/utils.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index 7d884eb..ff8c6e0 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -39,3 +39,34 @@ def convert_header_to_unicode(header): DEFAULT_CHARSET, ) return unicode(header, DEFAULT_CHARSET, 'replace') + + +def get_body_from_message(message, maintype, subtype): + """ + Fetchs the body message matching main/sub content type. + """ + body = six.text_type('') + for part in message.walk(): + if part.get_content_maintype() == maintype and \ + part.get_content_subtype() == subtype: + charset = part.get_content_charset() + this_part = part.get_payload(decode=True) + if charset: + this_part = this_part.decode(charset, 'replace') + + try: + body += this_part + except ValueError: + # Since it did not declare a charset, and we + # *should* be 7-bit clean right now, let's assume it + # is ASCII. + body += this_part.decode('ascii', 'replace') + logger.warning( + 'Error encountered while decoding text ' + 'payload from an incorrectly-constructed ' + 'e-mail; payload was converted to ASCII with ' + 'replacement, but some data may not be ' + 'represented as the sender intended.' + ) + + return body From 826d0b1d88e280cbe574ad32dbf6e5b4be23dfa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ariel=20Gerardo=20R=C3=ADos?= Date: Mon, 1 Sep 2014 15:31:29 -0300 Subject: [PATCH 57/86] Fix test methods for new accessors. --- django_mailbox/tests/test_process_email.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index b63b44d..c1c0955 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -65,7 +65,7 @@ class TestProcessEmail(EmailMessageTestCase): msg = mailbox.process_incoming_message(message) expected_results = 'Hello there!' - actual_results = msg.get_text_body().strip() + actual_results = msg.text.strip() self.assertEqual( expected_results, @@ -80,7 +80,7 @@ class TestProcessEmail(EmailMessageTestCase): message.get_email_object = lambda: email_object - actual_text = message.get_text_body() + actual_text = message.text expected_text = ( 'The one of us with a bike pump is far ahead, ' 'but a man stopped to help us and gave us his pump.' @@ -122,11 +122,11 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) - actual_text = msg.get_text_body() expected_text = six.u( 'This message contains funny UTF16 characters like this one: ' '"\xc2\xa0" and this one "\xe2\x9c\xbf".' ) + actual_text = msg.text self.assertEqual( expected_text, @@ -151,7 +151,7 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) - msg.get_text_body() + msg.text def test_message_with_valid_content_in_single_byte_encoding(self): email_object = self._get_email_object( @@ -160,8 +160,7 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) - actual_body = msg.get_text_body() - + actual_text = msg.text expected_body = six.u( '\u042d\u0442\u043e ' '\u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0435 ' @@ -172,7 +171,7 @@ class TestProcessEmail(EmailMessageTestCase): ) self.assertEqual( - actual_body, + actual_text, expected_body, ) From 5c8385de21d98b1e4c89d3fb24cb267215b795f9 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 1 Sep 2014 12:57:00 -0700 Subject: [PATCH 58/86] Dropping support for Python 3.2; the whole unicode literal thing has been a little troublesome over the years. --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5cf4dc1..59295ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - "2.6" - "2.7" - - "3.2" - "3.3" env: - DJANGO=1.5.6 From 906011416db76f4320a5e1b454d59b5c9b714541 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 1 Sep 2014 13:02:11 -0700 Subject: [PATCH 59/86] Updating supported django versions. --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 59295ba..5f45d6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,9 @@ python: - "2.7" - "3.3" env: - - DJANGO=1.5.6 - - DJANGO=1.6.3 + - DJANGO=1.4.14 + - DJANGO=1.5.9 + - DJANGO=1.6.6 install: - pip install -q Django==$DJANGO --use-mirrors - pip install -q -e . --use-mirrors From ee91ad7d28b26bd47976f0bb1b8002e0d9dee7ee Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 1 Sep 2014 13:07:39 -0700 Subject: [PATCH 60/86] Exclude Django 1.4.14 with Python 3k. --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5f45d6c..9644893 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,10 @@ env: - DJANGO=1.4.14 - DJANGO=1.5.9 - DJANGO=1.6.6 +matrix: + exclude: + - env: DJANGO=1.4.14 + python: "3.3" install: - pip install -q Django==$DJANGO --use-mirrors - pip install -q -e . --use-mirrors From 13eeb74234c65e2a33ae331f6c076e2c1072efc7 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 1 Sep 2014 13:35:37 -0700 Subject: [PATCH 61/86] Releasing version 4.0. --- CHANGELOG.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..64df677 --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,51 @@ +Changelog +========= + +4.0 +--- + +* Adds ``html`` property returning the HTML contents of + ``django_mailbox.models.Message`` instances. + Thanks `@ariel17 `_! +* Adds translation support. + Thanks `@ariel17 `_! +* **Drops support for Python 3.2**. The fact that only versions of + Python newer than 3.2 allow unicode literals has convinced me + that supporting Python 3.2 is probably more trouble than it's worth. + Please let me know if you were using Python 3.2, and I've left you + out in the cold; I'm willing to fix Python 3.2 support if it is + actively used. + +3.4 +--- + +* Adds ``gmail`` transport allowing one to use Google + OAuth credentials for gathering messages from gmail. + Thanks `@alexlovelltroy `_! + +3.3 +--- + +* Adds functionality to ``imap`` transport allowing one to + archive processed e-mails. + Thanks `@yellowcap `_! + +3.2 +--- + +* Fixes `#13 `_; + Python 3 support had been broken for some time. Thanks for catching that, + `@greendee `_! + +3.1 +--- + +* Fixes a wide variety of unicode-related errors. + +3.0 +--- + +* Restructures message storage such that non-text message attachments + are stored as files, rather than in the database in their original + (probably base64-encoded) blobs. +* So many new tests. diff --git a/setup.py b/setup.py index 3cbfc2e..7b640dd 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='3.4.2', + version='4.0', url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From f44b4bbc4685a76b44cf3b16fda82ba40580def5 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 1 Sep 2014 13:38:02 -0700 Subject: [PATCH 62/86] Adding a couple new badges. --- readme.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/readme.rst b/readme.rst index 9aa09bc..96db909 100644 --- a/readme.rst +++ b/readme.rst @@ -1,6 +1,13 @@ .. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master :target: https://travis-ci.org/coddingtonbear/django-mailbox +.. image:: https://badge.fury.io/py/django-mailbox.png + :target: http://badge.fury.io/py/django-mailbox + +.. image:: https://pypip.in/d/django-mailbox/badge.png + :target: https://pypi.python.org/pypi/django-mailbox + + Easily pull messages from POP3, IMAP, or local mailboxes into Django models. This Django application will allow you to specify mailboxes that you would like consumed for incoming content; From bd56716e74129dd26c4996e2db03bd1df9f37f3f Mon Sep 17 00:00:00 2001 From: Will Date: Sat, 6 Sep 2014 16:31:12 -0700 Subject: [PATCH 63/86] Fix deprecation warnings after Django 1.7 upgrade --- django_mailbox/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 67e44d6..2d9eeff 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -74,7 +74,7 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr( class ActiveMailboxManager(models.Manager): - def get_query_set(self): + def get_queryset(self): return super(ActiveMailboxManager, self).get_query_set().filter( active=True, ) @@ -351,21 +351,21 @@ class Mailbox(models.Model): class IncomingMessageManager(models.Manager): - def get_query_set(self): + def get_queryset(self): return super(IncomingMessageManager, self).get_query_set().filter( outgoing=False, ) class OutgoingMessageManager(models.Manager): - def get_query_set(self): + def get_queryset(self): return super(OutgoingMessageManager, self).get_query_set().filter( outgoing=True, ) class UnreadMessageManager(models.Manager): - def get_query_set(self): + def get_queryset(self): return super(UnreadMessageManager, self).get_query_set().filter( read=None ) From 1406d465d6dc045930a0284552038bd30bbbc29c Mon Sep 17 00:00:00 2001 From: Will Date: Sat, 6 Sep 2014 19:18:47 -0700 Subject: [PATCH 64/86] Adjust usages of get_query_set to get_queryset --- django_mailbox/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 2d9eeff..e36bf00 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -75,7 +75,7 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr( class ActiveMailboxManager(models.Manager): def get_queryset(self): - return super(ActiveMailboxManager, self).get_query_set().filter( + return super(ActiveMailboxManager, self).get_queryset().filter( active=True, ) @@ -352,21 +352,21 @@ class Mailbox(models.Model): class IncomingMessageManager(models.Manager): def get_queryset(self): - return super(IncomingMessageManager, self).get_query_set().filter( + return super(IncomingMessageManager, self).get_queryset().filter( outgoing=False, ) class OutgoingMessageManager(models.Manager): def get_queryset(self): - return super(OutgoingMessageManager, self).get_query_set().filter( + return super(OutgoingMessageManager, self).get_queryset().filter( outgoing=True, ) class UnreadMessageManager(models.Manager): def get_queryset(self): - return super(UnreadMessageManager, self).get_query_set().filter( + return super(UnreadMessageManager, self).get_queryset().filter( read=None ) From 7e424d45313d004e85de7e6d06b403da3f9598c6 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 7 Sep 2014 12:21:48 -0700 Subject: [PATCH 65/86] Release 4.0.1. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7b640dd..2d36d86 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='4.0', + version='4.0.1', url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 684f124c454c4d81f42936fb4e5cf23564dcc9e5 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 8 Sep 2014 19:45:32 -0700 Subject: [PATCH 66/86] Rewriting readme. --- readme.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/readme.rst b/readme.rst index 96db909..65be1da 100644 --- a/readme.rst +++ b/readme.rst @@ -8,10 +8,13 @@ :target: https://pypi.python.org/pypi/django-mailbox -Easily pull messages from POP3, IMAP, or local mailboxes into Django models. +Easily ingest messages from POP3, IMAP, or local mailboxes into your Django application. -This Django application will allow you to specify mailboxes that you would like consumed for incoming content; -the e-mail will be stored in the database, and you can process it at will (or, if you're in a hurry, by subscribing to a signal). +This app allows you to either ingest e-mail content from common e-mail services (as long as the service provides POP3 or IMAP support), +or directly recieve e-mail messages from stdin (for locally processing messages from Postfix or Exim4). + +These ingested messages will be stored in the database as Django models and you can process their content at will, +or if you're in a hurry, by subscribing to a signal. - Documentation for django-mailbox is available on `ReadTheDocs `_. From 079a8e52e1c60cb8fbc91ee302029dfba40dd18e Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 8 Sep 2014 19:47:28 -0700 Subject: [PATCH 67/86] Minor wording and formatting changes. --- readme.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.rst b/readme.rst index 65be1da..f32bb6b 100644 --- a/readme.rst +++ b/readme.rst @@ -11,10 +11,10 @@ Easily ingest messages from POP3, IMAP, or local mailboxes into your Django application. This app allows you to either ingest e-mail content from common e-mail services (as long as the service provides POP3 or IMAP support), -or directly recieve e-mail messages from stdin (for locally processing messages from Postfix or Exim4). +or directly recieve e-mail messages from ``stdin`` (for locally processing messages from Postfix or Exim4). These ingested messages will be stored in the database as Django models and you can process their content at will, -or if you're in a hurry, by subscribing to a signal. +or -- if you're in a hurry -- by using a signal receiver. - Documentation for django-mailbox is available on `ReadTheDocs `_. From 9cd99456371abdf1891c12711f6d87514d72944c Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 9 Sep 2014 15:01:22 -0700 Subject: [PATCH 68/86] Removing bit.ly icon. --- readme.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/readme.rst b/readme.rst index f32bb6b..213ee5f 100644 --- a/readme.rst +++ b/readme.rst @@ -22,10 +22,3 @@ or -- if you're in a hurry -- by using a signal receiver. `Github `_. - Test status available on `Travis-CI `_. - - - -.. image:: https://d2weczhvl823v0.cloudfront.net/coddingtonbear/django-mailbox/trend.png - :alt: Bitdeli badge - :target: https://bitdeli.com/free - From 9730c34c7f4823cc400c39a5accec14c50dd1c3b Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Wed, 10 Sep 2014 14:38:12 -0700 Subject: [PATCH 69/86] This is a better preposition. --- readme.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.rst b/readme.rst index 213ee5f..2a75204 100644 --- a/readme.rst +++ b/readme.rst @@ -13,7 +13,7 @@ Easily ingest messages from POP3, IMAP, or local mailboxes into your Django appl This app allows you to either ingest e-mail content from common e-mail services (as long as the service provides POP3 or IMAP support), or directly recieve e-mail messages from ``stdin`` (for locally processing messages from Postfix or Exim4). -These ingested messages will be stored in the database as Django models and you can process their content at will, +These ingested messages will be stored in the database in Django models and you can process their content at will, or -- if you're in a hurry -- by using a signal receiver. - Documentation for django-mailbox is available on From f8e70aa5d19547c033d7e693f73e69d10abd5bab Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 14 Jan 2014 22:20:12 -0800 Subject: [PATCH 70/86] Attempting to narrow down the issue @rulzart is encountering. Conflicts: django_mailbox/transports/generic.py --- django_mailbox/transports/generic.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_mailbox/transports/generic.py b/django_mailbox/transports/generic.py index 5a9562d..cf4bd78 100644 --- a/django_mailbox/transports/generic.py +++ b/django_mailbox/transports/generic.py @@ -1,3 +1,5 @@ +import sys + from .base import EmailTransport @@ -7,7 +9,9 @@ class GenericFileMailbox(EmailTransport): def __init__(self, path): super(GenericFileMailbox, self).__init__() - self._path = path + self._path = path.encode( + sys.getfilesystemencoding() + ) def get_instance(self): return self._variant(self._path) From ecc8ac8bcc37ca24a47754ac707491fd36e91e5f Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 14 Oct 2014 19:33:08 -0700 Subject: [PATCH 71/86] Release 4.0.2. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d36d86..6da4b61 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='4.0.1', + version='4.0.2', url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From 2d82c00c5b56fd7a3b2b8a3d7ed8efc123bd71f0 Mon Sep 17 00:00:00 2001 From: Patrick Craston Date: Fri, 7 Nov 2014 16:56:24 +0000 Subject: [PATCH 72/86] convert migrations to work with Django 1.7, add Django 1.7.1 to travis test matrix --- .travis.yml | 34 ++++-- django_mailbox/migrations/0001_initial.py | 108 +++++++++--------- django_mailbox/runtests.py | 3 + .../south_migrations/0001_initial.py | 59 ++++++++++ .../0002_auto__chg_field_mailbox_uri.py | 0 .../0003_auto__add_field_mailbox_active.py | 0 .../0004_auto__add_field_message_outgoing.py | 0 .../0005_rename_fields.py | 0 ...006_auto__add_field_message_in_reply_to.py | 0 ...__add_field_message_from_header__add_fi.py | 0 .../0008_populate_from_to_fields.py | 0 .../0009_remove_references_table.py | 0 ...0010_auto__add_field_mailbox_from_email.py | 0 .../0011_auto__add_field_message_read.py | 0 .../0012_auto__add_messageattachment.py | 0 ...to__add_field_messageattachment_message.py | 0 .../0014_migrate_message_attachments.py | 0 ...to__add_field_messageattachment_headers.py | 0 .../0016_auto__add_field_message_encoded.py | 0 django_mailbox/south_migrations/__init__.py | 0 .../tests/messages/generic_message.eml | 1 - .../messages/message_with_attachment.eml | 2 +- 22 files changed, 139 insertions(+), 68 deletions(-) create mode 100644 django_mailbox/south_migrations/0001_initial.py rename django_mailbox/{migrations => south_migrations}/0002_auto__chg_field_mailbox_uri.py (100%) rename django_mailbox/{migrations => south_migrations}/0003_auto__add_field_mailbox_active.py (100%) rename django_mailbox/{migrations => south_migrations}/0004_auto__add_field_message_outgoing.py (100%) rename django_mailbox/{migrations => south_migrations}/0005_rename_fields.py (100%) rename django_mailbox/{migrations => south_migrations}/0006_auto__add_field_message_in_reply_to.py (100%) rename django_mailbox/{migrations => south_migrations}/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py (100%) rename django_mailbox/{migrations => south_migrations}/0008_populate_from_to_fields.py (100%) rename django_mailbox/{migrations => south_migrations}/0009_remove_references_table.py (100%) rename django_mailbox/{migrations => south_migrations}/0010_auto__add_field_mailbox_from_email.py (100%) rename django_mailbox/{migrations => south_migrations}/0011_auto__add_field_message_read.py (100%) rename django_mailbox/{migrations => south_migrations}/0012_auto__add_messageattachment.py (100%) rename django_mailbox/{migrations => south_migrations}/0013_auto__add_field_messageattachment_message.py (100%) rename django_mailbox/{migrations => south_migrations}/0014_migrate_message_attachments.py (100%) rename django_mailbox/{migrations => south_migrations}/0015_auto__add_field_messageattachment_headers.py (100%) rename django_mailbox/{migrations => south_migrations}/0016_auto__add_field_message_encoded.py (100%) create mode 100644 django_mailbox/south_migrations/__init__.py diff --git a/.travis.yml b/.travis.yml index 9644893..cb03bb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,28 @@ language: python + python: - - "2.6" - - "2.7" - - "3.3" + - "2.6" + - "2.7" + - "3.3" + env: - - DJANGO=1.4.14 - - DJANGO=1.5.9 - - DJANGO=1.6.6 + - DJANGO=1.4.14 + - DJANGO=1.5.9 + - DJANGO=1.6.8 + - DJANGO=1.7.1 + matrix: - exclude: - - env: DJANGO=1.4.14 - python: "3.3" + allow_failures: + # Allow failures for legacy django versions + - env: DJANGO=1.4.14 + - env: DJANGO=1.5.9 + exclude: + - env: DJANGO=1.4.14 + python: "3.3" + - env: DJANGO=1.7.1 + python: "2.6" install: - - pip install -q Django==$DJANGO --use-mirrors - - pip install -q -e . --use-mirrors + - pip install -q Django==$DJANGO --use-mirrors + - pip install -q -e . --use-mirrors script: - - python setup.py test + - python setup.py test diff --git a/django_mailbox/migrations/0001_initial.py b/django_mailbox/migrations/0001_initial.py index 4faa4fa..dfb6ffa 100644 --- a/django_mailbox/migrations/0001_initial.py +++ b/django_mailbox/migrations/0001_initial.py @@ -1,59 +1,59 @@ # -*- coding: utf-8 -*- -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models +from __future__ import unicode_literals + +from django.db import models, migrations -class Migration(SchemaMigration): +class Migration(migrations.Migration): - def forwards(self, orm): - # Adding model 'Mailbox' - db.create_table('django_mailbox_mailbox', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('name', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)), - )) - db.send_create_signal('django_mailbox', ['Mailbox']) + dependencies = [ + ] - # Adding model 'Message' - db.create_table('django_mailbox_message', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('mailbox', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['django_mailbox.Mailbox'])), - ('subject', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('message_id', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('from_address', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('body', self.gf('django.db.models.fields.TextField')()), - ('received', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), - )) - db.send_create_signal('django_mailbox', ['Message']) - - - def backwards(self, orm): - # Deleting model 'Mailbox' - db.delete_table('django_mailbox_mailbox') - - # Deleting model 'Message' - db.delete_table('django_mailbox_message') - - - models = { - 'django_mailbox.mailbox': { - 'Meta': {'object_name': 'Mailbox'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'django_mailbox.message': { - 'Meta': {'object_name': 'Message'}, - 'body': ('django.db.models.fields.TextField', [], {}), - 'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['django_mailbox.Mailbox']"}), - 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - } - } - - complete_apps = ['django_mailbox'] \ No newline at end of file + operations = [ + migrations.CreateModel( + name='Mailbox', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255, verbose_name='Name')), + ('uri', models.CharField(default=None, max_length=255, blank=True, help_text="Example: imap+ssl://myusername:mypassword@someserver

Internet transports include 'imap' and 'pop3'; common local file transports include 'maildir', 'mbox', and less commonly 'babyl', 'mh', and 'mmdf'.

Be sure to urlencode your username and password should they contain illegal characters (like @, :, etc).", null=True, verbose_name='URI')), + ('from_email', models.CharField(default=None, max_length=255, blank=True, help_text="Example: MailBot <mailbot@yourdomain.com>
'From' header to set for outgoing email.

If you do not use this e-mail inbox for outgoing mail, this setting is unnecessary.
If you send e-mail without setting this, your 'From' header will'be set to match the setting `DEFAULT_FROM_EMAIL`.", null=True, verbose_name='From email')), + ('active', models.BooleanField(default=True, help_text='Check this e-mail inbox for new e-mail messages during polling cycles. This checkbox does not have an effect upon whether mail is collected here when this mailbox receives mail from a pipe, and does not affect whether e-mail messages can be dispatched from this mailbox. ', verbose_name='Active')), + ], + options={ + 'verbose_name_plural': 'Mailboxes', + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Message', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('subject', models.CharField(max_length=255, verbose_name='Subject')), + ('message_id', models.CharField(max_length=255, verbose_name='Message ID')), + ('from_header', models.CharField(max_length=255, verbose_name='From header')), + ('to_header', models.TextField(verbose_name='To header')), + ('outgoing', models.BooleanField(default=False, verbose_name='Outgoing')), + ('body', models.TextField(verbose_name='Body')), + ('encoded', models.BooleanField(default=False, help_text='True if the e-mail body is Base64 encoded', verbose_name='Encoded')), + ('processed', models.DateTimeField(auto_now_add=True, verbose_name='Processed')), + ('read', models.DateTimeField(default=None, null=True, verbose_name='Read', blank=True)), + ('in_reply_to', models.ForeignKey(related_name='replies', verbose_name='In reply to', blank=True, to='django_mailbox.Message', null=True)), + ('mailbox', models.ForeignKey(related_name='messages', verbose_name='Mailbox', to='django_mailbox.Mailbox')), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='MessageAttachment', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('headers', models.TextField(null=True, verbose_name='Headers', blank=True)), + ('document', models.FileField(upload_to=b'mailbox_attachments/%Y/%m/%d/', verbose_name='Document')), + ('message', models.ForeignKey(related_name='attachments', verbose_name='Message', blank=True, to='django_mailbox.Message', null=True)), + ], + options={ + }, + bases=(models.Model,), + ), + ] diff --git a/django_mailbox/runtests.py b/django_mailbox/runtests.py index 62094fd..068871a 100644 --- a/django_mailbox/runtests.py +++ b/django_mailbox/runtests.py @@ -3,6 +3,7 @@ import sys from os.path import dirname, abspath +from django import setup from django.conf import settings if not settings.configured: @@ -27,6 +28,8 @@ def runtests(*test_args): test_args = ['django_mailbox'] parent = dirname(abspath(__file__)) sys.path.insert(0, parent) + # ensure that AppRegistry has loaded + setup() runner = DjangoTestSuiteRunner( verbosity=1, interactive=False, diff --git a/django_mailbox/south_migrations/0001_initial.py b/django_mailbox/south_migrations/0001_initial.py new file mode 100644 index 0000000..4faa4fa --- /dev/null +++ b/django_mailbox/south_migrations/0001_initial.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Mailbox' + db.create_table('django_mailbox_mailbox', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)), + )) + db.send_create_signal('django_mailbox', ['Mailbox']) + + # Adding model 'Message' + db.create_table('django_mailbox_message', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('mailbox', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['django_mailbox.Mailbox'])), + ('subject', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('message_id', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('from_address', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('body', self.gf('django.db.models.fields.TextField')()), + ('received', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + )) + db.send_create_signal('django_mailbox', ['Message']) + + + def backwards(self, orm): + # Deleting model 'Mailbox' + db.delete_table('django_mailbox_mailbox') + + # Deleting model 'Message' + db.delete_table('django_mailbox_message') + + + models = { + 'django_mailbox.mailbox': { + 'Meta': {'object_name': 'Mailbox'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'django_mailbox.message': { + 'Meta': {'object_name': 'Message'}, + 'body': ('django.db.models.fields.TextField', [], {}), + 'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['django_mailbox.Mailbox']"}), + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['django_mailbox'] \ No newline at end of file diff --git a/django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py b/django_mailbox/south_migrations/0002_auto__chg_field_mailbox_uri.py similarity index 100% rename from django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py rename to django_mailbox/south_migrations/0002_auto__chg_field_mailbox_uri.py diff --git a/django_mailbox/migrations/0003_auto__add_field_mailbox_active.py b/django_mailbox/south_migrations/0003_auto__add_field_mailbox_active.py similarity index 100% rename from django_mailbox/migrations/0003_auto__add_field_mailbox_active.py rename to django_mailbox/south_migrations/0003_auto__add_field_mailbox_active.py diff --git a/django_mailbox/migrations/0004_auto__add_field_message_outgoing.py b/django_mailbox/south_migrations/0004_auto__add_field_message_outgoing.py similarity index 100% rename from django_mailbox/migrations/0004_auto__add_field_message_outgoing.py rename to django_mailbox/south_migrations/0004_auto__add_field_message_outgoing.py diff --git a/django_mailbox/migrations/0005_rename_fields.py b/django_mailbox/south_migrations/0005_rename_fields.py similarity index 100% rename from django_mailbox/migrations/0005_rename_fields.py rename to django_mailbox/south_migrations/0005_rename_fields.py diff --git a/django_mailbox/migrations/0006_auto__add_field_message_in_reply_to.py b/django_mailbox/south_migrations/0006_auto__add_field_message_in_reply_to.py similarity index 100% rename from django_mailbox/migrations/0006_auto__add_field_message_in_reply_to.py rename to django_mailbox/south_migrations/0006_auto__add_field_message_in_reply_to.py diff --git a/django_mailbox/migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py b/django_mailbox/south_migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py similarity index 100% rename from django_mailbox/migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py rename to django_mailbox/south_migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py diff --git a/django_mailbox/migrations/0008_populate_from_to_fields.py b/django_mailbox/south_migrations/0008_populate_from_to_fields.py similarity index 100% rename from django_mailbox/migrations/0008_populate_from_to_fields.py rename to django_mailbox/south_migrations/0008_populate_from_to_fields.py diff --git a/django_mailbox/migrations/0009_remove_references_table.py b/django_mailbox/south_migrations/0009_remove_references_table.py similarity index 100% rename from django_mailbox/migrations/0009_remove_references_table.py rename to django_mailbox/south_migrations/0009_remove_references_table.py diff --git a/django_mailbox/migrations/0010_auto__add_field_mailbox_from_email.py b/django_mailbox/south_migrations/0010_auto__add_field_mailbox_from_email.py similarity index 100% rename from django_mailbox/migrations/0010_auto__add_field_mailbox_from_email.py rename to django_mailbox/south_migrations/0010_auto__add_field_mailbox_from_email.py diff --git a/django_mailbox/migrations/0011_auto__add_field_message_read.py b/django_mailbox/south_migrations/0011_auto__add_field_message_read.py similarity index 100% rename from django_mailbox/migrations/0011_auto__add_field_message_read.py rename to django_mailbox/south_migrations/0011_auto__add_field_message_read.py diff --git a/django_mailbox/migrations/0012_auto__add_messageattachment.py b/django_mailbox/south_migrations/0012_auto__add_messageattachment.py similarity index 100% rename from django_mailbox/migrations/0012_auto__add_messageattachment.py rename to django_mailbox/south_migrations/0012_auto__add_messageattachment.py diff --git a/django_mailbox/migrations/0013_auto__add_field_messageattachment_message.py b/django_mailbox/south_migrations/0013_auto__add_field_messageattachment_message.py similarity index 100% rename from django_mailbox/migrations/0013_auto__add_field_messageattachment_message.py rename to django_mailbox/south_migrations/0013_auto__add_field_messageattachment_message.py diff --git a/django_mailbox/migrations/0014_migrate_message_attachments.py b/django_mailbox/south_migrations/0014_migrate_message_attachments.py similarity index 100% rename from django_mailbox/migrations/0014_migrate_message_attachments.py rename to django_mailbox/south_migrations/0014_migrate_message_attachments.py diff --git a/django_mailbox/migrations/0015_auto__add_field_messageattachment_headers.py b/django_mailbox/south_migrations/0015_auto__add_field_messageattachment_headers.py similarity index 100% rename from django_mailbox/migrations/0015_auto__add_field_messageattachment_headers.py rename to django_mailbox/south_migrations/0015_auto__add_field_messageattachment_headers.py diff --git a/django_mailbox/migrations/0016_auto__add_field_message_encoded.py b/django_mailbox/south_migrations/0016_auto__add_field_message_encoded.py similarity index 100% rename from django_mailbox/migrations/0016_auto__add_field_message_encoded.py rename to django_mailbox/south_migrations/0016_auto__add_field_message_encoded.py diff --git a/django_mailbox/south_migrations/__init__.py b/django_mailbox/south_migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_mailbox/tests/messages/generic_message.eml b/django_mailbox/tests/messages/generic_message.eml index b3c9f33..bf5f0fe 100644 --- a/django_mailbox/tests/messages/generic_message.eml +++ b/django_mailbox/tests/messages/generic_message.eml @@ -1,5 +1,4 @@ MIME-Version: 1.0 -Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 11:53:53 -0800 (PST) X-Originating-IP: [24.22.122.177] Date: Sun, 20 Jan 2013 11:53:53 -0800 Delivered-To: test@adamcoddington.net diff --git a/django_mailbox/tests/messages/message_with_attachment.eml b/django_mailbox/tests/messages/message_with_attachment.eml index 34e7093..ac1cc43 100644 --- a/django_mailbox/tests/messages/message_with_attachment.eml +++ b/django_mailbox/tests/messages/message_with_attachment.eml @@ -17,8 +17,8 @@ This message has an attachment. --047d7b33dd729737fe04d3bde348 Content-Type: image/png; name="heart.png" Content-Disposition: attachment; filename="heart.png" -Content-Transfer-Encoding: base64 X-Attachment-Id: f_hc6mair60 +Content-Transfer-Encoding: base64 iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAFoTx1HAAAAzUlEQVQoz32RWxXDIBBEr4NIQEIl ICESkFAJkRAJSIgEpEQCEqYfu6QUkn7sCcyDGQiSACKSKCAkGwBJwhDwZQNMEiYAIBdQvk7rfaHf From 4eae6faf031188483e67ef3b198988788719f49e Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 7 Nov 2014 19:12:30 -0800 Subject: [PATCH 73/86] Given that 'classic' versions of django do not have a setup function: allow that to fail. --- django_mailbox/runtests.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/django_mailbox/runtests.py b/django_mailbox/runtests.py index 068871a..f5b0ff3 100644 --- a/django_mailbox/runtests.py +++ b/django_mailbox/runtests.py @@ -3,7 +3,10 @@ import sys from os.path import dirname, abspath -from django import setup +try: + from django import setup +except ImportError: + pass from django.conf import settings if not settings.configured: @@ -28,8 +31,12 @@ def runtests(*test_args): test_args = ['django_mailbox'] parent = dirname(abspath(__file__)) sys.path.insert(0, parent) - # ensure that AppRegistry has loaded - setup() + try: + # ensure that AppRegistry has loaded + setup() + except NameError: + # This version of Django is too old for an app registry. + pass runner = DjangoTestSuiteRunner( verbosity=1, interactive=False, From e4fb22a7f1ba91ce0ed10488864471483ba59651 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 7 Nov 2014 19:17:06 -0800 Subject: [PATCH 74/86] Let's still fail a build if we break things for Django 1.4 and 1.5; lots of people are trapped on old versions. --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb03bb1..82aaddd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,6 @@ env: - DJANGO=1.7.1 matrix: - allow_failures: - # Allow failures for legacy django versions - - env: DJANGO=1.4.14 - - env: DJANGO=1.5.9 exclude: - env: DJANGO=1.4.14 python: "3.3" From 2e23521220171530d9af4ff82cc3923188f6c99f Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sat, 8 Nov 2014 14:23:07 -0800 Subject: [PATCH 75/86] Release 4.1; adding Django 1.7 migrations support. --- CHANGELOG.rst | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 64df677..c88f41c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Changelog ========= +4.1 +--- + +* Adds Django 1.7 migrations support. + 4.0 --- diff --git a/setup.py b/setup.py index 6da4b61..5885062 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='4.0.2', + version='4.1', url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' From df4c7196db58ed5246b63ca1ce06b5e6919418c5 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 9 Nov 2014 21:55:21 -0800 Subject: [PATCH 76/86] Fix bug preventing south 1.0 from properly finding legacy migrations. Release 4.1.1 --- django_mailbox/__init__.py | 1 + setup.py | 15 +++++---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/django_mailbox/__init__.py b/django_mailbox/__init__.py index e69de29..47cbba7 100644 --- a/django_mailbox/__init__.py +++ b/django_mailbox/__init__.py @@ -0,0 +1 @@ +__version__ = '4.1.1' diff --git a/setup.py b/setup.py index 5885062..3023819 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,6 @@ -from setuptools import setup +from setuptools import find_packages, setup + +from django_mailbox import __version__ as version_string tests_require = [ 'django', @@ -11,7 +13,7 @@ gmail_oauth2_require = [ setup( name='django-mailbox', - version='4.1', + version=version_string, url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' @@ -40,14 +42,7 @@ setup( 'Topic :: Communications :: Email :: Post-Office :: POP3', 'Topic :: Communications :: Email :: Email Clients (MUA)', ], - packages=[ - 'django_mailbox', - 'django_mailbox.management', - 'django_mailbox.management.commands', - 'django_mailbox.migrations', - 'django_mailbox.transports', - 'django_mailbox.tests', - ], + packages=find_packages(), install_requires=[ 'six>=1.6.1' ] From e1b0763a46333ba4668b014bc6142ce072e7b1dc Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 10 Nov 2014 19:20:17 -0800 Subject: [PATCH 77/86] Store 'Delivered-To' as 'to_header' when 'to' is unspecified. --- django_mailbox/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index e36bf00..3c0a5c0 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -320,6 +320,8 @@ class Mailbox(models.Model): msg.from_header = convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = convert_header_to_unicode(message['to']) + elif 'Delivered-To' in message: + msg.to_header = convert_header_to_unicode(message['Delivered-To']) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) From aa59199c9b98ed317c6c95dc4018e21d1302858c Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sat, 15 Nov 2014 22:15:25 -0800 Subject: [PATCH 78/86] Interpret unknown encodings as ASCII. Fixes #34. --- django_mailbox/models.py | 11 ++++++ .../message_with_invalid_encoding.eml | 13 +++++++ .../tests/test_message_flattening.py | 17 +++++++++ django_mailbox/utils.py | 38 +++++++++++-------- 4 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 django_mailbox/tests/messages/message_with_invalid_encoding.eml diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 3c0a5c0..2f9dd13 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -299,6 +299,17 @@ class Mailbox(models.Model): # defined charset, if it can't, let's mash some things # inside the payload :-\ msg.get_payload(decode=True).decode(content_charset) + except LookupError: + logger.exception( + "Unknown encoding %s; interpreting as ascii!", + content_charset + ) + msg.set_payload( + msg.get_payload(decode=True).decode( + 'ascii', + 'ignore' + ) + ) except UnicodeDecodeError: msg.set_payload( msg.get_payload(decode=True).decode( diff --git a/django_mailbox/tests/messages/message_with_invalid_encoding.eml b/django_mailbox/tests/messages/message_with_invalid_encoding.eml new file mode 100644 index 0000000..fbe1f9f --- /dev/null +++ b/django_mailbox/tests/messages/message_with_invalid_encoding.eml @@ -0,0 +1,13 @@ +Reply-To: +From: "Refinance" +Subject: Apply For Loans @ 2% Per Annum +Date: Sun, 19 Oct 2014 11:48:49 +0100 +MIME-Version: 1.0 +Content-Type: text/plain; + charset="_iso-2022-jp$ESC" +Content-Transfer-Encoding: 7bit + +We offer loans to private individuals and corporate organizations at 2% interest rate. Interested serious applicants should apply via email with details of their requirements. + +Warm Regards, +Loan Team diff --git a/django_mailbox/tests/test_message_flattening.py b/django_mailbox/tests/test_message_flattening.py index 9574701..1399a34 100644 --- a/django_mailbox/tests/test_message_flattening.py +++ b/django_mailbox/tests/test_message_flattening.py @@ -85,3 +85,20 @@ class TestMessageFlattening(EmailMessageTestCase): actual_email_object, expected_email_object, ) + + def test_message_processing_unknown_encoding(self): + incoming_email_object = self._get_email_object( + 'message_with_invalid_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(incoming_email_object) + + expected_text = ( + "We offer loans to private individuals and corporate " + "organizations at 2% interest rate. Interested serious " + "applicants should apply via email with details of their " + "requirements.\n\nWarm Regards,\nLoan Team" + ) + actual_text = msg.text + + self.assertEqual(actual_text, expected_text) diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index ff8c6e0..9a802e1 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -52,21 +52,29 @@ def get_body_from_message(message, maintype, subtype): charset = part.get_content_charset() this_part = part.get_payload(decode=True) if charset: - this_part = this_part.decode(charset, 'replace') + try: + this_part = this_part.decode(charset, 'replace') + except LookupError: + this_part = this_part.decode('ascii', 'replace') + logger.warning( + 'Unknown encoding %s encountered while decoding ' + 'text payload. Interpreting as ASCII with ' + 'replacement, but some data may not be ' + 'represented as the sender intended.', + charset + ) + except ValueError: + this_part = this_part.decode('ascii', 'replace') + logger.warning( + 'Error encountered while decoding text ' + 'payload from an incorrectly-constructed ' + 'e-mail; payload was converted to ASCII with ' + 'replacement, but some data may not be ' + 'represented as the sender intended.' + ) + else: + this_part = this_part.decode('ascii', 'replace') - try: - body += this_part - except ValueError: - # Since it did not declare a charset, and we - # *should* be 7-bit clean right now, let's assume it - # is ASCII. - body += this_part.decode('ascii', 'replace') - logger.warning( - 'Error encountered while decoding text ' - 'payload from an incorrectly-constructed ' - 'e-mail; payload was converted to ASCII with ' - 'replacement, but some data may not be ' - 'represented as the sender intended.' - ) + body += this_part return body From f16d4f6d9c27c568e196de24287e3e7090bf4cae Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sat, 15 Nov 2014 22:18:19 -0800 Subject: [PATCH 79/86] Minor tweaks to logging. --- django_mailbox/models.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 2f9dd13..981883d 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -300,8 +300,8 @@ class Mailbox(models.Model): # inside the payload :-\ msg.get_payload(decode=True).decode(content_charset) except LookupError: - logger.exception( - "Unknown encoding %s; interpreting as ascii!", + logger.warning( + "Unknown encoding %s; interpreting as ASCII!", content_charset ) msg.set_payload( @@ -310,7 +310,11 @@ class Mailbox(models.Model): 'ignore' ) ) - except UnicodeDecodeError: + except ValueError: + logger.warning( + "Decoding error encountered; interpreting as ASCII!", + content_charset + ) msg.set_payload( msg.get_payload(decode=True).decode( content_charset, From 20d5c3956e8d46effcffa93565eeacdce4603ca2 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sat, 15 Nov 2014 22:19:32 -0800 Subject: [PATCH 80/86] Release 4.1.2; Added handling for processing unknown encodings. --- django_mailbox/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/__init__.py b/django_mailbox/__init__.py index 47cbba7..96e5585 100644 --- a/django_mailbox/__init__.py +++ b/django_mailbox/__init__.py @@ -1 +1 @@ -__version__ = '4.1.1' +__version__ = '4.1.2' From 4c1eff52029edb1bedb909a5c4d455e69d87683c Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Wed, 19 Nov 2014 23:42:15 -0800 Subject: [PATCH 81/86] Minor clarification. --- docs/topics/polling.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/topics/polling.rst b/docs/topics/polling.rst index 0134104..3f1c970 100644 --- a/docs/topics/polling.rst +++ b/docs/topics/polling.rst @@ -18,8 +18,10 @@ this method will gather new messages from the server. Using the Django Admin ---------------------- -Check the box next to each of the mailboxes you'd like to fetch e-mail from, -and select the 'Get new mail' option. +From the 'Mailboxes' page in the Django Admin, +check the box next to each of the mailboxes you'd like to fetch e-mail from, +select 'Get new mail' from the action selector at the top of the list +of mailboxes, then click 'Go'. Using a cron job ---------------- From 571f57d957ac360cb14916f72d8d8d24297a5eb7 Mon Sep 17 00:00:00 2001 From: Philippe Date: Mon, 15 Dec 2014 14:50:14 +0100 Subject: [PATCH 82/86] Fix encoded attachment's filenames --- django_mailbox/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 3c0a5c0..8404402 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -647,7 +647,9 @@ class MessageAttachment(models.Model): def get_filename(self): file_name = self._get_rehydrated_headers().get_filename() - if file_name: + if isinstance(file_name, six.text_type): + return file_name + elif file_name: return convert_header_to_unicode(file_name) else: return None From d276f7e9a96f362df6ddcc70cd0c94066e50d275 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Wed, 31 Dec 2014 09:40:36 -0600 Subject: [PATCH 83/86] Adding envelope headers to the Admin interface. --- django_mailbox/admin.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/django_mailbox/admin.py b/django_mailbox/admin.py index aa77b57..bc3c25a 100644 --- a/django_mailbox/admin.py +++ b/django_mailbox/admin.py @@ -64,6 +64,12 @@ class MessageAdmin(admin.ModelAdmin): def subject(self, msg): return convert_header_to_unicode(msg.subject) + def envelope_headers(self, msg): + email = msg.get_email_object() + return '\n'.join( + [('%s: %s' % (h, v)) for h, v in email.items()] + ) + inlines = [ MessageAttachmentInline, ] @@ -89,6 +95,7 @@ class MessageAdmin(admin.ModelAdmin): 'in_reply_to', ) readonly_fields = ( + 'envelope_headers', 'text', 'html', ) From c3cd5a28ae3cb20d7177b0fdb45aeb94448a6dd7 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Wed, 31 Dec 2014 09:41:08 -0600 Subject: [PATCH 84/86] Version 4.2; adds 'envelope_headers' to admin. --- django_mailbox/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/__init__.py b/django_mailbox/__init__.py index 96e5585..feecdb9 100644 --- a/django_mailbox/__init__.py +++ b/django_mailbox/__init__.py @@ -1 +1 @@ -__version__ = '4.1.2' +__version__ = '4.2' From 7fd5b02054a8a32b79533aa2e07a1c31f80427c3 Mon Sep 17 00:00:00 2001 From: Ben Segal Date: Sat, 31 Jan 2015 15:46:49 -0800 Subject: [PATCH 85/86] If no message_ids, return before checking for small_message_ids --- django_mailbox/transports/imap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index 90435eb..b7f2cd1 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -72,13 +72,13 @@ class ImapTransport(EmailTransport): def get_message(self): message_ids = self._get_all_message_ids() + if not message_ids: + return + # Limit the uids to the small ones if we care about that if self.max_message_size: message_ids = self._get_small_message_ids(message_ids) - if not message_ids: - return - if self.archive: typ, folders = self.server.list(pattern=self.archive) if folders[0] is None: From 793f92cfcf79e9626d4c08bcc40f2e55bdec0d49 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 6 Feb 2015 15:31:17 -0800 Subject: [PATCH 86/86] Release 4.2.1; fixes #41: if no messages_ids, return before checking for small_message_ids. --- django_mailbox/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/__init__.py b/django_mailbox/__init__.py index feecdb9..0d6a4f2 100644 --- a/django_mailbox/__init__.py +++ b/django_mailbox/__init__.py @@ -1 +1 @@ -__version__ = '4.2' +__version__ = '4.2.1'