diff --git a/.gitignore b/.gitignore index 0d20b64..72dc87e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,10 @@ *.pyc +bin/* +lib/* +src/* +dist/* +share/* +docs/_build/* +include/* +.Python +*egg* diff --git a/.travis.yml b/.travis.yml index 5e88b63..82aaddd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,24 @@ language: python + python: - - "2.6" - - "2.7" - - "3.2" + - "2.6" + - "2.7" + - "3.3" + env: - - DJANGO=1.5.5 - - DJANGO=1.6.1 + - DJANGO=1.4.14 + - DJANGO=1.5.9 + - DJANGO=1.6.8 + - DJANGO=1.7.1 + +matrix: + exclude: + - env: DJANGO=1.4.14 + python: "3.3" + - env: DJANGO=1.7.1 + python: "2.6" install: - - pip install -q Django==$DJANGO --use-mirrors - - pip install -q -e . --use-mirrors + - pip install -q Django==$DJANGO --use-mirrors + - pip install -q -e . --use-mirrors script: - - python setup.py test + - python setup.py test diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..c88f41c --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,56 @@ +Changelog +========= + +4.1 +--- + +* Adds Django 1.7 migrations support. + +4.0 +--- + +* Adds ``html`` property returning the HTML contents of + ``django_mailbox.models.Message`` instances. + Thanks `@ariel17 `_! +* Adds translation support. + Thanks `@ariel17 `_! +* **Drops support for Python 3.2**. The fact that only versions of + Python newer than 3.2 allow unicode literals has convinced me + that supporting Python 3.2 is probably more trouble than it's worth. + Please let me know if you were using Python 3.2, and I've left you + out in the cold; I'm willing to fix Python 3.2 support if it is + actively used. + +3.4 +--- + +* Adds ``gmail`` transport allowing one to use Google + OAuth credentials for gathering messages from gmail. + Thanks `@alexlovelltroy `_! + +3.3 +--- + +* Adds functionality to ``imap`` transport allowing one to + archive processed e-mails. + Thanks `@yellowcap `_! + +3.2 +--- + +* Fixes `#13 `_; + Python 3 support had been broken for some time. Thanks for catching that, + `@greendee `_! + +3.1 +--- + +* Fixes a wide variety of unicode-related errors. + +3.0 +--- + +* Restructures message storage such that non-text message attachments + are stored as files, rather than in the database in their original + (probably base64-encoded) blobs. +* So many new tests. diff --git a/django_mailbox/__init__.py b/django_mailbox/__init__.py index e69de29..0d6a4f2 100644 --- a/django_mailbox/__init__.py +++ b/django_mailbox/__init__.py @@ -0,0 +1 @@ +__version__ = '4.2.1' diff --git a/django_mailbox/admin.py b/django_mailbox/admin.py index ea989f1..bc3c25a 100644 --- a/django_mailbox/admin.py +++ b/django_mailbox/admin.py @@ -1,3 +1,11 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Model configuration in application ``django_mailbox`` for administration +console. +""" + import logging from django.conf import settings @@ -5,7 +13,8 @@ from django.contrib import admin from django_mailbox.models import MessageAttachment, Message, Mailbox from django_mailbox.signals import message_received -from django_mailbox.utils import decode_header +from django_mailbox.utils import convert_header_to_unicode + logger = logging.getLogger(__name__) @@ -21,6 +30,8 @@ def resend_message_received_signal(message_admin, request, queryset): for message in queryset.all(): logger.debug('Resending \'message_received\' signal for %s' % message) message_received.send(sender=message_admin, message=message) + + resend_message_received_signal.short_description = ( 'Re-send message received signal' ) @@ -51,7 +62,13 @@ class MessageAdmin(admin.ModelAdmin): return msg.attachments.count() def subject(self, msg): - return decode_header(msg.subject) + return convert_header_to_unicode(msg.subject) + + def envelope_headers(self, msg): + email = msg.get_email_object() + return '\n'.join( + [('%s: %s' % (h, v)) for h, v in email.items()] + ) inlines = [ MessageAttachmentInline, @@ -78,7 +95,9 @@ class MessageAdmin(admin.ModelAdmin): 'in_reply_to', ) readonly_fields = ( + 'envelope_headers', 'text', + 'html', ) actions = [resend_message_received_signal] diff --git a/django_mailbox/google_utils.py b/django_mailbox/google_utils.py new file mode 100644 index 0000000..b834f59 --- /dev/null +++ b/django_mailbox/google_utils.py @@ -0,0 +1,111 @@ +import logging + +from django.conf import settings +import requests +from social.apps.django_app.default.models import UserSocialAuth + + +logger = logging.getLogger(__name__) + + +class AccessTokenNotFound(Exception): + pass + + +class RefreshTokenNotFound(Exception): + pass + + +def get_google_consumer_key(): + return settings.SOCIAL_AUTH_GOOGLE_OAUTH2_KEY + + +def get_google_consumer_secret(): + return settings.SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET + + +def get_google_access_token(email): + # TODO: This should be cacheable + try: + me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") + return me.extra_data['access_token'] + except (UserSocialAuth.DoesNotExist, KeyError): + raise AccessTokenNotFound + + +def update_google_extra_data(email, extra_data): + try: + me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") + me.extra_data = extra_data + me.save() + except (UserSocialAuth.DoesNotExist, KeyError): + raise AccessTokenNotFound + + +def get_google_refresh_token(email): + try: + me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2") + return me.extra_data['refresh_token'] + except (UserSocialAuth.DoesNotExist, KeyError): + raise RefreshTokenNotFound + + +def google_api_get(email, url): + headers = dict( + Authorization="Bearer %s" % get_google_access_token(email), + ) + r = requests.get(url, headers=headers) + logger.info("I got a %s", r.status_code) + if r.status_code == 401: + # Go use the refresh token + refresh_authorization(email) + r = requests.get(url, headers=headers) + logger.info("I got a %s", r.status_code) + if r.status_code == 200: + try: + return r.json() + except ValueError: + return r.text + + +def google_api_post(email, url, post_data, authorized=True): + # TODO: Make this a lot less ugly. especially the 401 handling + headers = dict() + if authorized is True: + headers.update(dict( + Authorization="Bearer %s" % get_google_access_token(email), + )) + r = requests.post(url, headers=headers, data=post_data) + if r.status_code == 401: + refresh_authorization(email) + r = requests.post(url, headers=headers, data=post_data) + if r.status_code == 200: + try: + return r.json() + except ValueError: + return r.text + + +def refresh_authorization(email): + refresh_token = get_google_refresh_token(email) + post_data = dict( + refresh_token=refresh_token, + client_id=get_google_consumer_key(), + client_secret=get_google_consumer_secret(), + grant_type='refresh_token', + ) + results = google_api_post( + email, + "https://accounts.google.com/o/oauth2/token?access_type=offline", + post_data, + authorized=False) + results.update({'refresh_token': refresh_token}) + update_google_extra_data(email, results) + + +def fetch_user_info(email): + result = google_api_get( + email, + "https://www.googleapis.com/oauth2/v1/userinfo?alt=json" + ) + return result diff --git a/django_mailbox/management/commands/getmail.py b/django_mailbox/management/commands/getmail.py index 6041782..31b2dbd 100644 --- a/django_mailbox/management/commands/getmail.py +++ b/django_mailbox/management/commands/getmail.py @@ -1,17 +1,30 @@ +import logging + from django.core.management.base import BaseCommand from django_mailbox.models import Mailbox + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + class Command(BaseCommand): def handle(self, *args, **options): mailboxes = Mailbox.active_mailboxes.all() if args: - mailboxes = mailboxes.filter(name = ' '.join(args)) + mailboxes = mailboxes.filter( + name=' '.join(args) + ) for mailbox in mailboxes: - self.stdout.write('Gathering messages for %s\n' % mailbox.name) + logger.info( + 'Gathering messages for %s', + mailbox.name + ) messages = mailbox.get_new_mail() for message in messages: - self.stdout.write('Received %s (from %s)\n' % ( - message.subject, - message.from_address - )) + logger.info( + 'Received %s (from %s)', + message.subject, + message.from_address + ) diff --git a/django_mailbox/management/commands/processincomingmessage.py b/django_mailbox/management/commands/processincomingmessage.py index 8d269b8..2fd9cba 100644 --- a/django_mailbox/management/commands/processincomingmessage.py +++ b/django_mailbox/management/commands/processincomingmessage.py @@ -7,9 +7,11 @@ from django.core.management.base import BaseCommand from django_mailbox.models import Mailbox + logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) + class Command(BaseCommand): args = "<[Mailbox Name (optional)]>" command = "Receive incoming mail via stdin" @@ -22,14 +24,17 @@ class Command(BaseCommand): else: mailbox = self.get_mailbox_for_message(message) mailbox.process_incoming_message(message) - logger.info("Message received from %s" % message['from']) + logger.info( + "Message received from %s", + message['from'] + ) else: logger.warning("Message not processable.") def get_mailbox_by_name(self, name): mailbox, created = Mailbox.objects.get_or_create( - name=name, - ) + name=name, + ) return mailbox def get_mailbox_for_message(self, message): diff --git a/django_mailbox/migrations/0001_initial.py b/django_mailbox/migrations/0001_initial.py index 4faa4fa..dfb6ffa 100644 --- a/django_mailbox/migrations/0001_initial.py +++ b/django_mailbox/migrations/0001_initial.py @@ -1,59 +1,59 @@ # -*- coding: utf-8 -*- -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models +from __future__ import unicode_literals + +from django.db import models, migrations -class Migration(SchemaMigration): +class Migration(migrations.Migration): - def forwards(self, orm): - # Adding model 'Mailbox' - db.create_table('django_mailbox_mailbox', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('name', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)), - )) - db.send_create_signal('django_mailbox', ['Mailbox']) + dependencies = [ + ] - # Adding model 'Message' - db.create_table('django_mailbox_message', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('mailbox', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['django_mailbox.Mailbox'])), - ('subject', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('message_id', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('from_address', self.gf('django.db.models.fields.CharField')(max_length=255)), - ('body', self.gf('django.db.models.fields.TextField')()), - ('received', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), - )) - db.send_create_signal('django_mailbox', ['Message']) - - - def backwards(self, orm): - # Deleting model 'Mailbox' - db.delete_table('django_mailbox_mailbox') - - # Deleting model 'Message' - db.delete_table('django_mailbox_message') - - - models = { - 'django_mailbox.mailbox': { - 'Meta': {'object_name': 'Mailbox'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - }, - 'django_mailbox.message': { - 'Meta': {'object_name': 'Message'}, - 'body': ('django.db.models.fields.TextField', [], {}), - 'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['django_mailbox.Mailbox']"}), - 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}) - } - } - - complete_apps = ['django_mailbox'] \ No newline at end of file + operations = [ + migrations.CreateModel( + name='Mailbox', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('name', models.CharField(max_length=255, verbose_name='Name')), + ('uri', models.CharField(default=None, max_length=255, blank=True, help_text="Example: imap+ssl://myusername:mypassword@someserver

Internet transports include 'imap' and 'pop3'; common local file transports include 'maildir', 'mbox', and less commonly 'babyl', 'mh', and 'mmdf'.

Be sure to urlencode your username and password should they contain illegal characters (like @, :, etc).", null=True, verbose_name='URI')), + ('from_email', models.CharField(default=None, max_length=255, blank=True, help_text="Example: MailBot <mailbot@yourdomain.com>
'From' header to set for outgoing email.

If you do not use this e-mail inbox for outgoing mail, this setting is unnecessary.
If you send e-mail without setting this, your 'From' header will'be set to match the setting `DEFAULT_FROM_EMAIL`.", null=True, verbose_name='From email')), + ('active', models.BooleanField(default=True, help_text='Check this e-mail inbox for new e-mail messages during polling cycles. This checkbox does not have an effect upon whether mail is collected here when this mailbox receives mail from a pipe, and does not affect whether e-mail messages can be dispatched from this mailbox. ', verbose_name='Active')), + ], + options={ + 'verbose_name_plural': 'Mailboxes', + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='Message', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('subject', models.CharField(max_length=255, verbose_name='Subject')), + ('message_id', models.CharField(max_length=255, verbose_name='Message ID')), + ('from_header', models.CharField(max_length=255, verbose_name='From header')), + ('to_header', models.TextField(verbose_name='To header')), + ('outgoing', models.BooleanField(default=False, verbose_name='Outgoing')), + ('body', models.TextField(verbose_name='Body')), + ('encoded', models.BooleanField(default=False, help_text='True if the e-mail body is Base64 encoded', verbose_name='Encoded')), + ('processed', models.DateTimeField(auto_now_add=True, verbose_name='Processed')), + ('read', models.DateTimeField(default=None, null=True, verbose_name='Read', blank=True)), + ('in_reply_to', models.ForeignKey(related_name='replies', verbose_name='In reply to', blank=True, to='django_mailbox.Message', null=True)), + ('mailbox', models.ForeignKey(related_name='messages', verbose_name='Mailbox', to='django_mailbox.Mailbox')), + ], + options={ + }, + bases=(models.Model,), + ), + migrations.CreateModel( + name='MessageAttachment', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('headers', models.TextField(null=True, verbose_name='Headers', blank=True)), + ('document', models.FileField(upload_to=b'mailbox_attachments/%Y/%m/%d/', verbose_name='Document')), + ('message', models.ForeignKey(related_name='attachments', verbose_name='Message', blank=True, to='django_mailbox.Message', null=True)), + ], + options={ + }, + bases=(models.Model,), + ), + ] diff --git a/django_mailbox/models.py b/django_mailbox/models.py index e604caa..0329322 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -1,30 +1,39 @@ -import base64 -import email -from email.header import decode_header +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Models declaration for application ``django_mailbox``. +""" + +from email.encoders import encode_base64 from email.message import Message as EmailMessage from email.utils import formatdate, parseaddr -from email.encoders import encode_base64 -import urllib +from quopri import encode as encode_quopri +import base64 +import email +import logging import mimetypes import os.path -from quopri import encode as encode_quopri import sys import uuid -try: - import urllib.parse as urlparse -except ImportError: - import urlparse +import six +from six.moves.urllib.parse import parse_qs, unquote, urlparse from django.conf import settings -from django.core.mail.message import make_msgid from django.core.files.base import ContentFile +from django.core.mail.message import make_msgid from django.db import models -from django_mailbox.transports import Pop3Transport, ImapTransport,\ - MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ - MMDFTransport +from django.utils.translation import ugettext as _ + +from .utils import convert_header_to_unicode, get_body_from_message from django_mailbox.signals import message_received -import six +from django_mailbox.transports import Pop3Transport, ImapTransport, \ + MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ + MMDFTransport, GmailImapTransport + + +logger = logging.getLogger(__name__) STRIP_UNALLOWED_MIMETYPES = getattr( @@ -32,6 +41,7 @@ STRIP_UNALLOWED_MIMETYPES = getattr( 'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES', False ) + ALLOWED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_ALLOWED_MIMETYPES', @@ -40,6 +50,7 @@ ALLOWED_MIMETYPES = getattr( 'text/html' ] ) + TEXT_STORED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_TEXT_STORED_MIMETYPES', @@ -48,11 +59,13 @@ TEXT_STORED_MIMETYPES = getattr( 'text/html' ] ) + ALTERED_MESSAGE_HEADER = getattr( settings, 'DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER', 'X-Django-Mailbox-Altered-Message' ) + ATTACHMENT_INTERPOLATION_HEADER = getattr( settings, 'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER', @@ -61,17 +74,22 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr( class ActiveMailboxManager(models.Manager): - def get_query_set(self): - return super(ActiveMailboxManager, self).get_query_set().filter( + def get_queryset(self): + return super(ActiveMailboxManager, self).get_queryset().filter( active=True, ) class Mailbox(models.Model): - name = models.CharField(max_length=255) - uri = models.CharField( + name = models.CharField( + _(u'Name'), max_length=255, - help_text=( + ) + + uri = models.CharField( + _(u'URI'), + max_length=255, + help_text=(_( "Example: imap+ssl://myusername:mypassword@someserver
" "
" "Internet transports include 'imap' and 'pop3'; " @@ -80,14 +98,16 @@ class Mailbox(models.Model): "
" "Be sure to urlencode your username and password should they " "contain illegal characters (like @, :, etc)." - ), + )), blank=True, null=True, default=None, ) + from_email = models.CharField( + _(u'From email'), max_length=255, - help_text=( + help_text=(_( "Example: MailBot <mailbot@yourdomain.com>
" "'From' header to set for outgoing email.
" "
" @@ -95,19 +115,21 @@ class Mailbox(models.Model): "setting is unnecessary.
" "If you send e-mail without setting this, your 'From' header will'" "be set to match the setting `DEFAULT_FROM_EMAIL`." - ), + )), blank=True, null=True, default=None, ) + active = models.BooleanField( - help_text=( + _(u'Active'), + help_text=(_( "Check this e-mail inbox for new e-mail messages during polling " "cycles. This checkbox does not have an effect upon whether " "mail is collected here when this mailbox receives mail from a " "pipe, and does not affect whether e-mail messages can be " "dispatched from this mailbox. " - ), + )), blank=True, default=True, ) @@ -117,7 +139,11 @@ class Mailbox(models.Model): @property def _protocol_info(self): - return urlparse.urlparse(self.uri) + return urlparse(self.uri) + + @property + def _query_string(self): + return parse_qs(self._protocol_info.query) @property def _domain(self): @@ -129,11 +155,11 @@ class Mailbox(models.Model): @property def username(self): - return urllib.unquote(self._protocol_info.username) + return unquote(self._protocol_info.username) @property def password(self): - return urllib.unquote(self._protocol_info.password) + return unquote(self._protocol_info.password) @property def location(self): @@ -150,6 +176,13 @@ class Mailbox(models.Model): def use_ssl(self): return '+ssl' in self._protocol_info.scheme.lower() + @property + def archive(self): + archive_folder = self._query_string.get('archive', None) + if not archive_folder: + return None + return archive_folder[0] + def get_connection(self): if not self.uri: return None @@ -157,7 +190,16 @@ class Mailbox(models.Model): conn = ImapTransport( self.location, port=self.port if self.port else None, - ssl=self.use_ssl + ssl=self.use_ssl, + archive=self.archive + ) + conn.connect(self.username, self.password) + elif self.type == 'gmail': + conn = GmailImapTransport( + self.location, + port=self.port if self.port else None, + ssl=True, + archive=self.archive ) conn.connect(self.username, self.password) elif self.type == 'pop3': @@ -249,6 +291,36 @@ class Mailbox(models.Model): placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk) new = placeholder else: + content_charset = msg.get_content_charset() + if not content_charset: + content_charset = 'ascii' + try: + # Make sure that the payload can be properly decoded in the + # defined charset, if it can't, let's mash some things + # inside the payload :-\ + msg.get_payload(decode=True).decode(content_charset) + except LookupError: + logger.warning( + "Unknown encoding %s; interpreting as ASCII!", + content_charset + ) + msg.set_payload( + msg.get_payload(decode=True).decode( + 'ascii', + 'ignore' + ) + ) + except ValueError: + logger.warning( + "Decoding error encountered; interpreting as ASCII!", + content_charset + ) + msg.set_payload( + msg.get_payload(decode=True).decode( + content_charset, + 'ignore' + ) + ) new = msg return new @@ -256,13 +328,15 @@ class Mailbox(models.Model): msg = Message() msg.mailbox = self if 'subject' in message: - msg.subject = message['subject'][0:255] + msg.subject = convert_header_to_unicode(message['subject'])[0:255] if 'message-id' in message: msg.message_id = message['message-id'][0:255] if 'from' in message: - msg.from_header = message['from'] + msg.from_header = convert_header_to_unicode(message['from']) if 'to' in message: - msg.to_header = message['to'] + msg.to_header = convert_header_to_unicode(message['to']) + elif 'Delivered-To' in message: + msg.to_header = convert_header_to_unicode(message['Delivered-To']) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) @@ -294,55 +368,83 @@ class Mailbox(models.Model): class IncomingMessageManager(models.Manager): - def get_query_set(self): - return super(IncomingMessageManager, self).get_query_set().filter( + def get_queryset(self): + return super(IncomingMessageManager, self).get_queryset().filter( outgoing=False, ) class OutgoingMessageManager(models.Manager): - def get_query_set(self): - return super(OutgoingMessageManager, self).get_query_set().filter( + def get_queryset(self): + return super(OutgoingMessageManager, self).get_queryset().filter( outgoing=True, ) class UnreadMessageManager(models.Manager): - def get_query_set(self): - return super(UnreadMessageManager, self).get_query_set().filter( + def get_queryset(self): + return super(UnreadMessageManager, self).get_queryset().filter( read=None ) class Message(models.Model): - mailbox = models.ForeignKey(Mailbox, related_name='messages') - subject = models.CharField(max_length=255) - message_id = models.CharField(max_length=255) + mailbox = models.ForeignKey( + Mailbox, + related_name='messages', + verbose_name=_(u'Mailbox'), + ) + + subject = models.CharField( + _(u'Subject'), + max_length=255 + ) + + message_id = models.CharField( + _(u'Message ID'), + max_length=255 + ) + in_reply_to = models.ForeignKey( 'django_mailbox.Message', related_name='replies', blank=True, null=True, + verbose_name=_(u'In reply to'), ) + from_header = models.CharField( + _('From header'), max_length=255, ) - to_header = models.TextField() + + to_header = models.TextField( + _(u'To header'), + ) + outgoing = models.BooleanField( + _(u'Outgoing'), default=False, blank=True, ) - body = models.TextField() + body = models.TextField( + _(u'Body'), + ) + encoded = models.BooleanField( + _(u'Encoded'), default=False, - help_text='True if the e-mail body is Base64 encoded' + help_text=_('True if the e-mail body is Base64 encoded'), ) processed = models.DateTimeField( + _('Processed'), auto_now_add=True ) + read = models.DateTimeField( + _(u'Read'), default=None, blank=True, null=True, @@ -412,28 +514,22 @@ class Message(models.Model): @property def text(self): - return self.get_text_body() - - def get_text_body(self): - def get_body_from_message(message): - body = '' - for part in message.walk(): - if ( - part.get_content_maintype() == 'text' - and part.get_content_subtype() == 'plain' - ): - charset = part.get_content_charset() - this_part = part.get_payload(decode=True) - if charset: - this_part = this_part.decode(charset, 'replace') - - body += this_part - return body - + """ + Returns the message body matching content type 'text/plain'. + """ return get_body_from_message( - self.get_email_object() + self.get_email_object(), 'text', 'plain' ).replace('=\n', '').strip() + @property + def html(self): + """ + Returns the message body matching content type 'text/html'. + """ + return get_body_from_message( + self.get_email_object(), 'text', 'html' + ).replace('\n', '').strip() + def _rehydrate(self, msg): new = EmailMessage() if msg.is_multipart(): @@ -495,7 +591,7 @@ class Message(models.Model): return self.body.encode('utf-8') def set_body(self, body): - if sys.version_info >= (3, 0): + if six.PY3: body = body.encode('utf-8') self.encoded = True self.body = base64.b64encode(body).decode('ascii') @@ -503,10 +599,10 @@ class Message(models.Model): def get_email_object(self): """ Returns an `email.message.Message` instance for this message.""" body = self.get_body() - if sys.version_info < (3, 0): - flat = email.message_from_string(body) - else: + if six.PY3: flat = email.message_from_bytes(body) + else: + flat = email.message_from_string(body) return self._rehydrate(flat) def delete(self, *args, **kwargs): @@ -525,9 +621,19 @@ class MessageAttachment(models.Model): related_name='attachments', null=True, blank=True, + verbose_name=_('Message'), + ) + + headers = models.TextField( + _(u'Headers'), + null=True, + blank=True, + ) + + document = models.FileField( + _(u'Document'), + upload_to='mailbox_attachments/%Y/%m/%d/' ) - headers = models.TextField(null=True, blank=True) - document = models.FileField(upload_to='mailbox_attachments/%Y/%m/%d/') def delete(self, *args, **kwargs): self.document.delete() @@ -556,11 +662,10 @@ class MessageAttachment(models.Model): def get_filename(self): file_name = self._get_rehydrated_headers().get_filename() - if file_name: - encoded_subject, encoding = decode_header(file_name)[0] - if encoding: - encoded_subject = encoded_subject.decode(encoding) - return encoded_subject + if isinstance(file_name, six.text_type): + return file_name + elif file_name: + return convert_header_to_unicode(file_name) else: return None diff --git a/django_mailbox/runtests.py b/django_mailbox/runtests.py index 62094fd..f5b0ff3 100644 --- a/django_mailbox/runtests.py +++ b/django_mailbox/runtests.py @@ -3,6 +3,10 @@ import sys from os.path import dirname, abspath +try: + from django import setup +except ImportError: + pass from django.conf import settings if not settings.configured: @@ -27,6 +31,12 @@ def runtests(*test_args): test_args = ['django_mailbox'] parent = dirname(abspath(__file__)) sys.path.insert(0, parent) + try: + # ensure that AppRegistry has loaded + setup() + except NameError: + # This version of Django is too old for an app registry. + pass runner = DjangoTestSuiteRunner( verbosity=1, interactive=False, diff --git a/django_mailbox/south_migrations/0001_initial.py b/django_mailbox/south_migrations/0001_initial.py new file mode 100644 index 0000000..4faa4fa --- /dev/null +++ b/django_mailbox/south_migrations/0001_initial.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'Mailbox' + db.create_table('django_mailbox_mailbox', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)), + )) + db.send_create_signal('django_mailbox', ['Mailbox']) + + # Adding model 'Message' + db.create_table('django_mailbox_message', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('mailbox', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['django_mailbox.Mailbox'])), + ('subject', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('message_id', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('from_address', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('body', self.gf('django.db.models.fields.TextField')()), + ('received', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + )) + db.send_create_signal('django_mailbox', ['Message']) + + + def backwards(self, orm): + # Deleting model 'Mailbox' + db.delete_table('django_mailbox_mailbox') + + # Deleting model 'Message' + db.delete_table('django_mailbox_message') + + + models = { + 'django_mailbox.mailbox': { + 'Meta': {'object_name': 'Mailbox'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'django_mailbox.message': { + 'Meta': {'object_name': 'Message'}, + 'body': ('django.db.models.fields.TextField', [], {}), + 'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['django_mailbox.Mailbox']"}), + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['django_mailbox'] \ No newline at end of file diff --git a/django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py b/django_mailbox/south_migrations/0002_auto__chg_field_mailbox_uri.py similarity index 100% rename from django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py rename to django_mailbox/south_migrations/0002_auto__chg_field_mailbox_uri.py diff --git a/django_mailbox/migrations/0003_auto__add_field_mailbox_active.py b/django_mailbox/south_migrations/0003_auto__add_field_mailbox_active.py similarity index 100% rename from django_mailbox/migrations/0003_auto__add_field_mailbox_active.py rename to django_mailbox/south_migrations/0003_auto__add_field_mailbox_active.py diff --git a/django_mailbox/migrations/0004_auto__add_field_message_outgoing.py b/django_mailbox/south_migrations/0004_auto__add_field_message_outgoing.py similarity index 100% rename from django_mailbox/migrations/0004_auto__add_field_message_outgoing.py rename to django_mailbox/south_migrations/0004_auto__add_field_message_outgoing.py diff --git a/django_mailbox/migrations/0005_rename_fields.py b/django_mailbox/south_migrations/0005_rename_fields.py similarity index 100% rename from django_mailbox/migrations/0005_rename_fields.py rename to django_mailbox/south_migrations/0005_rename_fields.py diff --git a/django_mailbox/migrations/0006_auto__add_field_message_in_reply_to.py b/django_mailbox/south_migrations/0006_auto__add_field_message_in_reply_to.py similarity index 100% rename from django_mailbox/migrations/0006_auto__add_field_message_in_reply_to.py rename to django_mailbox/south_migrations/0006_auto__add_field_message_in_reply_to.py diff --git a/django_mailbox/migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py b/django_mailbox/south_migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py similarity index 100% rename from django_mailbox/migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py rename to django_mailbox/south_migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py diff --git a/django_mailbox/migrations/0008_populate_from_to_fields.py b/django_mailbox/south_migrations/0008_populate_from_to_fields.py similarity index 100% rename from django_mailbox/migrations/0008_populate_from_to_fields.py rename to django_mailbox/south_migrations/0008_populate_from_to_fields.py diff --git a/django_mailbox/migrations/0009_remove_references_table.py b/django_mailbox/south_migrations/0009_remove_references_table.py similarity index 100% rename from django_mailbox/migrations/0009_remove_references_table.py rename to django_mailbox/south_migrations/0009_remove_references_table.py diff --git a/django_mailbox/migrations/0010_auto__add_field_mailbox_from_email.py b/django_mailbox/south_migrations/0010_auto__add_field_mailbox_from_email.py similarity index 100% rename from django_mailbox/migrations/0010_auto__add_field_mailbox_from_email.py rename to django_mailbox/south_migrations/0010_auto__add_field_mailbox_from_email.py diff --git a/django_mailbox/migrations/0011_auto__add_field_message_read.py b/django_mailbox/south_migrations/0011_auto__add_field_message_read.py similarity index 100% rename from django_mailbox/migrations/0011_auto__add_field_message_read.py rename to django_mailbox/south_migrations/0011_auto__add_field_message_read.py diff --git a/django_mailbox/migrations/0012_auto__add_messageattachment.py b/django_mailbox/south_migrations/0012_auto__add_messageattachment.py similarity index 100% rename from django_mailbox/migrations/0012_auto__add_messageattachment.py rename to django_mailbox/south_migrations/0012_auto__add_messageattachment.py diff --git a/django_mailbox/migrations/0013_auto__add_field_messageattachment_message.py b/django_mailbox/south_migrations/0013_auto__add_field_messageattachment_message.py similarity index 100% rename from django_mailbox/migrations/0013_auto__add_field_messageattachment_message.py rename to django_mailbox/south_migrations/0013_auto__add_field_messageattachment_message.py diff --git a/django_mailbox/migrations/0014_migrate_message_attachments.py b/django_mailbox/south_migrations/0014_migrate_message_attachments.py similarity index 100% rename from django_mailbox/migrations/0014_migrate_message_attachments.py rename to django_mailbox/south_migrations/0014_migrate_message_attachments.py diff --git a/django_mailbox/migrations/0015_auto__add_field_messageattachment_headers.py b/django_mailbox/south_migrations/0015_auto__add_field_messageattachment_headers.py similarity index 100% rename from django_mailbox/migrations/0015_auto__add_field_messageattachment_headers.py rename to django_mailbox/south_migrations/0015_auto__add_field_messageattachment_headers.py diff --git a/django_mailbox/migrations/0016_auto__add_field_message_encoded.py b/django_mailbox/south_migrations/0016_auto__add_field_message_encoded.py similarity index 100% rename from django_mailbox/migrations/0016_auto__add_field_message_encoded.py rename to django_mailbox/south_migrations/0016_auto__add_field_message_encoded.py diff --git a/django_mailbox/south_migrations/__init__.py b/django_mailbox/south_migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_mailbox/tests/__init__.py b/django_mailbox/tests/__init__.py index 42d5abe..b83434c 100644 --- a/django_mailbox/tests/__init__.py +++ b/django_mailbox/tests/__init__.py @@ -1,3 +1,4 @@ from .test_mailbox import * from .test_message_flattening import * from .test_process_email import * +from .test_transports import * diff --git a/django_mailbox/tests/base.py b/django_mailbox/tests/base.py index b062797..72c3249 100644 --- a/django_mailbox/tests/base.py +++ b/django_mailbox/tests/base.py @@ -1,12 +1,23 @@ import email import os.path +import six + from django.test import TestCase -import sys from django_mailbox import models from django_mailbox.models import Mailbox, Message +def get_email_as_text(name): + with open( + os.path.join( + os.path.dirname(__file__), + 'messages', + name, + ), + 'rb' + ) as f: + return f.read() class EmailMessageTestCase(TestCase): ALLOWED_EXTRA_HEADERS = [ @@ -22,7 +33,7 @@ class EmailMessageTestCase(TestCase): self.mailbox = Mailbox.objects.create() super(EmailMessageTestCase, self).setUp() - def _get_email_object(self, name): + def _get_email_as_text(self, name): with open( os.path.join( os.path.dirname(__file__), @@ -31,10 +42,14 @@ class EmailMessageTestCase(TestCase): ), 'rb' ) as f: - if sys.version_info < (3, 0): - return email.message_from_string(f.read()) - else: - return email.message_from_bytes(f.read()) + return f.read() + + def _get_email_object(self, name): + copy = self._get_email_as_text(name) + if six.PY3: + return email.message_from_bytes(copy) + else: + return email.message_from_string(copy) def _headers_identical(self, left, right, header=None): """ Check if headers are (close enough to) identical. diff --git a/django_mailbox/tests/messages/generic_message.eml b/django_mailbox/tests/messages/generic_message.eml index b3c9f33..bf5f0fe 100644 --- a/django_mailbox/tests/messages/generic_message.eml +++ b/django_mailbox/tests/messages/generic_message.eml @@ -1,5 +1,4 @@ MIME-Version: 1.0 -Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 11:53:53 -0800 (PST) X-Originating-IP: [24.22.122.177] Date: Sun, 20 Jan 2013 11:53:53 -0800 Delivered-To: test@adamcoddington.net diff --git a/django_mailbox/tests/messages/message_with_attachment.eml b/django_mailbox/tests/messages/message_with_attachment.eml index 34e7093..ac1cc43 100644 --- a/django_mailbox/tests/messages/message_with_attachment.eml +++ b/django_mailbox/tests/messages/message_with_attachment.eml @@ -17,8 +17,8 @@ This message has an attachment. --047d7b33dd729737fe04d3bde348 Content-Type: image/png; name="heart.png" Content-Disposition: attachment; filename="heart.png" -Content-Transfer-Encoding: base64 X-Attachment-Id: f_hc6mair60 +Content-Transfer-Encoding: base64 iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAFoTx1HAAAAzUlEQVQoz32RWxXDIBBEr4NIQEIl ICESkFAJkRAJSIgEpEQCEqYfu6QUkn7sCcyDGQiSACKSKCAkGwBJwhDwZQNMEiYAIBdQvk7rfaHf diff --git a/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml b/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml index 8860e9d..ea0d874 100644 --- a/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml +++ b/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml @@ -1,5 +1,5 @@ Date: Fri, 30 Jan 2012 11:30:01 PST -Subject: Tjest +Subject: Tjest Wrong From: "Somebody" To: idontknow@somewhere.com MIME-Version: 1.0 diff --git a/django_mailbox/tests/messages/message_with_invalid_encoding.eml b/django_mailbox/tests/messages/message_with_invalid_encoding.eml new file mode 100644 index 0000000..fbe1f9f --- /dev/null +++ b/django_mailbox/tests/messages/message_with_invalid_encoding.eml @@ -0,0 +1,13 @@ +Reply-To: +From: "Refinance" +Subject: Apply For Loans @ 2% Per Annum +Date: Sun, 19 Oct 2014 11:48:49 +0100 +MIME-Version: 1.0 +Content-Type: text/plain; + charset="_iso-2022-jp$ESC" +Content-Transfer-Encoding: 7bit + +We offer loans to private individuals and corporate organizations at 2% interest rate. Interested serious applicants should apply via email with details of their requirements. + +Warm Regards, +Loan Team diff --git a/django_mailbox/tests/messages/test_UnicodeDecodeError.eml b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml similarity index 97% rename from django_mailbox/tests/messages/test_UnicodeDecodeError.eml rename to django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml index f699a20..e6c96d8 100644 --- a/django_mailbox/tests/messages/test_UnicodeDecodeError.eml +++ b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml @@ -21,7 +21,7 @@ Received: from [144.206.32.69] by e.mail.ru with HTTP; Mon, 21 Apr 2014 17:01:45 +0400 From: =?UTF-8?B?dGVzdCB0ZXN0?= To: mr.test32@yandex.ru -Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ! +Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ! Content-Type: text/html; charset="iso-8859-1" diff --git a/django_mailbox/tests/test_message_flattening.py b/django_mailbox/tests/test_message_flattening.py index 9574701..1399a34 100644 --- a/django_mailbox/tests/test_message_flattening.py +++ b/django_mailbox/tests/test_message_flattening.py @@ -85,3 +85,20 @@ class TestMessageFlattening(EmailMessageTestCase): actual_email_object, expected_email_object, ) + + def test_message_processing_unknown_encoding(self): + incoming_email_object = self._get_email_object( + 'message_with_invalid_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(incoming_email_object) + + expected_text = ( + "We offer loans to private individuals and corporate " + "organizations at 2% interest rate. Interested serious " + "applicants should apply via email with details of their " + "requirements.\n\nWarm Regards,\nLoan Team" + ) + actual_text = msg.text + + self.assertEqual(actual_text, expected_text) diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 0dea582..c1c0955 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -1,4 +1,5 @@ import os.path +import sys import six @@ -64,7 +65,7 @@ class TestProcessEmail(EmailMessageTestCase): msg = mailbox.process_incoming_message(message) expected_results = 'Hello there!' - actual_results = msg.get_text_body().strip() + actual_results = msg.text.strip() self.assertEqual( expected_results, @@ -79,7 +80,7 @@ class TestProcessEmail(EmailMessageTestCase): message.get_email_object = lambda: email_object - actual_text = message.get_text_body() + actual_text = message.text expected_text = ( 'The one of us with a bike pump is far ahead, ' 'but a man stopped to help us and gave us his pump.' @@ -121,11 +122,11 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) - actual_text = msg.get_text_body() expected_text = six.u( 'This message contains funny UTF16 characters like this one: ' '"\xc2\xa0" and this one "\xe2\x9c\xbf".' ) + actual_text = msg.text self.assertEqual( expected_text, @@ -150,7 +151,7 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) - msg.get_text_body() + msg.text def test_message_with_valid_content_in_single_byte_encoding(self): email_object = self._get_email_object( @@ -159,8 +160,7 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) - actual_body = msg.get_text_body() - + actual_text = msg.text expected_body = six.u( '\u042d\u0442\u043e ' '\u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0435 ' @@ -171,6 +171,32 @@ class TestProcessEmail(EmailMessageTestCase): ) self.assertEqual( - actual_body, + actual_text, expected_body, ) + + def test_message_with_single_byte_subject_encoding(self): + email_object = self._get_email_object( + 'message_with_single_byte_extended_subject_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(email_object) + + expected_subject = six.u( + '\u00D3\u00E7\u00ED\u00E0\u00E9 \u00EA\u00E0\u00EA ' + '\u00E7\u00E0\u00F0\u00E0\u00E1\u00E0\u00F2\u00FB\u00E2' + '\u00E0\u00F2\u00FC \u00EE\u00F2 1000$ \u00E2 ' + '\u00ED\u00E5\u00E4\u00E5\u00EB\u00FE!' + ) + actual_subject = msg.subject + self.assertEqual(actual_subject, expected_subject) + + if sys.version_info >= (3, 3): + # There were various bugfixes in Py3k's email module, + # this is apparently one of them. + expected_from = six.u('test test ') + else: + expected_from = six.u('test test') + actual_from = msg.from_header + + self.assertEqual(expected_from, actual_from) diff --git a/django_mailbox/tests/test_transports.py b/django_mailbox/tests/test_transports.py new file mode 100644 index 0000000..b701eae --- /dev/null +++ b/django_mailbox/tests/test_transports.py @@ -0,0 +1,132 @@ +import mock + +from django.test.utils import override_settings + +from django_mailbox.tests.base import EmailMessageTestCase, get_email_as_text +from django_mailbox.transports import ImapTransport, Pop3Transport + +FAKE_UID_SEARCH_ANSWER = ('OK', ['18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44']) +FAKE_UID_FETCH_SIZES = ('OK', ['1 (UID 18 RFC822.SIZE 58070000000)', '2 (UID 19 RFC822.SIZE 2593)']) +FAKE_UID_FETCH_MSG = ('OK', [('1 (UID 18 RFC822 {5807}', get_email_as_text('generic_message.eml') ),]) +FAKE_UID_COPY_MSG = ('OK', ['[COPYUID 1 2 2] (Success)']) +FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS = ('OK', ['(\\HasNoChildren \\All) "/" "[Gmail]/All Mail"']) + +class IMAPTestCase(EmailMessageTestCase): + def setUp(self): + def imap_server_uid_method(*args): + cmd = args[0] + arg2 = args[2] + if cmd == 'search': + return FAKE_UID_SEARCH_ANSWER + if cmd == 'copy': + return FAKE_UID_COPY_MSG + if cmd == 'fetch': + if arg2 == '(RFC822.SIZE)': + return FAKE_UID_FETCH_SIZES + if arg2 == '(RFC822)': + return FAKE_UID_FETCH_MSG + def imap_server_list_method(pattern=None): + return FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS + + self.imap_server = mock.Mock() + self.imap_server.uid = imap_server_uid_method + self.imap_server.list = imap_server_list_method + super(IMAPTestCase, self).setUp() + + +class TestImapTransport(IMAPTestCase): + def setUp(self): + super(TestImapTransport, self).setUp() + self.arbitrary_hostname = 'one.two.three' + self.arbitrary_port = 100 + self.ssl = False + self.transport = ImapTransport( + self.arbitrary_hostname, + self.arbitrary_port, + self.ssl + ) + self.transport.server = self.imap_server + + def test_get_email_message(self): + actual_messages = list(self.transport.get_message()) + self.assertEqual(len(actual_messages), 27) + actual_message = actual_messages[0] + expected_message = self._get_email_object('generic_message.eml') + self.assertEqual(expected_message, actual_message) + +class TestImapArchivedTransport(TestImapTransport): + def setUp(self): + super(TestImapArchivedTransport, self).setUp() + self.archive = 'Archive' + self.transport = ImapTransport( + self.arbitrary_hostname, + self.arbitrary_port, + self.ssl, + self.archive + ) + self.transport.server = self.imap_server + +class TestMaxSizeImapTransport(TestImapTransport): + + @override_settings(DJANGO_MAILBOX_MAX_MESSAGE_SIZE=5807) + def setUp(self): + super(TestMaxSizeImapTransport, self).setUp() + + self.transport = ImapTransport( + self.arbitrary_hostname, + self.arbitrary_port, + self.ssl, + ) + self.transport.server = self.imap_server + + def test_size_limit(self): + all_message_ids = self.transport._get_all_message_ids() + small_message_ids = self.transport._get_small_message_ids(all_message_ids) + self.assertEqual(len(small_message_ids), 1) + + def test_get_email_message(self): + actual_messages = list(self.transport.get_message()) + self.assertEqual(len(actual_messages), 1) + actual_message = actual_messages[0] + expected_message = self._get_email_object('generic_message.eml') + self.assertEqual(expected_message, actual_message) + + +class TestPop3Transport(EmailMessageTestCase): + def setUp(self): + self.arbitrary_hostname = 'one.two.three' + self.arbitrary_port = 100 + self.ssl = False + self.transport = Pop3Transport( + self.arbitrary_hostname, + self.arbitrary_port, + self.ssl + ) + self.transport.server = None + super(TestPop3Transport, self).setUp() + + def test_get_email_message(self): + with mock.patch.object(self.transport, 'server') as server: + # Consider this value arbitrary, the second parameter + # should have one entry per message in the inbox + server.list.return_value = [None, ['some_msg']] + server.retr.return_value = [ + '+OK message follows', + [ + line.encode('ascii') + for line in self._get_email_as_text( + 'generic_message.eml' + ).decode('ascii').split('\n') + ], + 10018, # Some arbitrary size, ideally matching the above + ] + + actual_messages = list(self.transport.get_message()) + + self.assertEqual(len(actual_messages), 1) + + actual_message = actual_messages[0] + expected_message = self._get_email_object('generic_message.eml') + + self.assertEqual(expected_message, actual_message) + diff --git a/django_mailbox/transports/__init__.py b/django_mailbox/transports/__init__.py index fc17082..03b31f0 100644 --- a/django_mailbox/transports/__init__.py +++ b/django_mailbox/transports/__init__.py @@ -5,3 +5,4 @@ from django_mailbox.transports.mbox import MboxTransport from django_mailbox.transports.babyl import BabylTransport from django_mailbox.transports.mh import MHTransport from django_mailbox.transports.mmdf import MMDFTransport +from django_mailbox.transports.gmail import GmailImapTransport diff --git a/django_mailbox/transports/base.py b/django_mailbox/transports/base.py new file mode 100644 index 0000000..753df52 --- /dev/null +++ b/django_mailbox/transports/base.py @@ -0,0 +1,19 @@ +import email + +import six + +# Do *not* remove this, we need to use this in subclasses of EmailTransport +if six.PY3: + from email.errors import MessageParseError +else: + from email.Errors import MessageParseError + + +class EmailTransport(object): + def get_email_from_bytes(self, contents): + if six.PY3: + message = email.message_from_bytes(contents) + else: + message = email.message_from_string(contents) + + return message diff --git a/django_mailbox/transports/generic.py b/django_mailbox/transports/generic.py index 059d078..cf4bd78 100644 --- a/django_mailbox/transports/generic.py +++ b/django_mailbox/transports/generic.py @@ -1,10 +1,17 @@ -class GenericFileMailbox(object): +import sys + +from .base import EmailTransport + + +class GenericFileMailbox(EmailTransport): _variant = None _path = None def __init__(self, path): super(GenericFileMailbox, self).__init__() - self._path = path + self._path = path.encode( + sys.getfilesystemencoding() + ) def get_instance(self): return self._variant(self._path) diff --git a/django_mailbox/transports/gmail.py b/django_mailbox/transports/gmail.py new file mode 100644 index 0000000..0922f9a --- /dev/null +++ b/django_mailbox/transports/gmail.py @@ -0,0 +1,57 @@ +import logging + +from django_mailbox.transports.imap import ImapTransport + + +logger = logging.getLogger(__name__) + + +class GmailImapTransport(ImapTransport): + + def connect(self, username, password): + # Try to use oauth2 first. It's much safer + try: + self._connect_oauth(username) + except (TypeError, ValueError) as e: + logger.warning("Couldn't do oauth2 because %s" % e) + self.server = self.transport(self.hostname, self.port) + typ, msg = self.server.login(username, password) + self.server.select() + + def _connect_oauth(self, username): + # username should be an email address that has already been authorized + # for gmail access + try: + from django_mailbox.google_utils import ( + get_google_access_token, + fetch_user_info, + AccessTokenNotFound, + ) + except ImportError: + raise ValueError( + "Install python-social-auth to use oauth2 auth for gmail" + ) + + access_token = None + while access_token is None: + try: + access_token = get_google_access_token(username) + google_email_address = fetch_user_info(username)['email'] + except TypeError: + # This means that the google process took too long + # Trying again is the right thing to do + pass + except AccessTokenNotFound: + raise ValueError( + "No Token available in python-social-auth for %s" % ( + username + ) + ) + + auth_string = 'user=%s\1auth=Bearer %s\1\1' % ( + google_email_address, + access_token + ) + self.server = self.transport(self.hostname, self.port) + self.server.authenticate('XOAUTH2', lambda x: auth_string) + self.server.select() diff --git a/django_mailbox/transports/imap.py b/django_mailbox/transports/imap.py index 2ba3db2..b7f2cd1 100644 --- a/django_mailbox/transports/imap.py +++ b/django_mailbox/transports/imap.py @@ -1,11 +1,24 @@ -import email from imaplib import IMAP4, IMAP4_SSL +import logging + +from django.conf import settings + +from .base import EmailTransport, MessageParseError -class ImapTransport(object): - def __init__(self, hostname, port=None, ssl=False): +logger = logging.getLogger(__name__) + + +class ImapTransport(EmailTransport): + def __init__(self, hostname, port=None, ssl=False, archive=''): + self.max_message_size = getattr( + settings, + 'DJANGO_MAILBOX_MAX_MESSAGE_SIZE', + False + ) self.hostname = hostname self.port = port + self.archive = archive if ssl: self.transport = IMAP4_SSL if not self.port: @@ -20,19 +33,69 @@ class ImapTransport(object): typ, msg = self.server.login(username, password) self.server.select() - def get_message(self): - typ, inbox = self.server.search(None, 'ALL') + def _get_all_message_ids(self): + # Fetch all the message uids + response, message_ids = self.server.uid('search', None, 'ALL') + message_id_string = message_ids[0].strip() + # Usually `message_id_string` will be a list of space-separated + # ids; we must make sure that it isn't an empty string before + # splitting into individual UIDs. + if message_id_string: + return message_id_string.split(' ') + return [] - if not inbox[0]: + def _get_small_message_ids(self, message_ids): + # Using existing message uids, get the sizes and + # return only those that are under the size + # limit + safe_message_ids = [] + + status, data = self.server.uid( + 'fetch', + ','.join(message_ids), + '(RFC822.SIZE)' + ) + + for each_msg in data: + try: + uid = each_msg.split(' ')[2] + size = each_msg.split(' ')[4].rstrip(')') + if int(size) <= int(self.max_message_size): + safe_message_ids.append(uid) + except ValueError as e: + logger.warning( + "ValueError: %s working on %s" % (e, each_msg[0]) + ) + pass + return safe_message_ids + + def get_message(self): + message_ids = self._get_all_message_ids() + + if not message_ids: return - for key in inbox[0].split(): + # Limit the uids to the small ones if we care about that + if self.max_message_size: + message_ids = self._get_small_message_ids(message_ids) + + if self.archive: + typ, folders = self.server.list(pattern=self.archive) + if folders[0] is None: + # If the archive folder does not exist, create it + self.server.create(self.archive) + + for uid in message_ids: try: - typ, msg_contents = self.server.fetch(key, '(RFC822)') - message = email.message_from_string(msg_contents[0][1]) + typ, msg_contents = self.server.uid('fetch', uid, '(RFC822)') + message = self.get_email_from_bytes(msg_contents[0][1]) yield message - except email.Errors.MessageParseError: + except MessageParseError: continue - self.server.store(key, "+FLAGS", "\\Deleted") + + if self.archive: + self.server.uid('copy', uid, self.archive) + + self.server.uid('store', uid, "+FLAGS", "(\\Deleted)") self.server.expunge() return diff --git a/django_mailbox/transports/pop3.py b/django_mailbox/transports/pop3.py index 652fbe7..8520557 100644 --- a/django_mailbox/transports/pop3.py +++ b/django_mailbox/transports/pop3.py @@ -1,8 +1,11 @@ -import email +import six + from poplib import POP3, POP3_SSL +from .base import EmailTransport, MessageParseError -class Pop3Transport(object): + +class Pop3Transport(EmailTransport): def __init__(self, hostname, port=None, ssl=False): self.hostname = hostname self.port = port @@ -20,14 +23,21 @@ class Pop3Transport(object): self.server.user(username) self.server.pass_(password) + def get_message_body(self, message_lines): + if six.PY3: + return six.binary_type('\r\n', 'ascii').join(message_lines) + return '\r\n'.join(message_lines) + def get_message(self): message_count = len(self.server.list()[1]) for i in range(message_count): try: - msg_contents = "\r\n".join(self.server.retr(i + 1)[1]) - message = email.message_from_string(msg_contents) + msg_contents = self.get_message_body( + self.server.retr(i + 1)[1] + ) + message = self.get_email_from_bytes(msg_contents) yield message - except email.Errors.MessageParseError: + except MessageParseError: continue self.server.dele(i + 1) self.server.quit() diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index b977ce6..9a802e1 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -1,6 +1,8 @@ import email.header import logging +import six + from django.conf import settings @@ -10,16 +12,24 @@ logger = logging.getLogger(__name__) DEFAULT_CHARSET = getattr( settings, 'DJANGO_MAILBOX_DEFAULT_CHARSET', - 'ascii', + 'iso8859-1', ) -def decode_header(header): +def convert_header_to_unicode(header): + def _decode(value, encoding): + if isinstance(value, six.text_type): + return value + if not encoding or encoding == 'unknown-8bit': + encoding = DEFAULT_CHARSET + return value.decode(encoding, 'REPLACE') + try: return ''.join( [ - unicode(t[0], t[1] or DEFAULT_CHARSET) - for t in email.header.decode_header(header) + ( + _decode(bytestr, encoding) + ) for bytestr, encoding in email.header.decode_header(header) ] ) except UnicodeDecodeError: @@ -29,3 +39,42 @@ def decode_header(header): DEFAULT_CHARSET, ) return unicode(header, DEFAULT_CHARSET, 'replace') + + +def get_body_from_message(message, maintype, subtype): + """ + Fetchs the body message matching main/sub content type. + """ + body = six.text_type('') + for part in message.walk(): + if part.get_content_maintype() == maintype and \ + part.get_content_subtype() == subtype: + charset = part.get_content_charset() + this_part = part.get_payload(decode=True) + if charset: + try: + this_part = this_part.decode(charset, 'replace') + except LookupError: + this_part = this_part.decode('ascii', 'replace') + logger.warning( + 'Unknown encoding %s encountered while decoding ' + 'text payload. Interpreting as ASCII with ' + 'replacement, but some data may not be ' + 'represented as the sender intended.', + charset + ) + except ValueError: + this_part = this_part.decode('ascii', 'replace') + logger.warning( + 'Error encountered while decoding text ' + 'payload from an incorrectly-constructed ' + 'e-mail; payload was converted to ASCII with ' + 'replacement, but some data may not be ' + 'represented as the sender intended.' + ) + else: + this_part = this_part.decode('ascii', 'replace') + + body += this_part + + return body diff --git a/docs/conf.py b/docs/conf.py index 1ed33fb..aa866ac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,16 +41,16 @@ master_doc = 'index' # General information about the project. project = u'django-mailbox' -copyright = u'2013, Adam Coddington' +copyright = u'2014, Adam Coddington' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '3.0' +version = '3.3' # The full version, including alpha/beta/rc tags. -release = '3.0' +release = '3.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/index.rst b/docs/index.rst index 4d5c0b4..0ecaaa4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,8 +6,8 @@ Django-mailbox ============== -.. image:: https://travis-ci.org/latestrevision/django-mailbox.png?branch=master - :target: https://travis-ci.org/latestrevision/django-mailbox +.. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master + :target: https://travis-ci.org/coddingtonbear/django-mailbox How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, or had to otherwise construct an application driven by e-mail? @@ -20,9 +20,12 @@ Contents: .. toctree:: :maxdepth: 3 - :glob: - topics/* + topics/installation + topics/mailbox_types + topics/polling + topics/signal + topics/appendix Indices and tables diff --git a/docs/topics/appendix.rst b/docs/topics/appendix.rst new file mode 100644 index 0000000..c11f726 --- /dev/null +++ b/docs/topics/appendix.rst @@ -0,0 +1,8 @@ +Appendix +======== + +.. toctree:: + :maxdepth: 3 + :glob: + + appendix/* diff --git a/docs/topics/message-storage.rst b/docs/topics/appendix/message-storage.rst similarity index 100% rename from docs/topics/message-storage.rst rename to docs/topics/appendix/message-storage.rst diff --git a/docs/topics/settings.rst b/docs/topics/appendix/settings.rst similarity index 91% rename from docs/topics/settings.rst rename to docs/topics/appendix/settings.rst index 6023046..290e884 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/appendix/settings.rst @@ -58,3 +58,11 @@ Settings a message instance's ``get_email_object`` method being called. Value of this field is the primary key of the ``django_mailbox.MessageAttachment`` instance currently storing this payload component's contents. + +* ``DJANGO_MAILBOX_MAX_MESSAGE_SIZE`` + + * Default: ``False`` + * Type: ``integer`` + * If this is set, it will be read as a number of + bytes. Any messages above that size will not be + downloaded. ``2000000`` is 2 Megabytes. diff --git a/docs/topics/installation.rst b/docs/topics/installation.rst index 6286070..bb5cc5a 100644 --- a/docs/topics/installation.rst +++ b/docs/topics/installation.rst @@ -2,16 +2,36 @@ Installation ============ -You can either install from pip:: +1. You can either install from pip:: - pip install django-mailbox + pip install django-mailbox + + *or* checkout and install the source from the `github repository `_:: + + git clone https://github.com/coddingtonbear/django-mailbox.git + cd django-mailbox + python setup.py install -*or* checkout and install the source from the `github repository `_:: +2. After you have installed the package, + add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in + your project's ``settings.py`` file. - git clone https://github.com/latestrevision/django-mailbox.git - cd django-mailbox - python setup.py install +3. From your project folder, run ``python manage.py migrate django_mailbox`` to + create the required database tables. -After you have installed the package, -you should add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in your project's ``settings.py`` file. +4. Head to your project's Django Admin and create a mailbox to consume. + + +.. note:: + + Once you have entered a mailbox to consume, you can easily verify that you + have properly configured your mailbox by either: + + * From the Django Admin, using the 'Get New Mail' action from the action + dropdown on the Mailbox changelist + (http://yourproject.com/admin/django_mailbox/mailbox/). + * *Or* from a shell opened to your project's directory, using the + ``getmail`` management command by running:: + + python manage.py getmail diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index 0e60bae..e3c9244 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -7,22 +7,25 @@ POP3 and IMAP as well as local file-based mailboxes. .. table:: 'Protocol' Options - ============ ============== =============================================== - Mailbox Type 'Protocol':// Notes - ============ ============== =============================================== - POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` - IMAP ``imap://`` Can also specify SSL with ``imap+ssl://`` + ============ ================ ================================================================================================================================================================= + Mailbox Type 'Protocol':// Notes + ============ ================ ================================================================================================================================================================= + POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` + IMAP ``imap://`` Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI. + Gmail IMAP ``gmail+ssl://`` Uses OAuth authentication for Gmail's IMAP transport. See :ref:`gmail-oauth` for details. Maildir ``maildir://`` Mbox ``mbox://`` Babyl ``babyl://`` MH ``mh://`` MMDF ``mmdf://`` - Piped Mail *empty* See :ref:`receiving-mail-from-exim4-or-postfix` - ============ ============== =============================================== + Piped Mail *empty* See :ref:`receiving-mail-from-exim4-or-postfix` + ============ ================ ================================================================================================================================================================= + .. warning:: - This will delete any messages it can find in the inbox you specify; + Unless you are using IMAP's 'Archive' feature, + this will delete any messages it can find in the inbox you specify; do not use an e-mail inbox that you would like to share between applications. @@ -39,16 +42,44 @@ Basic IMAP Example: ``imap://username:password@server`` Basic POP3 Example: ``pop3://username:password@server`` Most mailboxes these days are SSL-enabled; -if yours is, add ``+ssl`` to your URI. +if yours is, add ``+ssl`` to the protocol section of your URI. Also, if your username or password include any non-ascii characters, they should be URL-encoded (for example, if your username includes an ``@``, it should be changed to ``%40`` in your URI). -If you have an account named ``youremailaddress@gmail.com`` with a password -of ``1234`` on GMail, which uses a POP3 server of 'pop.gmail.com' and requires -SSL, you would enter the following as your URI:: +If you are using an IMAP Mailbox, you can archive all messages before they +are deleted from the inbox. To archive emails, add the archive folder +name as a query paremeter to the uri. For example, if your mailbox has a +folder named ``myarchivefolder`` that you would like to copy messages to +after processing, add ``?archive=myarchivefolder`` to the end of the URI. - pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com +For a verbose example, if you have an account named +``youremailaddress@gmail.com`` with a password +of ``1234`` on GMail, which uses a IMAP server of ``imap.gmail.com`` (requiring +SSL) and you would like to archive all processed emails +into a folder named ``Archived``, you +would enter the following as your URI:: + + imap+ssl://youremailaddress%40gmail.com:1234@imap.gmail.com?archive=Archived + +.. _gmail-oauth: + +Gmail IMAP with Oauth2 authentication +------------------------------------- + +For added security, Gmail supports using OAuth2 for authentication_. +To handle the handshake and storing the credentials, use python-social-auth_. + +.. _authentication: https://developers.google.com/gmail/xoauth2_protocol +.. _python-social-auth: http://psa.matiasaguirre.net/ + +The Gmail Mailbox is also a regular IMAP mailbox, +but the password you specify will be ignored if OAuth2 authentication succeeds. +It will fall back to use your specified password as needed. + +Build your URI accordingly:: + + gmail+ssl://youremailaddress%40gmail.com:oauth2@imap.gmail.com?archive=Archived Local File-based Mailboxes diff --git a/docs/topics/polling.rst b/docs/topics/polling.rst index 7b52d47..3f1c970 100644 --- a/docs/topics/polling.rst +++ b/docs/topics/polling.rst @@ -18,8 +18,10 @@ this method will gather new messages from the server. Using the Django Admin ---------------------- -Check the box next to each of the mailboxes you'd like to fetch e-mail from, -and select the 'Get new mail' option. +From the 'Mailboxes' page in the Django Admin, +check the box next to each of the mailboxes you'd like to fetch e-mail from, +select 'Get new mail' from the action selector at the top of the list +of mailboxes, then click 'Go'. Using a cron job ---------------- @@ -58,8 +60,8 @@ start by adding a new router configuration to your Exim4 configuration like:: django_mailbox: debug_print = 'R: django_mailbox for $localpart@$domain' driver = accept - domains = +local_domains transport = send_to_django_mailbox + domains = mydomain.com local_parts = emailusernameone : emailusernametwo Make sure that the e-mail addresses you would like handled by Django Mailbox @@ -72,6 +74,14 @@ For example, if one of the e-mail addresses targeted at this machine is ``jane@example.com``, the contents of ``local_parts`` would be, simply ``jane``. +.. note:: + + If you would like messages addressed to *any* account *@mydomain.com* + to be delivered to django_mailbox, simply omit the above ``local_parts`` + setting. In the same vein, if you would like messages addressed to + any domain or any local domains, you can omit the ``domains`` setting + or set it to ``+local_domains`` respectively. + Next, a new transport configuration to your Exim4 configuration:: send_to_django_mailbox: diff --git a/readme.rst b/readme.rst index 8f23136..2a75204 100644 --- a/readme.rst +++ b/readme.rst @@ -1,12 +1,20 @@ .. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master :target: https://travis-ci.org/coddingtonbear/django-mailbox -How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, -or had to otherwise construct an application driven by e-mail? -One too many times, I'm sure. +.. image:: https://badge.fury.io/py/django-mailbox.png + :target: http://badge.fury.io/py/django-mailbox -This small Django application will allow you to specify mailboxes that you would like consumed for incoming content; -the e-mail will be stored, and you can process it at will (or, if you're in a hurry, by subscribing to a signal). +.. image:: https://pypip.in/d/django-mailbox/badge.png + :target: https://pypi.python.org/pypi/django-mailbox + + +Easily ingest messages from POP3, IMAP, or local mailboxes into your Django application. + +This app allows you to either ingest e-mail content from common e-mail services (as long as the service provides POP3 or IMAP support), +or directly recieve e-mail messages from ``stdin`` (for locally processing messages from Postfix or Exim4). + +These ingested messages will be stored in the database in Django models and you can process their content at will, +or -- if you're in a hurry -- by using a signal receiver. - Documentation for django-mailbox is available on `ReadTheDocs `_. @@ -14,10 +22,3 @@ the e-mail will be stored, and you can process it at will (or, if you're in a hu `Github `_. - Test status available on `Travis-CI `_. - - - -.. image:: https://d2weczhvl823v0.cloudfront.net/coddingtonbear/django-mailbox/trend.png - :alt: Bitdeli badge - :target: https://bitdeli.com/free - diff --git a/setup.py b/setup.py index a7557e6..3023819 100755 --- a/setup.py +++ b/setup.py @@ -1,13 +1,20 @@ -from setuptools import setup +from setuptools import find_packages, setup + +from django_mailbox import __version__ as version_string tests_require = [ 'django', + 'mock', +] + +gmail_oauth2_require = [ + 'python-social-auth', ] setup( name='django-mailbox', - version='3.0.3', - url='http://github.com/latestrevision/django-mailbox/', + version=version_string, + url='http://github.com/coddingtonbear/django-mailbox/', description=( 'Import mail from POP3, IMAP, local mailboxes or directly from ' 'Postfix or Exim4 into your Django application automatically.' @@ -15,7 +22,10 @@ setup( author='Adam Coddington', author_email='me@adamcoddington.net', tests_require=tests_require, - extras_require={'test': tests_require}, + extras_require={ + 'test': tests_require, + 'gmail-oauth2': gmail_oauth2_require + }, test_suite='django_mailbox.runtests.runtests', classifiers=[ 'Framework :: Django', @@ -32,15 +42,8 @@ setup( 'Topic :: Communications :: Email :: Post-Office :: POP3', 'Topic :: Communications :: Email :: Email Clients (MUA)', ], - packages=[ - 'django_mailbox', - 'django_mailbox.management', - 'django_mailbox.management.commands', - 'django_mailbox.migrations', - 'django_mailbox.transports', - 'django_mailbox.tests', - ], + packages=find_packages(), install_requires=[ - 'six' + 'six>=1.6.1' ] )