Merge pull request #1 from coddingtonbear/master

merge

.gitignore

@@ -1 +1,10 @@
 *.pyc
+bin/*
+lib/*
+src/*
+dist/*
+share/*
+docs/_build/*
+include/*
+.Python
+*egg*

.travis.yml

@@ -1,11 +1,22 @@
 language: python
+
 python:
   - "2.6"
   - "2.7"
-    - "3.2"
+  - "3.3"
+
 env:
-    - DJANGO=1.5.5
+  - DJANGO=1.4.14
-    - DJANGO=1.6.1
+  - DJANGO=1.5.9
+  - DJANGO=1.6.8
+  - DJANGO=1.7.1
+
+matrix:
+  exclude:
+    - env: DJANGO=1.4.14
+      python: "3.3"
+    - env: DJANGO=1.7.1
+      python: "2.6"
 install:
   - pip install -q Django==$DJANGO --use-mirrors
   - pip install -q -e . --use-mirrors

CHANGELOG.rst

@@ -0,0 +1,56 @@
+Changelog
+=========
+
+4.1
+---
+
+* Adds Django 1.7 migrations support.
+
+4.0
+---
+
+* Adds ``html`` property returning the HTML contents of 
+  ``django_mailbox.models.Message`` instances.
+  Thanks `@ariel17 <https://github.com/ariel17>`_!
+* Adds translation support.
+  Thanks `@ariel17 <https://github.com/ariel17>`_!
+* **Drops support for Python 3.2**.  The fact that only versions of
+  Python newer than 3.2 allow unicode literals has convinced me
+  that supporting Python 3.2 is probably more trouble than it's worth.
+  Please let me know if you were using Python 3.2, and I've left you
+  out in the cold; I'm willing to fix Python 3.2 support if it is
+  actively used.
+
+3.4
+---
+
+* Adds ``gmail`` transport allowing one to use Google
+  OAuth credentials for gathering messages from gmail.
+  Thanks `@alexlovelltroy <https://github.com/alexlovelltroy>`_!
+
+3.3
+---
+
+* Adds functionality to ``imap`` transport allowing one to
+  archive processed e-mails.
+  Thanks `@yellowcap <https://github.com/yellowcap>`_!
+
+3.2
+---
+
+* Fixes `#13 <https://github.com/coddingtonbear/django-mailbox/issues/13>`_;
+  Python 3 support had been broken for some time.  Thanks for catching that,
+  `@greendee <https://github.com/greendee>`_!
+
+3.1
+---
+
+* Fixes a wide variety of unicode-related errors.
+
+3.0
+---
+
+* Restructures message storage such that non-text message attachments
+  are stored as files, rather than in the database in their original
+  (probably base64-encoded) blobs.
+* So many new tests.

django_mailbox/__init__.py

@@ -0,0 +1 @@
+__version__ = '4.2.1'

django_mailbox/admin.py

@@ -1,3 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Model configuration in application ``django_mailbox`` for administration
+console.
+"""
+
 import logging
 
 from django.conf import settings
@@ -5,7 +13,8 @@ from django.contrib import admin
 
 from django_mailbox.models import MessageAttachment, Message, Mailbox
 from django_mailbox.signals import message_received
-from django_mailbox.utils import decode_header
+from django_mailbox.utils import convert_header_to_unicode
+
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +30,8 @@ def resend_message_received_signal(message_admin, request, queryset):
     for message in queryset.all():
         logger.debug('Resending \'message_received\' signal for %s' % message)
         message_received.send(sender=message_admin, message=message)
+
+
 resend_message_received_signal.short_description = (
     'Re-send message received signal'
 )
@@ -51,7 +62,13 @@ class MessageAdmin(admin.ModelAdmin):
         return msg.attachments.count()
 
     def subject(self, msg):
-        return decode_header(msg.subject)
+        return convert_header_to_unicode(msg.subject)
+
+    def envelope_headers(self, msg):
+        email = msg.get_email_object()
+        return '\n'.join(
+            [('%s: %s' % (h, v)) for h, v in email.items()]
+        )
 
     inlines = [
         MessageAttachmentInline,
@@ -78,7 +95,9 @@ class MessageAdmin(admin.ModelAdmin):
         'in_reply_to',
     )
     readonly_fields = (
+        'envelope_headers',
         'text',
+        'html',
     )
     actions = [resend_message_received_signal]
 

django_mailbox/google_utils.py

@@ -0,0 +1,111 @@
+import logging
+
+from django.conf import settings
+import requests
+from social.apps.django_app.default.models import UserSocialAuth
+
+
+logger = logging.getLogger(__name__)
+
+
+class AccessTokenNotFound(Exception):
+    pass
+
+
+class RefreshTokenNotFound(Exception):
+    pass
+
+
+def get_google_consumer_key():
+    return settings.SOCIAL_AUTH_GOOGLE_OAUTH2_KEY
+
+
+def get_google_consumer_secret():
+    return settings.SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET
+
+
+def get_google_access_token(email):
+    # TODO: This should be cacheable
+    try:
+        me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2")
+        return me.extra_data['access_token']
+    except (UserSocialAuth.DoesNotExist, KeyError):
+        raise AccessTokenNotFound
+
+
+def update_google_extra_data(email, extra_data):
+    try:
+        me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2")
+        me.extra_data = extra_data
+        me.save()
+    except (UserSocialAuth.DoesNotExist, KeyError):
+        raise AccessTokenNotFound
+
+
+def get_google_refresh_token(email):
+    try:
+        me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2")
+        return me.extra_data['refresh_token']
+    except (UserSocialAuth.DoesNotExist, KeyError):
+        raise RefreshTokenNotFound
+
+
+def google_api_get(email, url):
+    headers = dict(
+        Authorization="Bearer %s" % get_google_access_token(email),
+    )
+    r = requests.get(url, headers=headers)
+    logger.info("I got a %s", r.status_code)
+    if r.status_code == 401:
+        # Go use the refresh token
+        refresh_authorization(email)
+        r = requests.get(url, headers=headers)
+        logger.info("I got a %s", r.status_code)
+    if r.status_code == 200:
+        try:
+            return r.json()
+        except ValueError:
+            return r.text
+
+
+def google_api_post(email, url, post_data, authorized=True):
+    # TODO: Make this a lot less ugly. especially the 401 handling
+    headers = dict()
+    if authorized is True:
+        headers.update(dict(
+            Authorization="Bearer %s" % get_google_access_token(email),
+        ))
+    r = requests.post(url, headers=headers, data=post_data)
+    if r.status_code == 401:
+        refresh_authorization(email)
+        r = requests.post(url, headers=headers, data=post_data)
+    if r.status_code == 200:
+        try:
+            return r.json()
+        except ValueError:
+            return r.text
+
+
+def refresh_authorization(email):
+    refresh_token = get_google_refresh_token(email)
+    post_data = dict(
+        refresh_token=refresh_token,
+        client_id=get_google_consumer_key(),
+        client_secret=get_google_consumer_secret(),
+        grant_type='refresh_token',
+    )
+    results = google_api_post(
+        email,
+        "https://accounts.google.com/o/oauth2/token?access_type=offline",
+        post_data,
+        authorized=False)
+    results.update({'refresh_token': refresh_token})
+    update_google_extra_data(email, results)
+
+
+def fetch_user_info(email):
+    result = google_api_get(
+        email,
+        "https://www.googleapis.com/oauth2/v1/userinfo?alt=json"
+    )
+    return result

django_mailbox/management/commands/getmail.py

@@ -1,17 +1,30 @@
+import logging
+
 from django.core.management.base import BaseCommand
 
 from django_mailbox.models import Mailbox
 
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
 class Command(BaseCommand):
     def handle(self, *args, **options):
         mailboxes = Mailbox.active_mailboxes.all()
         if args:
-            mailboxes = mailboxes.filter(name = ' '.join(args))
+            mailboxes = mailboxes.filter(
+                name=' '.join(args)
+            )
         for mailbox in mailboxes:
-            self.stdout.write('Gathering messages for %s\n' % mailbox.name)
+            logger.info(
+                'Gathering messages for %s',
+                mailbox.name
+            )
             messages = mailbox.get_new_mail()
             for message in messages:
-                self.stdout.write('Received %s (from %s)\n' % (
+                logger.info(
+                    'Received %s (from %s)',
                     message.subject,
                     message.from_address
-                    ))
+                )

django_mailbox/management/commands/processincomingmessage.py

@@ -7,9 +7,11 @@ from django.core.management.base import BaseCommand
 
 from django_mailbox.models import Mailbox
 
+
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
+
 class Command(BaseCommand):
     args = "<[Mailbox Name (optional)]>"
     command = "Receive incoming mail via stdin"
@@ -22,7 +24,10 @@ class Command(BaseCommand):
             else:
                 mailbox = self.get_mailbox_for_message(message)
             mailbox.process_incoming_message(message)
-            logger.info("Message received from %s" % message['from'])
+            logger.info(
+                "Message received from %s",
+                message['from']
+            )
         else:
             logger.warning("Message not processable.")
 

django_mailbox/migrations/0001_initial.py

@@ -1,59 +1,59 @@
 # -*- coding: utf-8 -*-
-import datetime
+from __future__ import unicode_literals
-from south.db import db
+
-from south.v2 import SchemaMigration
+from django.db import models, migrations
-from django.db import models
 
 
-class Migration(SchemaMigration):
+class Migration(migrations.Migration):
 
-    def forwards(self, orm):
+    dependencies = [
-        # Adding model 'Mailbox'
+    ]
-        db.create_table('django_mailbox_mailbox', (
-            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
-            ('name', self.gf('django.db.models.fields.CharField')(max_length=255)),
-            ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)),
-        ))
-        db.send_create_signal('django_mailbox', ['Mailbox'])
 
-        # Adding model 'Message'
+    operations = [
-        db.create_table('django_mailbox_message', (
+        migrations.CreateModel(
-            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            name='Mailbox',
-            ('mailbox', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['django_mailbox.Mailbox'])),
+            fields=[
-            ('subject', self.gf('django.db.models.fields.CharField')(max_length=255)),
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-            ('message_id', self.gf('django.db.models.fields.CharField')(max_length=255)),
+                ('name', models.CharField(max_length=255, verbose_name='Name')),
-            ('from_address', self.gf('django.db.models.fields.CharField')(max_length=255)),
+                ('uri', models.CharField(default=None, max_length=255, blank=True, help_text="Example: imap+ssl://myusername:mypassword@someserver <br /><br />Internet transports include 'imap' and 'pop3'; common local file transports include 'maildir', 'mbox', and less commonly 'babyl', 'mh', and 'mmdf'. <br /><br />Be sure to urlencode your username and password should they contain illegal characters (like @, :, etc).", null=True, verbose_name='URI')),
-            ('body', self.gf('django.db.models.fields.TextField')()),
+                ('from_email', models.CharField(default=None, max_length=255, blank=True, help_text="Example: MailBot &lt;mailbot@yourdomain.com&gt;<br />'From' header to set for outgoing email.<br /><br />If you do not use this e-mail inbox for outgoing mail, this setting is unnecessary.<br />If you send e-mail without setting this, your 'From' header will'be set to match the setting `DEFAULT_FROM_EMAIL`.", null=True, verbose_name='From email')),
-            ('received', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+                ('active', models.BooleanField(default=True, help_text='Check this e-mail inbox for new e-mail messages during polling cycles.  This checkbox does not have an effect upon whether mail is collected here when this mailbox receives mail from a pipe, and does not affect whether e-mail messages can be dispatched from this mailbox. ', verbose_name='Active')),
-        ))
+            ],
-        db.send_create_signal('django_mailbox', ['Message'])
+            options={
-
+                'verbose_name_plural': 'Mailboxes',
-
-    def backwards(self, orm):
-        # Deleting model 'Mailbox'
-        db.delete_table('django_mailbox_mailbox')
-
-        # Deleting model 'Message'
-        db.delete_table('django_mailbox_message')
-
-
-    models = {
-        'django_mailbox.mailbox': {
-            'Meta': {'object_name': 'Mailbox'},
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
-            'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'})
             },
-        'django_mailbox.message': {
+            bases=(models.Model,),
-            'Meta': {'object_name': 'Message'},
+        ),
-            'body': ('django.db.models.fields.TextField', [], {}),
+        migrations.CreateModel(
-            'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            name='Message',
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            fields=[
-            'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['django_mailbox.Mailbox']"}),
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-            'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+                ('subject', models.CharField(max_length=255, verbose_name='Subject')),
-            'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+                ('message_id', models.CharField(max_length=255, verbose_name='Message ID')),
-            'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+                ('from_header', models.CharField(max_length=255, verbose_name='From header')),
-        }
+                ('to_header', models.TextField(verbose_name='To header')),
-    }
+                ('outgoing', models.BooleanField(default=False, verbose_name='Outgoing')),
-
+                ('body', models.TextField(verbose_name='Body')),
-    complete_apps = ['django_mailbox']
+                ('encoded', models.BooleanField(default=False, help_text='True if the e-mail body is Base64 encoded', verbose_name='Encoded')),
+                ('processed', models.DateTimeField(auto_now_add=True, verbose_name='Processed')),
+                ('read', models.DateTimeField(default=None, null=True, verbose_name='Read', blank=True)),
+                ('in_reply_to', models.ForeignKey(related_name='replies', verbose_name='In reply to', blank=True, to='django_mailbox.Message', null=True)),
+                ('mailbox', models.ForeignKey(related_name='messages', verbose_name='Mailbox', to='django_mailbox.Mailbox')),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='MessageAttachment',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('headers', models.TextField(null=True, verbose_name='Headers', blank=True)),
+                ('document', models.FileField(upload_to=b'mailbox_attachments/%Y/%m/%d/', verbose_name='Document')),
+                ('message', models.ForeignKey(related_name='attachments', verbose_name='Message', blank=True, to='django_mailbox.Message', null=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+    ]

django_mailbox/models.py

@@ -1,30 +1,39 @@
-import base64
+#!/usr/bin/env python
-import email
+# -*- coding: utf-8 -*-
-from email.header import decode_header
+
+"""
+Models declaration for application ``django_mailbox``.
+"""
+
+from email.encoders import encode_base64
 from email.message import Message as EmailMessage
 from email.utils import formatdate, parseaddr
-from email.encoders import encode_base64
+from quopri import encode as encode_quopri
-import urllib
+import base64
+import email
+import logging
 import mimetypes
 import os.path
-from quopri import encode as encode_quopri
 import sys
 import uuid
 
-try:
+import six
-    import urllib.parse as urlparse
+from six.moves.urllib.parse import parse_qs, unquote, urlparse
-except ImportError:
-    import urlparse
 
 from django.conf import settings
-from django.core.mail.message import make_msgid
 from django.core.files.base import ContentFile
+from django.core.mail.message import make_msgid
 from django.db import models
-from django_mailbox.transports import Pop3Transport, ImapTransport,\
+from django.utils.translation import ugettext as _
-    MaildirTransport, MboxTransport, BabylTransport, MHTransport, \
+
-    MMDFTransport
+from .utils import convert_header_to_unicode, get_body_from_message
 from django_mailbox.signals import message_received
-import six
+from django_mailbox.transports import Pop3Transport, ImapTransport, \
+    MaildirTransport, MboxTransport, BabylTransport, MHTransport, \
+    MMDFTransport, GmailImapTransport
+
+
+logger = logging.getLogger(__name__)
 
 
 STRIP_UNALLOWED_MIMETYPES = getattr(
@@ -32,6 +41,7 @@ STRIP_UNALLOWED_MIMETYPES = getattr(
     'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES',
     False
 )
+
 ALLOWED_MIMETYPES = getattr(
     settings,
     'DJANGO_MAILBOX_ALLOWED_MIMETYPES',
@@ -40,6 +50,7 @@ ALLOWED_MIMETYPES = getattr(
         'text/html'
     ]
 )
+
 TEXT_STORED_MIMETYPES = getattr(
     settings,
     'DJANGO_MAILBOX_TEXT_STORED_MIMETYPES',
@@ -48,11 +59,13 @@ TEXT_STORED_MIMETYPES = getattr(
         'text/html'
     ]
 )
+
 ALTERED_MESSAGE_HEADER = getattr(
     settings,
     'DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER',
     'X-Django-Mailbox-Altered-Message'
 )
+
 ATTACHMENT_INTERPOLATION_HEADER = getattr(
     settings,
     'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER',
@@ -61,17 +74,22 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr(
 
 
 class ActiveMailboxManager(models.Manager):
-    def get_query_set(self):
+    def get_queryset(self):
-        return super(ActiveMailboxManager, self).get_query_set().filter(
+        return super(ActiveMailboxManager, self).get_queryset().filter(
             active=True,
         )
 
 
 class Mailbox(models.Model):
-    name = models.CharField(max_length=255)
+    name = models.CharField(
-    uri = models.CharField(
+        _(u'Name'),
         max_length=255,
-        help_text=(
+    )
+
+    uri = models.CharField(
+        _(u'URI'),
+        max_length=255,
+        help_text=(_(
             "Example: imap+ssl://myusername:mypassword@someserver <br />"
             "<br />"
             "Internet transports include 'imap' and 'pop3'; "
@@ -80,14 +98,16 @@ class Mailbox(models.Model):
             "<br />"
             "Be sure to urlencode your username and password should they "
             "contain illegal characters (like @, :, etc)."
-        ),
+        )),
         blank=True,
         null=True,
         default=None,
     )
+
     from_email = models.CharField(
+        _(u'From email'),
         max_length=255,
-        help_text=(
+        help_text=(_(
             "Example: MailBot &lt;mailbot@yourdomain.com&gt;<br />"
             "'From' header to set for outgoing email.<br />"
             "<br />"
@@ -95,19 +115,21 @@ class Mailbox(models.Model):
             "setting is unnecessary.<br />"
             "If you send e-mail without setting this, your 'From' header will'"
             "be set to match the setting `DEFAULT_FROM_EMAIL`."
-        ),
+        )),
         blank=True,
         null=True,
         default=None,
     )
+
     active = models.BooleanField(
-        help_text=(
+        _(u'Active'),
+        help_text=(_(
             "Check this e-mail inbox for new e-mail messages during polling "
             "cycles.  This checkbox does not have an effect upon whether "
             "mail is collected here when this mailbox receives mail from a "
             "pipe, and does not affect whether e-mail messages can be "
             "dispatched from this mailbox. "
-        ),
+        )),
         blank=True,
         default=True,
     )
@@ -117,7 +139,11 @@ class Mailbox(models.Model):
 
     @property
     def _protocol_info(self):
-        return urlparse.urlparse(self.uri)
+        return urlparse(self.uri)
+
+    @property
+    def _query_string(self):
+        return parse_qs(self._protocol_info.query)
 
     @property
     def _domain(self):
@@ -129,11 +155,11 @@ class Mailbox(models.Model):
 
     @property
     def username(self):
-        return urllib.unquote(self._protocol_info.username)
+        return unquote(self._protocol_info.username)
 
     @property
     def password(self):
-        return urllib.unquote(self._protocol_info.password)
+        return unquote(self._protocol_info.password)
 
     @property
     def location(self):
@@ -150,6 +176,13 @@ class Mailbox(models.Model):
     def use_ssl(self):
         return '+ssl' in self._protocol_info.scheme.lower()
 
+    @property
+    def archive(self):
+        archive_folder = self._query_string.get('archive', None)
+        if not archive_folder:
+            return None
+        return archive_folder[0]
+
     def get_connection(self):
         if not self.uri:
             return None
@@ -157,7 +190,16 @@ class Mailbox(models.Model):
             conn = ImapTransport(
                 self.location,
                 port=self.port if self.port else None,
-                ssl=self.use_ssl
+                ssl=self.use_ssl,
+                archive=self.archive
+            )
+            conn.connect(self.username, self.password)
+        elif self.type == 'gmail':
+            conn = GmailImapTransport(
+                self.location,
+                port=self.port if self.port else None,
+                ssl=True,
+                archive=self.archive
             )
             conn.connect(self.username, self.password)
         elif self.type == 'pop3':
@@ -249,6 +291,36 @@ class Mailbox(models.Model):
             placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk)
             new = placeholder
         else:
+            content_charset = msg.get_content_charset()
+            if not content_charset:
+                content_charset = 'ascii'
+            try:
+                # Make sure that the payload can be properly decoded in the
+                # defined charset, if it can't, let's mash some things
+                # inside the payload :-\
+                msg.get_payload(decode=True).decode(content_charset)
+            except LookupError:
+                logger.warning(
+                    "Unknown encoding %s; interpreting as ASCII!",
+                    content_charset
+                )
+                msg.set_payload(
+                    msg.get_payload(decode=True).decode(
+                        'ascii',
+                        'ignore'
+                    )
+                )
+            except ValueError:
+                logger.warning(
+                    "Decoding error encountered; interpreting as ASCII!",
+                    content_charset
+                )
+                msg.set_payload(
+                    msg.get_payload(decode=True).decode(
+                        content_charset,
+                        'ignore'
+                    )
+                )
             new = msg
         return new
 
@@ -256,13 +328,15 @@ class Mailbox(models.Model):
         msg = Message()
         msg.mailbox = self
         if 'subject' in message:
-            msg.subject = message['subject'][0:255]
+            msg.subject = convert_header_to_unicode(message['subject'])[0:255]
         if 'message-id' in message:
             msg.message_id = message['message-id'][0:255]
         if 'from' in message:
-            msg.from_header = message['from']
+            msg.from_header = convert_header_to_unicode(message['from'])
         if 'to' in message:
-            msg.to_header = message['to']
+            msg.to_header = convert_header_to_unicode(message['to'])
+        elif 'Delivered-To' in message:
+            msg.to_header = convert_header_to_unicode(message['Delivered-To'])
         msg.save()
         message = self._get_dehydrated_message(message, msg)
         msg.set_body(message.as_string())
@@ -294,55 +368,83 @@ class Mailbox(models.Model):
 
 
 class IncomingMessageManager(models.Manager):
-    def get_query_set(self):
+    def get_queryset(self):
-        return super(IncomingMessageManager, self).get_query_set().filter(
+        return super(IncomingMessageManager, self).get_queryset().filter(
             outgoing=False,
         )
 
 
 class OutgoingMessageManager(models.Manager):
-    def get_query_set(self):
+    def get_queryset(self):
-        return super(OutgoingMessageManager, self).get_query_set().filter(
+        return super(OutgoingMessageManager, self).get_queryset().filter(
             outgoing=True,
         )
 
 
 class UnreadMessageManager(models.Manager):
-    def get_query_set(self):
+    def get_queryset(self):
-        return super(UnreadMessageManager, self).get_query_set().filter(
+        return super(UnreadMessageManager, self).get_queryset().filter(
             read=None
         )
 
 
 class Message(models.Model):
-    mailbox = models.ForeignKey(Mailbox, related_name='messages')
+    mailbox = models.ForeignKey(
-    subject = models.CharField(max_length=255)
+        Mailbox,
-    message_id = models.CharField(max_length=255)
+        related_name='messages',
+        verbose_name=_(u'Mailbox'),
+    )
+
+    subject = models.CharField(
+        _(u'Subject'),
+        max_length=255
+    )
+
+    message_id = models.CharField(
+        _(u'Message ID'),
+        max_length=255
+    )
+
     in_reply_to = models.ForeignKey(
         'django_mailbox.Message',
         related_name='replies',
         blank=True,
         null=True,
+        verbose_name=_(u'In reply to'),
     )
+
     from_header = models.CharField(
+        _('From header'),
         max_length=255,
     )
-    to_header = models.TextField()
+
+    to_header = models.TextField(
+        _(u'To header'),
+    )
+
     outgoing = models.BooleanField(
+        _(u'Outgoing'),
         default=False,
         blank=True,
     )
 
-    body = models.TextField()
+    body = models.TextField(
+        _(u'Body'),
+    )
+
     encoded = models.BooleanField(
+        _(u'Encoded'),
         default=False,
-        help_text='True if the e-mail body is Base64 encoded'
+        help_text=_('True if the e-mail body is Base64 encoded'),
     )
 
     processed = models.DateTimeField(
+        _('Processed'),
         auto_now_add=True
     )
+
     read = models.DateTimeField(
+        _(u'Read'),
         default=None,
         blank=True,
         null=True,
@@ -412,28 +514,22 @@ class Message(models.Model):
 
     @property
     def text(self):
-        return self.get_text_body()
+        """
-
+        Returns the message body matching content type 'text/plain'.
-    def get_text_body(self):
+        """
-        def get_body_from_message(message):
-            body = ''
-            for part in message.walk():
-                if (
-                    part.get_content_maintype() == 'text'
-                    and part.get_content_subtype() == 'plain'
-                ):
-                    charset = part.get_content_charset()
-                    this_part = part.get_payload(decode=True)
-                    if charset:
-                        this_part = this_part.decode(charset, 'replace')
-
-                    body += this_part
-            return body
-
         return get_body_from_message(
-            self.get_email_object()
+            self.get_email_object(), 'text', 'plain'
         ).replace('=\n', '').strip()
 
+    @property
+    def html(self):
+        """
+        Returns the message body matching content type 'text/html'.
+        """
+        return get_body_from_message(
+            self.get_email_object(), 'text', 'html'
+        ).replace('\n', '').strip()
+
     def _rehydrate(self, msg):
         new = EmailMessage()
         if msg.is_multipart():
@@ -495,7 +591,7 @@ class Message(models.Model):
         return self.body.encode('utf-8')
 
     def set_body(self, body):
-        if sys.version_info >= (3, 0):
+        if six.PY3:
             body = body.encode('utf-8')
         self.encoded = True
         self.body = base64.b64encode(body).decode('ascii')
@@ -503,10 +599,10 @@ class Message(models.Model):
     def get_email_object(self):
         """ Returns an `email.message.Message` instance for this message."""
         body = self.get_body()
-        if sys.version_info < (3, 0):
+        if six.PY3:
-            flat = email.message_from_string(body)
-        else:
             flat = email.message_from_bytes(body)
+        else:
+            flat = email.message_from_string(body)
         return self._rehydrate(flat)
 
     def delete(self, *args, **kwargs):
@@ -525,9 +621,19 @@ class MessageAttachment(models.Model):
         related_name='attachments',
         null=True,
         blank=True,
+        verbose_name=_('Message'),
+    )
+
+    headers = models.TextField(
+        _(u'Headers'),
+        null=True,
+        blank=True,
+    )
+
+    document = models.FileField(
+        _(u'Document'),
+        upload_to='mailbox_attachments/%Y/%m/%d/'
     )
-    headers = models.TextField(null=True, blank=True)
-    document = models.FileField(upload_to='mailbox_attachments/%Y/%m/%d/')
 
     def delete(self, *args, **kwargs):
         self.document.delete()
@@ -556,11 +662,10 @@ class MessageAttachment(models.Model):
 
     def get_filename(self):
         file_name = self._get_rehydrated_headers().get_filename()
-        if file_name:
+        if isinstance(file_name, six.text_type):
-            encoded_subject, encoding = decode_header(file_name)[0]
+            return file_name
-            if encoding:
+        elif file_name:
-                encoded_subject = encoded_subject.decode(encoding)
+            return convert_header_to_unicode(file_name)
-            return encoded_subject
         else:
             return None
 

django_mailbox/runtests.py

@@ -3,6 +3,10 @@ import sys
 
 from os.path import dirname, abspath
 
+try:
+    from django import setup
+except ImportError:
+    pass
 from django.conf import settings
 
 if not settings.configured:
@@ -27,6 +31,12 @@ def runtests(*test_args):
         test_args = ['django_mailbox']
     parent = dirname(abspath(__file__))
     sys.path.insert(0, parent)
+    try:
+        # ensure that AppRegistry has loaded
+        setup()
+    except NameError:
+        # This version of Django is too old for an app registry.
+        pass
     runner = DjangoTestSuiteRunner(
         verbosity=1,
         interactive=False,

django_mailbox/south_migrations/0001_initial.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Mailbox'
+        db.create_table('django_mailbox_mailbox', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)),
+        ))
+        db.send_create_signal('django_mailbox', ['Mailbox'])
+
+        # Adding model 'Message'
+        db.create_table('django_mailbox_message', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('mailbox', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['django_mailbox.Mailbox'])),
+            ('subject', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('message_id', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('from_address', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('body', self.gf('django.db.models.fields.TextField')()),
+            ('received', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+        ))
+        db.send_create_signal('django_mailbox', ['Message'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'Mailbox'
+        db.delete_table('django_mailbox_mailbox')
+
+        # Deleting model 'Message'
+        db.delete_table('django_mailbox_message')
+
+
+    models = {
+        'django_mailbox.mailbox': {
+            'Meta': {'object_name': 'Mailbox'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        'django_mailbox.message': {
+            'Meta': {'object_name': 'Message'},
+            'body': ('django.db.models.fields.TextField', [], {}),
+            'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['django_mailbox.Mailbox']"}),
+            'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['django_mailbox']

django_mailbox/tests/__init__.py

@@ -1,3 +1,4 @@
 from .test_mailbox import *
 from .test_message_flattening import *
 from .test_process_email import *
+from .test_transports import *

django_mailbox/tests/base.py

@@ -1,12 +1,23 @@
 import email
 import os.path
 
+import six
+
 from django.test import TestCase
-import sys
 
 from django_mailbox import models
 from django_mailbox.models import Mailbox, Message
 
+def get_email_as_text(name):
+    with open(
+        os.path.join(
+            os.path.dirname(__file__),
+            'messages',
+            name,
+        ),
+        'rb'
+    ) as f:
+        return f.read()
 
 class EmailMessageTestCase(TestCase):
     ALLOWED_EXTRA_HEADERS = [
@@ -22,7 +33,7 @@ class EmailMessageTestCase(TestCase):
         self.mailbox = Mailbox.objects.create()
         super(EmailMessageTestCase, self).setUp()
 
-    def _get_email_object(self, name):
+    def _get_email_as_text(self, name):
         with open(
             os.path.join(
                 os.path.dirname(__file__),
@@ -31,10 +42,14 @@ class EmailMessageTestCase(TestCase):
             ),
             'rb'
         ) as f:
-            if sys.version_info < (3, 0):
+            return f.read()
-                return email.message_from_string(f.read())
+
+    def _get_email_object(self, name):
+        copy = self._get_email_as_text(name)
+        if six.PY3:
+            return email.message_from_bytes(copy)
         else:
-                return email.message_from_bytes(f.read())
+            return email.message_from_string(copy)
 
     def _headers_identical(self, left, right, header=None):
         """ Check if headers are (close enough to) identical.

django_mailbox/tests/messages/generic_message.eml

@@ -1,5 +1,4 @@
 MIME-Version: 1.0
-Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 11:53:53 -0800 (PST)
 X-Originating-IP: [24.22.122.177]
 Date: Sun, 20 Jan 2013 11:53:53 -0800
 Delivered-To: test@adamcoddington.net

django_mailbox/tests/messages/message_with_attachment.eml

@@ -17,8 +17,8 @@ This message has an attachment.
 --047d7b33dd729737fe04d3bde348
 Content-Type: image/png; name="heart.png"
 Content-Disposition: attachment; filename="heart.png"
-Content-Transfer-Encoding: base64
 X-Attachment-Id: f_hc6mair60
+Content-Transfer-Encoding: base64
 
 iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAFoTx1HAAAAzUlEQVQoz32RWxXDIBBEr4NIQEIl
 ICESkFAJkRAJSIgEpEQCEqYfu6QUkn7sCcyDGQiSACKSKCAkGwBJwhDwZQNMEiYAIBdQvk7rfaHf

django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml

@@ -1,5 +1,5 @@
 Date: Fri, 30 Jan 2012 11:30:01 PST
-Subject: Tjest
+Subject: Tjest Wrong
 From: "Somebody" <somebody@somewhere.com>
 To: idontknow@somewhere.com
 MIME-Version: 1.0

django_mailbox/tests/messages/message_with_invalid_encoding.eml

@@ -0,0 +1,13 @@
+Reply-To: <private00@qq.com>
+From: "Refinance"<finance@khas.edu.tr>
+Subject: Apply For Loans @ 2% Per Annum
+Date: Sun, 19 Oct 2014 11:48:49 +0100
+MIME-Version: 1.0
+Content-Type: text/plain;
+	charset="_iso-2022-jp$ESC"
+Content-Transfer-Encoding: 7bit
+
+We offer loans to private individuals and corporate organizations at 2% interest rate. Interested serious applicants should apply via email with details of their requirements.
+
+Warm Regards,
+Loan Team

django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml

@@ -21,7 +21,7 @@ Received: from [144.206.32.69] by e.mail.ru with HTTP;
     Mon, 21 Apr 2014 17:01:45 +0400
 From: =?UTF-8?B?dGVzdCB0ZXN0?= <mr.test32@mail.ru>
 To: mr.test32@yandex.ru
-Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ!
+Subject: Узнай как зарабатывать от 1000$ в неделю!
 
 Content-Type: text/html;
     charset="iso-8859-1"

django_mailbox/tests/test_message_flattening.py

@@ -85,3 +85,20 @@ class TestMessageFlattening(EmailMessageTestCase):
             actual_email_object,
             expected_email_object,
         )
+
+    def test_message_processing_unknown_encoding(self):
+        incoming_email_object = self._get_email_object(
+            'message_with_invalid_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(incoming_email_object)
+
+        expected_text = (
+            "We offer loans to private individuals and corporate "
+            "organizations at 2% interest rate. Interested serious "
+            "applicants should apply via email with details of their "
+            "requirements.\n\nWarm Regards,\nLoan Team"
+        )
+        actual_text = msg.text
+
+        self.assertEqual(actual_text, expected_text)

django_mailbox/tests/test_process_email.py

@@ -1,4 +1,5 @@
 import os.path
+import sys
 
 import six
 
@@ -64,7 +65,7 @@ class TestProcessEmail(EmailMessageTestCase):
         msg = mailbox.process_incoming_message(message)
 
         expected_results = 'Hello there!'
-        actual_results = msg.get_text_body().strip()
+        actual_results = msg.text.strip()
 
         self.assertEqual(
             expected_results,
@@ -79,7 +80,7 @@ class TestProcessEmail(EmailMessageTestCase):
 
         message.get_email_object = lambda: email_object
 
-        actual_text = message.get_text_body()
+        actual_text = message.text
         expected_text = (
             'The one of us with a bike pump is far ahead, '
             'but a man stopped to help us and gave us his pump.'
@@ -121,11 +122,11 @@ class TestProcessEmail(EmailMessageTestCase):
 
         msg = self.mailbox.process_incoming_message(email_object)
 
-        actual_text = msg.get_text_body()
         expected_text = six.u(
             'This message contains funny UTF16 characters like this one: '
             '"\xc2\xa0" and this one "\xe2\x9c\xbf".'
         )
+        actual_text = msg.text
 
         self.assertEqual(
             expected_text,
@@ -150,7 +151,7 @@ class TestProcessEmail(EmailMessageTestCase):
 
         msg = self.mailbox.process_incoming_message(email_object)
 
-        msg.get_text_body()
+        msg.text
 
     def test_message_with_valid_content_in_single_byte_encoding(self):
         email_object = self._get_email_object(
@@ -159,8 +160,7 @@ class TestProcessEmail(EmailMessageTestCase):
 
         msg = self.mailbox.process_incoming_message(email_object)
 
-        actual_body = msg.get_text_body()
+        actual_text = msg.text
-
         expected_body = six.u(
             '\u042d\u0442\u043e '
             '\u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0435 '
@@ -171,6 +171,32 @@ class TestProcessEmail(EmailMessageTestCase):
         )
 
         self.assertEqual(
-            actual_body,
+            actual_text,
             expected_body,
         )
+
+    def test_message_with_single_byte_subject_encoding(self):
+        email_object = self._get_email_object(
+            'message_with_single_byte_extended_subject_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(email_object)
+
+        expected_subject = six.u(
+            '\u00D3\u00E7\u00ED\u00E0\u00E9 \u00EA\u00E0\u00EA '
+            '\u00E7\u00E0\u00F0\u00E0\u00E1\u00E0\u00F2\u00FB\u00E2'
+            '\u00E0\u00F2\u00FC \u00EE\u00F2 1000$ \u00E2 '
+            '\u00ED\u00E5\u00E4\u00E5\u00EB\u00FE!'
+        )
+        actual_subject = msg.subject
+        self.assertEqual(actual_subject, expected_subject)
+
+        if sys.version_info >= (3, 3):
+            # There were various bugfixes in Py3k's email module,
+            # this is apparently one of them.
+            expected_from = six.u('test test <mr.test32@mail.ru>')
+        else:
+            expected_from = six.u('test test<mr.test32@mail.ru>')
+        actual_from = msg.from_header
+
+        self.assertEqual(expected_from, actual_from)

django_mailbox/tests/test_transports.py

@@ -0,0 +1,132 @@
+import mock
+
+from django.test.utils import override_settings
+
+from django_mailbox.tests.base import EmailMessageTestCase, get_email_as_text
+from django_mailbox.transports import ImapTransport, Pop3Transport
+
+FAKE_UID_SEARCH_ANSWER = ('OK', ['18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44'])
+FAKE_UID_FETCH_SIZES = ('OK', ['1 (UID 18 RFC822.SIZE 58070000000)', '2 (UID 19 RFC822.SIZE 2593)'])
+FAKE_UID_FETCH_MSG = ('OK', [('1 (UID 18 RFC822 {5807}', get_email_as_text('generic_message.eml') ),])
+FAKE_UID_COPY_MSG = ('OK', ['[COPYUID 1 2 2] (Success)'])
+FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS = ('OK', ['(\\HasNoChildren \\All) "/" "[Gmail]/All Mail"'])
+
+class IMAPTestCase(EmailMessageTestCase):
+    def setUp(self):
+        def imap_server_uid_method(*args):
+            cmd = args[0]
+            arg2 = args[2]
+            if cmd == 'search':
+                return FAKE_UID_SEARCH_ANSWER
+            if cmd == 'copy':
+                return FAKE_UID_COPY_MSG
+            if cmd == 'fetch':
+                if arg2 == '(RFC822.SIZE)':
+                    return FAKE_UID_FETCH_SIZES
+                if arg2 == '(RFC822)':
+                    return FAKE_UID_FETCH_MSG
+        def imap_server_list_method(pattern=None):
+            return FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS
+
+        self.imap_server = mock.Mock()
+        self.imap_server.uid = imap_server_uid_method
+        self.imap_server.list = imap_server_list_method
+        super(IMAPTestCase, self).setUp()
+
+
+class TestImapTransport(IMAPTestCase):
+    def setUp(self):
+        super(TestImapTransport, self).setUp()
+        self.arbitrary_hostname = 'one.two.three'
+        self.arbitrary_port = 100
+        self.ssl = False
+        self.transport = ImapTransport(
+            self.arbitrary_hostname,
+            self.arbitrary_port,
+            self.ssl
+        )
+        self.transport.server = self.imap_server
+
+    def test_get_email_message(self):
+        actual_messages = list(self.transport.get_message())
+        self.assertEqual(len(actual_messages), 27)
+        actual_message = actual_messages[0]
+        expected_message = self._get_email_object('generic_message.eml')
+        self.assertEqual(expected_message, actual_message)
+
+class TestImapArchivedTransport(TestImapTransport):
+    def setUp(self):
+        super(TestImapArchivedTransport, self).setUp()
+        self.archive = 'Archive'
+        self.transport = ImapTransport(
+            self.arbitrary_hostname,
+            self.arbitrary_port,
+            self.ssl,
+            self.archive
+        )
+        self.transport.server = self.imap_server
+
+class TestMaxSizeImapTransport(TestImapTransport):
+
+    @override_settings(DJANGO_MAILBOX_MAX_MESSAGE_SIZE=5807)
+    def setUp(self):
+        super(TestMaxSizeImapTransport, self).setUp()
+
+        self.transport = ImapTransport(
+            self.arbitrary_hostname,
+            self.arbitrary_port,
+            self.ssl,
+        )
+        self.transport.server = self.imap_server
+
+    def test_size_limit(self):
+        all_message_ids = self.transport._get_all_message_ids()
+        small_message_ids = self.transport._get_small_message_ids(all_message_ids)
+        self.assertEqual(len(small_message_ids), 1)
+
+    def test_get_email_message(self):
+        actual_messages = list(self.transport.get_message())
+        self.assertEqual(len(actual_messages), 1)
+        actual_message = actual_messages[0]
+        expected_message = self._get_email_object('generic_message.eml')
+        self.assertEqual(expected_message, actual_message)
+
+
+class TestPop3Transport(EmailMessageTestCase):
+    def setUp(self):
+        self.arbitrary_hostname = 'one.two.three'
+        self.arbitrary_port = 100
+        self.ssl = False
+        self.transport = Pop3Transport(
+            self.arbitrary_hostname,
+            self.arbitrary_port,
+            self.ssl
+        )
+        self.transport.server = None
+        super(TestPop3Transport, self).setUp()
+
+    def test_get_email_message(self):
+        with mock.patch.object(self.transport, 'server') as server:
+            # Consider this value arbitrary, the second parameter
+            # should have one entry per message in the inbox
+            server.list.return_value = [None, ['some_msg']]
+            server.retr.return_value = [
+                '+OK message follows',
+                [
+                    line.encode('ascii')
+                    for line in self._get_email_as_text(
+                        'generic_message.eml'
+                    ).decode('ascii').split('\n')
+                ],
+                10018,  # Some arbitrary size, ideally matching the above
+            ]
+
+            actual_messages = list(self.transport.get_message())
+
+        self.assertEqual(len(actual_messages), 1)
+
+        actual_message = actual_messages[0]
+        expected_message = self._get_email_object('generic_message.eml')
+
+        self.assertEqual(expected_message, actual_message)
+

django_mailbox/transports/__init__.py

@@ -5,3 +5,4 @@ from django_mailbox.transports.mbox import MboxTransport
 from django_mailbox.transports.babyl import BabylTransport
 from django_mailbox.transports.mh import MHTransport
 from django_mailbox.transports.mmdf import MMDFTransport
+from django_mailbox.transports.gmail import GmailImapTransport

django_mailbox/transports/base.py

@@ -0,0 +1,19 @@
+import email
+
+import six
+
+# Do *not* remove this, we need to use this in subclasses of EmailTransport
+if six.PY3:
+    from email.errors import MessageParseError
+else:
+    from email.Errors import MessageParseError
+
+
+class EmailTransport(object):
+    def get_email_from_bytes(self, contents):
+        if six.PY3:
+            message = email.message_from_bytes(contents)
+        else:
+            message = email.message_from_string(contents)
+
+        return message

django_mailbox/transports/generic.py

@@ -1,10 +1,17 @@
-class GenericFileMailbox(object):
+import sys
+
+from .base import EmailTransport
+
+
+class GenericFileMailbox(EmailTransport):
     _variant = None
     _path = None
 
     def __init__(self, path):
         super(GenericFileMailbox, self).__init__()
-        self._path = path
+        self._path = path.encode(
+            sys.getfilesystemencoding()
+        )
 
     def get_instance(self):
         return self._variant(self._path)

django_mailbox/transports/gmail.py

@@ -0,0 +1,57 @@
+import logging
+
+from django_mailbox.transports.imap import ImapTransport
+
+
+logger = logging.getLogger(__name__)
+
+
+class GmailImapTransport(ImapTransport):
+
+    def connect(self, username, password):
+        # Try to use oauth2 first.  It's much safer
+        try:
+            self._connect_oauth(username)
+        except (TypeError, ValueError) as e:
+            logger.warning("Couldn't do oauth2 because %s" % e)
+            self.server = self.transport(self.hostname, self.port)
+            typ, msg = self.server.login(username, password)
+            self.server.select()
+
+    def _connect_oauth(self, username):
+        # username should be an email address that has already been authorized
+        # for gmail access
+        try:
+            from django_mailbox.google_utils import (
+                get_google_access_token,
+                fetch_user_info,
+                AccessTokenNotFound,
+            )
+        except ImportError:
+            raise ValueError(
+                "Install python-social-auth to use oauth2 auth for gmail"
+            )
+
+        access_token = None
+        while access_token is None:
+            try:
+                access_token = get_google_access_token(username)
+                google_email_address = fetch_user_info(username)['email']
+            except TypeError:
+                # This means that the google process took too long
+                # Trying again is the right thing to do
+                pass
+            except AccessTokenNotFound:
+                raise ValueError(
+                    "No Token available in python-social-auth for %s" % (
+                        username
+                    )
+                )
+
+        auth_string = 'user=%s\1auth=Bearer %s\1\1' % (
+            google_email_address,
+            access_token
+        )
+        self.server = self.transport(self.hostname, self.port)
+        self.server.authenticate('XOAUTH2', lambda x: auth_string)
+        self.server.select()

django_mailbox/transports/imap.py

@@ -1,11 +1,24 @@
-import email
 from imaplib import IMAP4, IMAP4_SSL
+import logging
+
+from django.conf import settings
+
+from .base import EmailTransport, MessageParseError
 
 
-class ImapTransport(object):
+logger = logging.getLogger(__name__)
-    def __init__(self, hostname, port=None, ssl=False):
+
+
+class ImapTransport(EmailTransport):
+    def __init__(self, hostname, port=None, ssl=False, archive=''):
+        self.max_message_size = getattr(
+            settings,
+            'DJANGO_MAILBOX_MAX_MESSAGE_SIZE',
+            False
+        )
         self.hostname = hostname
         self.port = port
+        self.archive = archive
         if ssl:
             self.transport = IMAP4_SSL
             if not self.port:
@@ -20,19 +33,69 @@ class ImapTransport(object):
         typ, msg = self.server.login(username, password)
         self.server.select()
 
-    def get_message(self):
+    def _get_all_message_ids(self):
-        typ, inbox = self.server.search(None, 'ALL')
+        # Fetch all the message uids
+        response, message_ids = self.server.uid('search', None, 'ALL')
+        message_id_string = message_ids[0].strip()
+        # Usually `message_id_string` will be a list of space-separated
+        # ids; we must make sure that it isn't an empty string before
+        # splitting into individual UIDs.
+        if message_id_string:
+            return message_id_string.split(' ')
+        return []
 
-        if not inbox[0]:
+    def _get_small_message_ids(self, message_ids):
+        # Using existing message uids, get the sizes and
+        # return only those that are under the size
+        # limit
+        safe_message_ids = []
+
+        status, data = self.server.uid(
+            'fetch',
+            ','.join(message_ids),
+            '(RFC822.SIZE)'
+        )
+
+        for each_msg in data:
+            try:
+                uid = each_msg.split(' ')[2]
+                size = each_msg.split(' ')[4].rstrip(')')
+                if int(size) <= int(self.max_message_size):
+                    safe_message_ids.append(uid)
+            except ValueError as e:
+                logger.warning(
+                    "ValueError: %s working on %s" % (e, each_msg[0])
+                )
+                pass
+        return safe_message_ids
+
+    def get_message(self):
+        message_ids = self._get_all_message_ids()
+
+        if not message_ids:
             return
 
-        for key in inbox[0].split():
+        # Limit the uids to the small ones if we care about that
+        if self.max_message_size:
+            message_ids = self._get_small_message_ids(message_ids)
+
+        if self.archive:
+            typ, folders = self.server.list(pattern=self.archive)
+            if folders[0] is None:
+                # If the archive folder does not exist, create it
+                self.server.create(self.archive)
+
+        for uid in message_ids:
             try:
-                typ, msg_contents = self.server.fetch(key, '(RFC822)')
+                typ, msg_contents = self.server.uid('fetch', uid, '(RFC822)')
-                message = email.message_from_string(msg_contents[0][1])
+                message = self.get_email_from_bytes(msg_contents[0][1])
                 yield message
-            except email.Errors.MessageParseError:
+            except MessageParseError:
                 continue
-            self.server.store(key, "+FLAGS", "\\Deleted")
+
+            if self.archive:
+                self.server.uid('copy', uid, self.archive)
+
+            self.server.uid('store', uid, "+FLAGS", "(\\Deleted)")
         self.server.expunge()
         return

django_mailbox/transports/pop3.py

@@ -1,8 +1,11 @@
-import email
+import six
+
 from poplib import POP3, POP3_SSL
 
+from .base import EmailTransport, MessageParseError
 
-class Pop3Transport(object):
+
+class Pop3Transport(EmailTransport):
     def __init__(self, hostname, port=None, ssl=False):
         self.hostname = hostname
         self.port = port
@@ -20,14 +23,21 @@ class Pop3Transport(object):
         self.server.user(username)
         self.server.pass_(password)
 
+    def get_message_body(self, message_lines):
+        if six.PY3:
+            return six.binary_type('\r\n', 'ascii').join(message_lines)
+        return '\r\n'.join(message_lines)
+
     def get_message(self):
         message_count = len(self.server.list()[1])
         for i in range(message_count):
             try:
-                msg_contents = "\r\n".join(self.server.retr(i + 1)[1])
+                msg_contents = self.get_message_body(
-                message = email.message_from_string(msg_contents)
+                    self.server.retr(i + 1)[1]
+                )
+                message = self.get_email_from_bytes(msg_contents)
                 yield message
-            except email.Errors.MessageParseError:
+            except MessageParseError:
                 continue
             self.server.dele(i + 1)
         self.server.quit()

django_mailbox/utils.py

@@ -1,6 +1,8 @@
 import email.header
 import logging
 
+import six
+
 from django.conf import settings
 
 
@@ -10,16 +12,24 @@ logger = logging.getLogger(__name__)
 DEFAULT_CHARSET = getattr(
     settings,
     'DJANGO_MAILBOX_DEFAULT_CHARSET',
-    'ascii',
+    'iso8859-1',
 )
 
 
-def decode_header(header):
+def convert_header_to_unicode(header):
+    def _decode(value, encoding):
+        if isinstance(value, six.text_type):
+            return value
+        if not encoding or encoding == 'unknown-8bit':
+            encoding = DEFAULT_CHARSET
+        return value.decode(encoding, 'REPLACE')
+
     try:
         return ''.join(
             [
-                unicode(t[0], t[1] or DEFAULT_CHARSET)
+                (
-                for t in email.header.decode_header(header)
+                    _decode(bytestr, encoding)
+                ) for bytestr, encoding in email.header.decode_header(header)
             ]
         )
     except UnicodeDecodeError:
@@ -29,3 +39,42 @@ def decode_header(header):
             DEFAULT_CHARSET,
         )
         return unicode(header, DEFAULT_CHARSET, 'replace')
+
+
+def get_body_from_message(message, maintype, subtype):
+    """
+    Fetchs the body message matching main/sub content type.
+    """
+    body = six.text_type('')
+    for part in message.walk():
+        if part.get_content_maintype() == maintype and \
+                part.get_content_subtype() == subtype:
+            charset = part.get_content_charset()
+            this_part = part.get_payload(decode=True)
+            if charset:
+                try:
+                    this_part = this_part.decode(charset, 'replace')
+                except LookupError:
+                    this_part = this_part.decode('ascii', 'replace')
+                    logger.warning(
+                        'Unknown encoding %s encountered while decoding '
+                        'text payload.  Interpreting as ASCII with '
+                        'replacement, but some data may not be '
+                        'represented as the sender intended.',
+                        charset
+                    )
+                except ValueError:
+                    this_part = this_part.decode('ascii', 'replace')
+                    logger.warning(
+                        'Error encountered while decoding text '
+                        'payload from an incorrectly-constructed '
+                        'e-mail; payload was converted to ASCII with '
+                        'replacement, but some data may not be '
+                        'represented as the sender intended.'
+                    )
+            else:
+                this_part = this_part.decode('ascii', 'replace')
+
+            body += this_part
+
+    return body

docs/conf.py

@@ -41,16 +41,16 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'django-mailbox'
-copyright = u'2013, Adam Coddington'
+copyright = u'2014, Adam Coddington'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '3.0'
+version = '3.3'
 # The full version, including alpha/beta/rc tags.
-release = '3.0'
+release = '3.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -6,8 +6,8 @@
 Django-mailbox
 ==============
 
-.. image:: https://travis-ci.org/latestrevision/django-mailbox.png?branch=master
+.. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master
-   :target: https://travis-ci.org/latestrevision/django-mailbox
+   :target: https://travis-ci.org/coddingtonbear/django-mailbox
 
 How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, 
 or had to otherwise construct an application driven by e-mail?
@@ -20,9 +20,12 @@ Contents:
 
 .. toctree::
    :maxdepth: 3
-   :glob:
 
-   topics/*
+   topics/installation
+   topics/mailbox_types
+   topics/polling
+   topics/signal
+   topics/appendix
 
 
 Indices and tables

docs/topics/appendix.rst

@@ -0,0 +1,8 @@
+Appendix
+========
+
+.. toctree::
+   :maxdepth: 3
+   :glob:
+
+   appendix/*

docs/topics/appendix/settings.rst

@@ -58,3 +58,11 @@ Settings
     a message instance's ``get_email_object`` method being called.  Value of
     this field is the primary key of the ``django_mailbox.MessageAttachment``
     instance currently storing this payload component's contents.
+
+* ``DJANGO_MAILBOX_MAX_MESSAGE_SIZE``
+
+  * Default: ``False``
+  * Type: ``integer``
+  * If this is set, it will be read as a number of
+    bytes.  Any messages above that size will not be
+    downloaded.  ``2000000`` is 2 Megabytes.

docs/topics/installation.rst

@@ -2,16 +2,36 @@
 Installation
 ============
 
-You can either install from pip::
+1. You can either install from pip::
 
        pip install django-mailbox
    
-*or* checkout and install the source from the `github repository <https://github.com/latestrevision/django-mailbox/>`_::
+   *or* checkout and install the source from the `github repository <https://github.com/coddingtonbear/django-mailbox/>`_::
    
-    git clone https://github.com/latestrevision/django-mailbox.git
+       git clone https://github.com/coddingtonbear/django-mailbox.git
        cd django-mailbox
        python setup.py install
 
-After you have installed the package, 
+2. After you have installed the package, 
-you should add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in your project's ``settings.py`` file.
+   add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in
+   your project's ``settings.py`` file.
+
+3. From your project folder, run ``python manage.py migrate django_mailbox`` to
+   create the required database tables.
+
+4. Head to your project's Django Admin and create a mailbox to consume.
+
+
+.. note::
+
+   Once you have entered a mailbox to consume, you can easily verify that you
+   have properly configured your mailbox by either:
+
+   * From the Django Admin, using the 'Get New Mail' action from the action
+     dropdown on the Mailbox changelist 
+     (http://yourproject.com/admin/django_mailbox/mailbox/).
+   * *Or* from a shell opened to your project's directory, using the
+     ``getmail`` management command by running::
+
+       python manage.py getmail
 

docs/topics/mailbox_types.rst

@@ -7,22 +7,25 @@ POP3 and IMAP as well as local file-based mailboxes.
 
 .. table:: 'Protocol' Options
 
-  ============ ============== ===============================================
+  ============ ================ =================================================================================================================================================================
   Mailbox Type 'Protocol'://    Notes
-  ============ ============== ===============================================
+  ============ ================ =================================================================================================================================================================
   POP3         ``pop3://``      Can also specify SSL with ``pop3+ssl://``
-  IMAP         ``imap://``    Can also specify SSL with ``imap+ssl://``
+  IMAP         ``imap://``      Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI.
+  Gmail IMAP   ``gmail+ssl://`` Uses OAuth authentication for  Gmail's IMAP transport.  See :ref:`gmail-oauth` for details.
   Maildir      ``maildir://``
   Mbox         ``mbox://``
   Babyl        ``babyl://``
   MH           ``mh://``
   MMDF         ``mmdf://``
   Piped Mail   *empty*          See :ref:`receiving-mail-from-exim4-or-postfix`
-  ============ ============== ===============================================
+  ============ ================ =================================================================================================================================================================
+
 
 .. warning::
 
-   This will delete any messages it can find in the inbox you specify; 
+   Unless you are using IMAP's 'Archive' feature,
+   this will delete any messages it can find in the inbox you specify;
    do not use an e-mail inbox that you would like to share between
    applications.
 
@@ -39,16 +42,44 @@ Basic IMAP Example: ``imap://username:password@server``
 Basic POP3 Example: ``pop3://username:password@server``
 
 Most mailboxes these days are SSL-enabled; 
-if yours is, add ``+ssl`` to your URI.  
+if yours is, add ``+ssl`` to the protocol section of your URI.  
 Also, if your username or password include any non-ascii characters,
 they should be URL-encoded  (for example, if your username includes an
 ``@``, it should be changed to ``%40`` in your URI).
 
-If you have an account named ``youremailaddress@gmail.com`` with a password
+If you are using an IMAP Mailbox, you can archive all messages before they
-of ``1234`` on GMail, which uses a POP3 server of 'pop.gmail.com' and requires
+are deleted from the inbox. To archive emails, add the archive folder
-SSL, you would enter the following as your URI::
+name as a query paremeter to the uri.  For example, if your mailbox has a
+folder named ``myarchivefolder`` that you would like to copy messages to
+after processing, add ``?archive=myarchivefolder`` to the end of the URI.
 
-    pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com
+For a verbose example, if you have an account named
+``youremailaddress@gmail.com`` with a password
+of ``1234`` on GMail, which uses a IMAP server of ``imap.gmail.com`` (requiring
+SSL) and you would like to archive all processed emails
+into a folder named ``Archived``, you
+would enter the following as your URI::
+
+    imap+ssl://youremailaddress%40gmail.com:1234@imap.gmail.com?archive=Archived
+
+.. _gmail-oauth:
+
+Gmail IMAP with Oauth2 authentication
+-------------------------------------
+
+For added security, Gmail supports using OAuth2 for authentication_.
+To handle the handshake and storing the credentials, use python-social-auth_.
+
+.. _authentication: https://developers.google.com/gmail/xoauth2_protocol
+.. _python-social-auth: http://psa.matiasaguirre.net/
+
+The Gmail Mailbox is also a regular IMAP mailbox,
+but the password you specify will be ignored if OAuth2 authentication succeeds.
+It will fall back to use your specified password as needed.
+
+Build your URI accordingly::
+
+    gmail+ssl://youremailaddress%40gmail.com:oauth2@imap.gmail.com?archive=Archived
 
 
 Local File-based Mailboxes

docs/topics/polling.rst

@@ -18,8 +18,10 @@ this method will gather new messages from the server.
 Using the Django Admin
 ----------------------
 
-Check the box next to each of the mailboxes you'd like to fetch e-mail from, 
+From the 'Mailboxes' page in the Django Admin,
-and select the 'Get new mail' option.
+check the box next to each of the mailboxes you'd like to fetch e-mail from, 
+select 'Get new mail' from the action selector at the top of the list
+of mailboxes, then click 'Go'.
 
 Using a cron job
 ----------------
@@ -58,8 +60,8 @@ start by adding a new router configuration to your Exim4 configuration like::
   django_mailbox:
     debug_print = 'R: django_mailbox for $localpart@$domain'
     driver = accept
-    domains = +local_domains
     transport = send_to_django_mailbox
+    domains = mydomain.com
     local_parts = emailusernameone : emailusernametwo
 
 Make sure that the e-mail addresses you would like handled by Django Mailbox
@@ -72,6 +74,14 @@ For example, if one of the e-mail addresses targeted at this machine is
 ``jane@example.com``, 
 the contents of ``local_parts`` would be, simply ``jane``.
 
+.. note::
+
+   If you would like messages addressed to *any* account *@mydomain.com*
+   to be delivered to django_mailbox, simply omit the above ``local_parts``
+   setting.  In the same vein, if you would like messages addressed to
+   any domain or any local domains, you can omit the ``domains`` setting
+   or set it to ``+local_domains`` respectively.
+
 Next, a new transport configuration to your Exim4 configuration::
 
   send_to_django_mailbox:

readme.rst

@@ -1,12 +1,20 @@
 .. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master
    :target: https://travis-ci.org/coddingtonbear/django-mailbox
 
-How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, 
+.. image:: https://badge.fury.io/py/django-mailbox.png
-or had to otherwise construct an application driven by e-mail?
+    :target: http://badge.fury.io/py/django-mailbox
-One too many times, I'm sure.
 
-This small Django application will allow you to specify mailboxes that you would like consumed for incoming content; 
+.. image:: https://pypip.in/d/django-mailbox/badge.png
-the e-mail will be stored, and you can process it at will (or, if you're in a hurry, by subscribing to a signal).
+    :target: https://pypi.python.org/pypi/django-mailbox
+
+
+Easily ingest messages from POP3, IMAP, or local mailboxes into your Django application.
+
+This app allows you to either ingest e-mail content from common e-mail services (as long as the service provides POP3 or IMAP support),
+or directly recieve e-mail messages from ``stdin`` (for locally processing messages from Postfix or Exim4).
+
+These ingested messages will be stored in the database in Django models and you can process their content at will,
+or -- if you're in a hurry -- by using a signal receiver.
 
 - Documentation for django-mailbox is available on
   `ReadTheDocs <http://django-mailbox.readthedocs.org/>`_.
@@ -14,10 +22,3 @@ the e-mail will be stored, and you can process it at will (or, if you're in a hu
   `Github <http://github.com/coddingtonbear/django-mailbox/issues>`_.
 - Test status available on
   `Travis-CI <https://travis-ci.org/coddingtonbear/django-mailbox>`_.
-
-
-
-.. image:: https://d2weczhvl823v0.cloudfront.net/coddingtonbear/django-mailbox/trend.png
-   :alt: Bitdeli badge
-   :target: https://bitdeli.com/free
-

setup.py

@@ -1,13 +1,20 @@
-from setuptools import setup
+from setuptools import find_packages, setup
+
+from django_mailbox import __version__ as version_string
 
 tests_require = [
     'django',
+    'mock',
+]
+
+gmail_oauth2_require = [
+    'python-social-auth',
 ]
 
 setup(
     name='django-mailbox',
-    version='3.0.3',
+    version=version_string,
-    url='http://github.com/latestrevision/django-mailbox/',
+    url='http://github.com/coddingtonbear/django-mailbox/',
     description=(
         'Import mail from POP3, IMAP, local mailboxes or directly from '
         'Postfix or Exim4 into your Django application automatically.'
@@ -15,7 +22,10 @@ setup(
     author='Adam Coddington',
     author_email='me@adamcoddington.net',
     tests_require=tests_require,
-    extras_require={'test': tests_require},
+    extras_require={
+        'test': tests_require,
+        'gmail-oauth2': gmail_oauth2_require
+    },
     test_suite='django_mailbox.runtests.runtests',
     classifiers=[
         'Framework :: Django',
@@ -32,15 +42,8 @@ setup(
         'Topic :: Communications :: Email :: Post-Office :: POP3',
         'Topic :: Communications :: Email :: Email Clients (MUA)',
     ],
-    packages=[
+    packages=find_packages(),
-        'django_mailbox',
-        'django_mailbox.management',
-        'django_mailbox.management.commands',
-        'django_mailbox.migrations',
-        'django_mailbox.transports',
-        'django_mailbox.tests',
-    ],
     install_requires=[
-        'six'
+        'six>=1.6.1'
     ]
 )