Merge pull request #1 from coddingtonbear/master

merge
2026-07-10 06:48:19 +02:00 · 2015-03-16 16:44:53 +03:00 · 2015-03-16 16:44:53 +03:00 · 1612102c12
commit 1612102c12
parent d3a766ffd6 793f92cfcf
55 changed files with 1134 additions and 242 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1 +1,10 @@
 *.pyc
+bin/*
+lib/*
+src/*
+dist/*
+share/*
+docs/_build/*
+include/*
+.Python
+*egg*
--- a/.travis.yml
+++ b/.travis.yml
@ -1,11 +1,22 @@
 language: python
+
 python:
  - "2.6"
  - "2.7"
-    - "3.2"
+  - "3.3"
+
 env:
-    - DJANGO=1.5.5
-    - DJANGO=1.6.1
+  - DJANGO=1.4.14
+  - DJANGO=1.5.9
+  - DJANGO=1.6.8
+  - DJANGO=1.7.1
+
+matrix:
+  exclude:
+    - env: DJANGO=1.4.14
+      python: "3.3"
+    - env: DJANGO=1.7.1
+      python: "2.6"
 install:
  - pip install -q Django==$DJANGO --use-mirrors
  - pip install -q -e . --use-mirrors
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@ -0,0 +1,56 @@
+Changelog
+=========
+
+4.1
+---
+
+* Adds Django 1.7 migrations support.
+
+4.0
+---
+
+* Adds ``html`` property returning the HTML contents of 
+  ``django_mailbox.models.Message`` instances.
+  Thanks `@ariel17 <https://github.com/ariel17>`_!
+* Adds translation support.
+  Thanks `@ariel17 <https://github.com/ariel17>`_!
+* **Drops support for Python 3.2**.  The fact that only versions of
+  Python newer than 3.2 allow unicode literals has convinced me
+  that supporting Python 3.2 is probably more trouble than it's worth.
+  Please let me know if you were using Python 3.2, and I've left you
+  out in the cold; I'm willing to fix Python 3.2 support if it is
+  actively used.
+
+3.4
+---
+
+* Adds ``gmail`` transport allowing one to use Google
+  OAuth credentials for gathering messages from gmail.
+  Thanks `@alexlovelltroy <https://github.com/alexlovelltroy>`_!
+
+3.3
+---
+
+* Adds functionality to ``imap`` transport allowing one to
+  archive processed e-mails.
+  Thanks `@yellowcap <https://github.com/yellowcap>`_!
+
+3.2
+---
+
+* Fixes `#13 <https://github.com/coddingtonbear/django-mailbox/issues/13>`_;
+  Python 3 support had been broken for some time.  Thanks for catching that,
+  `@greendee <https://github.com/greendee>`_!
+
+3.1
+---
+
+* Fixes a wide variety of unicode-related errors.
+
+3.0
+---
+
+* Restructures message storage such that non-text message attachments
+  are stored as files, rather than in the database in their original
+  (probably base64-encoded) blobs.
+* So many new tests.
--- a/django_mailbox/init.py
+++ b/django_mailbox/init.py
@ -0,0 +1 @@
+__version__ = '4.2.1'
--- a/django_mailbox/admin.py
+++ b/django_mailbox/admin.py
@ -1,3 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Model configuration in application ``django_mailbox`` for administration
+console.
+"""
+
 import logging

 from django.conf import settings
@ -5,7 +13,8 @@ from django.contrib import admin

 from django_mailbox.models import MessageAttachment, Message, Mailbox
 from django_mailbox.signals import message_received
-from django_mailbox.utils import decode_header
+from django_mailbox.utils import convert_header_to_unicode
+

 logger = logging.getLogger(__name__)

@ -21,6 +30,8 @@ def resend_message_received_signal(message_admin, request, queryset):
    for message in queryset.all():
        logger.debug('Resending \'message_received\' signal for %s' % message)
        message_received.send(sender=message_admin, message=message)
+
+
 resend_message_received_signal.short_description = (
    'Re-send message received signal'
 )
@ -51,7 +62,13 @@ class MessageAdmin(admin.ModelAdmin):
        return msg.attachments.count()

    def subject(self, msg):
-        return decode_header(msg.subject)
+        return convert_header_to_unicode(msg.subject)
+
+    def envelope_headers(self, msg):
+        email = msg.get_email_object()
+        return '\n'.join(
+            [('%s: %s' % (h, v)) for h, v in email.items()]
+        )

    inlines = [
        MessageAttachmentInline,
@ -78,7 +95,9 @@ class MessageAdmin(admin.ModelAdmin):
        'in_reply_to',
    )
    readonly_fields = (
+        'envelope_headers',
        'text',
+        'html',
    )
    actions = [resend_message_received_signal]

--- a/django_mailbox/google_utils.py
+++ b/django_mailbox/google_utils.py
@ -0,0 +1,111 @@
+import logging
+
+from django.conf import settings
+import requests
+from social.apps.django_app.default.models import UserSocialAuth
+
+
+logger = logging.getLogger(__name__)
+
+
+class AccessTokenNotFound(Exception):
+    pass
+
+
+class RefreshTokenNotFound(Exception):
+    pass
+
+
+def get_google_consumer_key():
+    return settings.SOCIAL_AUTH_GOOGLE_OAUTH2_KEY
+
+
+def get_google_consumer_secret():
+    return settings.SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET
+
+
+def get_google_access_token(email):
+    # TODO: This should be cacheable
+    try:
+        me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2")
+        return me.extra_data['access_token']
+    except (UserSocialAuth.DoesNotExist, KeyError):
+        raise AccessTokenNotFound
+
+
+def update_google_extra_data(email, extra_data):
+    try:
+        me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2")
+        me.extra_data = extra_data
+        me.save()
+    except (UserSocialAuth.DoesNotExist, KeyError):
+        raise AccessTokenNotFound
+
+
+def get_google_refresh_token(email):
+    try:
+        me = UserSocialAuth.objects.get(uid=email, provider="google-oauth2")
+        return me.extra_data['refresh_token']
+    except (UserSocialAuth.DoesNotExist, KeyError):
+        raise RefreshTokenNotFound
+
+
+def google_api_get(email, url):
+    headers = dict(
+        Authorization="Bearer %s" % get_google_access_token(email),
+    )
+    r = requests.get(url, headers=headers)
+    logger.info("I got a %s", r.status_code)
+    if r.status_code == 401:
+        # Go use the refresh token
+        refresh_authorization(email)
+        r = requests.get(url, headers=headers)
+        logger.info("I got a %s", r.status_code)
+    if r.status_code == 200:
+        try:
+            return r.json()
+        except ValueError:
+            return r.text
+
+
+def google_api_post(email, url, post_data, authorized=True):
+    # TODO: Make this a lot less ugly. especially the 401 handling
+    headers = dict()
+    if authorized is True:
+        headers.update(dict(
+            Authorization="Bearer %s" % get_google_access_token(email),
+        ))
+    r = requests.post(url, headers=headers, data=post_data)
+    if r.status_code == 401:
+        refresh_authorization(email)
+        r = requests.post(url, headers=headers, data=post_data)
+    if r.status_code == 200:
+        try:
+            return r.json()
+        except ValueError:
+            return r.text
+
+
+def refresh_authorization(email):
+    refresh_token = get_google_refresh_token(email)
+    post_data = dict(
+        refresh_token=refresh_token,
+        client_id=get_google_consumer_key(),
+        client_secret=get_google_consumer_secret(),
+        grant_type='refresh_token',
+    )
+    results = google_api_post(
+        email,
+        "https://accounts.google.com/o/oauth2/token?access_type=offline",
+        post_data,
+        authorized=False)
+    results.update({'refresh_token': refresh_token})
+    update_google_extra_data(email, results)
+
+
+def fetch_user_info(email):
+    result = google_api_get(
+        email,
+        "https://www.googleapis.com/oauth2/v1/userinfo?alt=json"
+    )
+    return result
--- a/django_mailbox/management/commands/getmail.py
+++ b/django_mailbox/management/commands/getmail.py
@ -1,17 +1,30 @@
+import logging
+
 from django.core.management.base import BaseCommand

 from django_mailbox.models import Mailbox

+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
 class Command(BaseCommand):
    def handle(self, *args, **options):
        mailboxes = Mailbox.active_mailboxes.all()
        if args:
-            mailboxes = mailboxes.filter(name = ' '.join(args))
+            mailboxes = mailboxes.filter(
+                name=' '.join(args)
+            )
        for mailbox in mailboxes:
-            self.stdout.write('Gathering messages for %s\n' % mailbox.name)
+            logger.info(
+                'Gathering messages for %s',
+                mailbox.name
+            )
            messages = mailbox.get_new_mail()
            for message in messages:
-                self.stdout.write('Received %s (from %s)\n' % (
+                logger.info(
+                    'Received %s (from %s)',
                    message.subject,
                    message.from_address
-                    ))
+                )
--- a/django_mailbox/management/commands/processincomingmessage.py
+++ b/django_mailbox/management/commands/processincomingmessage.py
@ -7,9 +7,11 @@ from django.core.management.base import BaseCommand

 from django_mailbox.models import Mailbox

+
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)

+
 class Command(BaseCommand):
    args = "<[Mailbox Name (optional)]>"
    command = "Receive incoming mail via stdin"
@ -22,7 +24,10 @@ class Command(BaseCommand):
            else:
                mailbox = self.get_mailbox_for_message(message)
            mailbox.process_incoming_message(message)
-            logger.info("Message received from %s" % message['from'])
+            logger.info(
+                "Message received from %s",
+                message['from']
+            )
        else:
            logger.warning("Message not processable.")

--- a/django_mailbox/migrations/0001_initial.py
+++ b/django_mailbox/migrations/0001_initial.py
@ -1,59 +1,59 @@
 # -*- coding: utf-8 -*-
-import datetime
-from south.db import db
-from south.v2 import SchemaMigration
-from django.db import models
+from __future__ import unicode_literals
+
+from django.db import models, migrations


-class Migration(SchemaMigration):
+class Migration(migrations.Migration):

-    def forwards(self, orm):
-        # Adding model 'Mailbox'
-        db.create_table('django_mailbox_mailbox', (
-            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
-            ('name', self.gf('django.db.models.fields.CharField')(max_length=255)),
-            ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)),
-        ))
-        db.send_create_signal('django_mailbox', ['Mailbox'])
+    dependencies = [
+    ]

-        # Adding model 'Message'
-        db.create_table('django_mailbox_message', (
-            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
-            ('mailbox', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['django_mailbox.Mailbox'])),
-            ('subject', self.gf('django.db.models.fields.CharField')(max_length=255)),
-            ('message_id', self.gf('django.db.models.fields.CharField')(max_length=255)),
-            ('from_address', self.gf('django.db.models.fields.CharField')(max_length=255)),
-            ('body', self.gf('django.db.models.fields.TextField')()),
-            ('received', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
-        ))
-        db.send_create_signal('django_mailbox', ['Message'])
-
-
-    def backwards(self, orm):
-        # Deleting model 'Mailbox'
-        db.delete_table('django_mailbox_mailbox')
-
-        # Deleting model 'Message'
-        db.delete_table('django_mailbox_message')
-
-
-    models = {
-        'django_mailbox.mailbox': {
-            'Meta': {'object_name': 'Mailbox'},
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
-            'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+    operations = [
+        migrations.CreateModel(
+            name='Mailbox',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('name', models.CharField(max_length=255, verbose_name='Name')),
+                ('uri', models.CharField(default=None, max_length=255, blank=True, help_text="Example: imap+ssl://myusername:mypassword@someserver <br /><br />Internet transports include 'imap' and 'pop3'; common local file transports include 'maildir', 'mbox', and less commonly 'babyl', 'mh', and 'mmdf'. <br /><br />Be sure to urlencode your username and password should they contain illegal characters (like @, :, etc).", null=True, verbose_name='URI')),
+                ('from_email', models.CharField(default=None, max_length=255, blank=True, help_text="Example: MailBot &lt;mailbot@yourdomain.com&gt;<br />'From' header to set for outgoing email.<br /><br />If you do not use this e-mail inbox for outgoing mail, this setting is unnecessary.<br />If you send e-mail without setting this, your 'From' header will'be set to match the setting `DEFAULT_FROM_EMAIL`.", null=True, verbose_name='From email')),
+                ('active', models.BooleanField(default=True, help_text='Check this e-mail inbox for new e-mail messages during polling cycles.  This checkbox does not have an effect upon whether mail is collected here when this mailbox receives mail from a pipe, and does not affect whether e-mail messages can be dispatched from this mailbox. ', verbose_name='Active')),
+            ],
+            options={
+                'verbose_name_plural': 'Mailboxes',
            },
-        'django_mailbox.message': {
-            'Meta': {'object_name': 'Message'},
-            'body': ('django.db.models.fields.TextField', [], {}),
-            'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
-            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['django_mailbox.Mailbox']"}),
-            'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
-            'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
-            'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'})
-        }
-    }
-
-    complete_apps = ['django_mailbox']
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='Message',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('subject', models.CharField(max_length=255, verbose_name='Subject')),
+                ('message_id', models.CharField(max_length=255, verbose_name='Message ID')),
+                ('from_header', models.CharField(max_length=255, verbose_name='From header')),
+                ('to_header', models.TextField(verbose_name='To header')),
+                ('outgoing', models.BooleanField(default=False, verbose_name='Outgoing')),
+                ('body', models.TextField(verbose_name='Body')),
+                ('encoded', models.BooleanField(default=False, help_text='True if the e-mail body is Base64 encoded', verbose_name='Encoded')),
+                ('processed', models.DateTimeField(auto_now_add=True, verbose_name='Processed')),
+                ('read', models.DateTimeField(default=None, null=True, verbose_name='Read', blank=True)),
+                ('in_reply_to', models.ForeignKey(related_name='replies', verbose_name='In reply to', blank=True, to='django_mailbox.Message', null=True)),
+                ('mailbox', models.ForeignKey(related_name='messages', verbose_name='Mailbox', to='django_mailbox.Mailbox')),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.CreateModel(
+            name='MessageAttachment',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('headers', models.TextField(null=True, verbose_name='Headers', blank=True)),
+                ('document', models.FileField(upload_to=b'mailbox_attachments/%Y/%m/%d/', verbose_name='Document')),
+                ('message', models.ForeignKey(related_name='attachments', verbose_name='Message', blank=True, to='django_mailbox.Message', null=True)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+    ]
--- a/django_mailbox/models.py
+++ b/django_mailbox/models.py
@ -1,30 +1,39 @@
-import base64
-import email
-from email.header import decode_header
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Models declaration for application ``django_mailbox``.
+"""
+
+from email.encoders import encode_base64
 from email.message import Message as EmailMessage
 from email.utils import formatdate, parseaddr
-from email.encoders import encode_base64
-import urllib
+from quopri import encode as encode_quopri
+import base64
+import email
+import logging
 import mimetypes
 import os.path
-from quopri import encode as encode_quopri
 import sys
 import uuid

-try:
-    import urllib.parse as urlparse
-except ImportError:
-    import urlparse
+import six
+from six.moves.urllib.parse import parse_qs, unquote, urlparse

 from django.conf import settings
-from django.core.mail.message import make_msgid
 from django.core.files.base import ContentFile
+from django.core.mail.message import make_msgid
 from django.db import models
+from django.utils.translation import ugettext as _
+
+from .utils import convert_header_to_unicode, get_body_from_message
+from django_mailbox.signals import message_received
 from django_mailbox.transports import Pop3Transport, ImapTransport, \
    MaildirTransport, MboxTransport, BabylTransport, MHTransport, \
-    MMDFTransport
-from django_mailbox.signals import message_received
-import six
+    MMDFTransport, GmailImapTransport
+
+
+logger = logging.getLogger(__name__)


 STRIP_UNALLOWED_MIMETYPES = getattr(
@ -32,6 +41,7 @@ STRIP_UNALLOWED_MIMETYPES = getattr(
    'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES',
    False
 )
+
 ALLOWED_MIMETYPES = getattr(
    settings,
    'DJANGO_MAILBOX_ALLOWED_MIMETYPES',
@ -40,6 +50,7 @@ ALLOWED_MIMETYPES = getattr(
        'text/html'
    ]
 )
+
 TEXT_STORED_MIMETYPES = getattr(
    settings,
    'DJANGO_MAILBOX_TEXT_STORED_MIMETYPES',
@ -48,11 +59,13 @@ TEXT_STORED_MIMETYPES = getattr(
        'text/html'
    ]
 )
+
 ALTERED_MESSAGE_HEADER = getattr(
    settings,
    'DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER',
    'X-Django-Mailbox-Altered-Message'
 )
+
 ATTACHMENT_INTERPOLATION_HEADER = getattr(
    settings,
    'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER',
@ -61,17 +74,22 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr(


 class ActiveMailboxManager(models.Manager):
-    def get_query_set(self):
-        return super(ActiveMailboxManager, self).get_query_set().filter(
+    def get_queryset(self):
+        return super(ActiveMailboxManager, self).get_queryset().filter(
            active=True,
        )


 class Mailbox(models.Model):
-    name = models.CharField(max_length=255)
-    uri = models.CharField(
+    name = models.CharField(
+        _(u'Name'),
        max_length=255,
-        help_text=(
+    )
+
+    uri = models.CharField(
+        _(u'URI'),
+        max_length=255,
+        help_text=(_(
            "Example: imap+ssl://myusername:mypassword@someserver <br />"
            "<br />"
            "Internet transports include 'imap' and 'pop3'; "
@ -80,14 +98,16 @@ class Mailbox(models.Model):
            "<br />"
            "Be sure to urlencode your username and password should they "
            "contain illegal characters (like @, :, etc)."
-        ),
+        )),
        blank=True,
        null=True,
        default=None,
    )
+
    from_email = models.CharField(
+        _(u'From email'),
        max_length=255,
-        help_text=(
+        help_text=(_(
            "Example: MailBot &lt;mailbot@yourdomain.com&gt;<br />"
            "'From' header to set for outgoing email.<br />"
            "<br />"
@ -95,19 +115,21 @@ class Mailbox(models.Model):
            "setting is unnecessary.<br />"
            "If you send e-mail without setting this, your 'From' header will'"
            "be set to match the setting `DEFAULT_FROM_EMAIL`."
-        ),
+        )),
        blank=True,
        null=True,
        default=None,
    )
+
    active = models.BooleanField(
-        help_text=(
+        _(u'Active'),
+        help_text=(_(
            "Check this e-mail inbox for new e-mail messages during polling "
            "cycles.  This checkbox does not have an effect upon whether "
            "mail is collected here when this mailbox receives mail from a "
            "pipe, and does not affect whether e-mail messages can be "
            "dispatched from this mailbox. "
-        ),
+        )),
        blank=True,
        default=True,
    )
@ -117,7 +139,11 @@ class Mailbox(models.Model):

    @property
    def _protocol_info(self):
-        return urlparse.urlparse(self.uri)
+        return urlparse(self.uri)
+
+    @property
+    def _query_string(self):
+        return parse_qs(self._protocol_info.query)

    @property
    def _domain(self):
@ -129,11 +155,11 @@ class Mailbox(models.Model):

    @property
    def username(self):
-        return urllib.unquote(self._protocol_info.username)
+        return unquote(self._protocol_info.username)

    @property
    def password(self):
-        return urllib.unquote(self._protocol_info.password)
+        return unquote(self._protocol_info.password)

    @property
    def location(self):
@ -150,6 +176,13 @@ class Mailbox(models.Model):
    def use_ssl(self):
        return '+ssl' in self._protocol_info.scheme.lower()

+    @property
+    def archive(self):
+        archive_folder = self._query_string.get('archive', None)
+        if not archive_folder:
+            return None
+        return archive_folder[0]
+
    def get_connection(self):
        if not self.uri:
            return None
@ -157,7 +190,16 @@ class Mailbox(models.Model):
            conn = ImapTransport(
                self.location,
                port=self.port if self.port else None,
-                ssl=self.use_ssl
+                ssl=self.use_ssl,
+                archive=self.archive
+            )
+            conn.connect(self.username, self.password)
+        elif self.type == 'gmail':
+            conn = GmailImapTransport(
+                self.location,
+                port=self.port if self.port else None,
+                ssl=True,
+                archive=self.archive
            )
            conn.connect(self.username, self.password)
        elif self.type == 'pop3':
@ -249,6 +291,36 @@ class Mailbox(models.Model):
            placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk)
            new = placeholder
        else:
+            content_charset = msg.get_content_charset()
+            if not content_charset:
+                content_charset = 'ascii'
+            try:
+                # Make sure that the payload can be properly decoded in the
+                # defined charset, if it can't, let's mash some things
+                # inside the payload :-\
+                msg.get_payload(decode=True).decode(content_charset)
+            except LookupError:
+                logger.warning(
+                    "Unknown encoding %s; interpreting as ASCII!",
+                    content_charset
+                )
+                msg.set_payload(
+                    msg.get_payload(decode=True).decode(
+                        'ascii',
+                        'ignore'
+                    )
+                )
+            except ValueError:
+                logger.warning(
+                    "Decoding error encountered; interpreting as ASCII!",
+                    content_charset
+                )
+                msg.set_payload(
+                    msg.get_payload(decode=True).decode(
+                        content_charset,
+                        'ignore'
+                    )
+                )
            new = msg
        return new

@ -256,13 +328,15 @@ class Mailbox(models.Model):
        msg = Message()
        msg.mailbox = self
        if 'subject' in message:
-            msg.subject = message['subject'][0:255]
+            msg.subject = convert_header_to_unicode(message['subject'])[0:255]
        if 'message-id' in message:
            msg.message_id = message['message-id'][0:255]
        if 'from' in message:
-            msg.from_header = message['from']
+            msg.from_header = convert_header_to_unicode(message['from'])
        if 'to' in message:
-            msg.to_header = message['to']
+            msg.to_header = convert_header_to_unicode(message['to'])
+        elif 'Delivered-To' in message:
+            msg.to_header = convert_header_to_unicode(message['Delivered-To'])
        msg.save()
        message = self._get_dehydrated_message(message, msg)
        msg.set_body(message.as_string())
@ -294,55 +368,83 @@ class Mailbox(models.Model):


 class IncomingMessageManager(models.Manager):
-    def get_query_set(self):
-        return super(IncomingMessageManager, self).get_query_set().filter(
+    def get_queryset(self):
+        return super(IncomingMessageManager, self).get_queryset().filter(
            outgoing=False,
        )


 class OutgoingMessageManager(models.Manager):
-    def get_query_set(self):
-        return super(OutgoingMessageManager, self).get_query_set().filter(
+    def get_queryset(self):
+        return super(OutgoingMessageManager, self).get_queryset().filter(
            outgoing=True,
        )


 class UnreadMessageManager(models.Manager):
-    def get_query_set(self):
-        return super(UnreadMessageManager, self).get_query_set().filter(
+    def get_queryset(self):
+        return super(UnreadMessageManager, self).get_queryset().filter(
            read=None
        )


 class Message(models.Model):
-    mailbox = models.ForeignKey(Mailbox, related_name='messages')
-    subject = models.CharField(max_length=255)
-    message_id = models.CharField(max_length=255)
+    mailbox = models.ForeignKey(
+        Mailbox,
+        related_name='messages',
+        verbose_name=_(u'Mailbox'),
+    )
+
+    subject = models.CharField(
+        _(u'Subject'),
+        max_length=255
+    )
+
+    message_id = models.CharField(
+        _(u'Message ID'),
+        max_length=255
+    )
+
    in_reply_to = models.ForeignKey(
        'django_mailbox.Message',
        related_name='replies',
        blank=True,
        null=True,
+        verbose_name=_(u'In reply to'),
    )
+
    from_header = models.CharField(
+        _('From header'),
        max_length=255,
    )
-    to_header = models.TextField()
+
+    to_header = models.TextField(
+        _(u'To header'),
+    )
+
    outgoing = models.BooleanField(
+        _(u'Outgoing'),
        default=False,
        blank=True,
    )

-    body = models.TextField()
+    body = models.TextField(
+        _(u'Body'),
+    )
+
    encoded = models.BooleanField(
+        _(u'Encoded'),
        default=False,
-        help_text='True if the e-mail body is Base64 encoded'
+        help_text=_('True if the e-mail body is Base64 encoded'),
    )

    processed = models.DateTimeField(
+        _('Processed'),
        auto_now_add=True
    )
+
    read = models.DateTimeField(
+        _(u'Read'),
        default=None,
        blank=True,
        null=True,
@ -412,28 +514,22 @@ class Message(models.Model):

    @property
    def text(self):
-        return self.get_text_body()
-
-    def get_text_body(self):
-        def get_body_from_message(message):
-            body = ''
-            for part in message.walk():
-                if (
-                    part.get_content_maintype() == 'text'
-                    and part.get_content_subtype() == 'plain'
-                ):
-                    charset = part.get_content_charset()
-                    this_part = part.get_payload(decode=True)
-                    if charset:
-                        this_part = this_part.decode(charset, 'replace')
-
-                    body += this_part
-            return body
-
+        """
+        Returns the message body matching content type 'text/plain'.
+        """
        return get_body_from_message(
-            self.get_email_object()
+            self.get_email_object(), 'text', 'plain'
        ).replace('=\n', '').strip()

+    @property
+    def html(self):
+        """
+        Returns the message body matching content type 'text/html'.
+        """
+        return get_body_from_message(
+            self.get_email_object(), 'text', 'html'
+        ).replace('\n', '').strip()
+
    def _rehydrate(self, msg):
        new = EmailMessage()
        if msg.is_multipart():
@ -495,7 +591,7 @@ class Message(models.Model):
        return self.body.encode('utf-8')

    def set_body(self, body):
-        if sys.version_info >= (3, 0):
+        if six.PY3:
            body = body.encode('utf-8')
        self.encoded = True
        self.body = base64.b64encode(body).decode('ascii')
@ -503,10 +599,10 @@ class Message(models.Model):
    def get_email_object(self):
        """ Returns an `email.message.Message` instance for this message."""
        body = self.get_body()
-        if sys.version_info < (3, 0):
-            flat = email.message_from_string(body)
-        else:
+        if six.PY3:
            flat = email.message_from_bytes(body)
+        else:
+            flat = email.message_from_string(body)
        return self._rehydrate(flat)

    def delete(self, *args, **kwargs):
@ -525,9 +621,19 @@ class MessageAttachment(models.Model):
        related_name='attachments',
        null=True,
        blank=True,
+        verbose_name=_('Message'),
+    )
+
+    headers = models.TextField(
+        _(u'Headers'),
+        null=True,
+        blank=True,
+    )
+
+    document = models.FileField(
+        _(u'Document'),
+        upload_to='mailbox_attachments/%Y/%m/%d/'
    )
-    headers = models.TextField(null=True, blank=True)
-    document = models.FileField(upload_to='mailbox_attachments/%Y/%m/%d/')

    def delete(self, *args, **kwargs):
        self.document.delete()
@ -556,11 +662,10 @@ class MessageAttachment(models.Model):

    def get_filename(self):
        file_name = self._get_rehydrated_headers().get_filename()
-        if file_name:
-            encoded_subject, encoding = decode_header(file_name)[0]
-            if encoding:
-                encoded_subject = encoded_subject.decode(encoding)
-            return encoded_subject
+        if isinstance(file_name, six.text_type):
+            return file_name
+        elif file_name:
+            return convert_header_to_unicode(file_name)
        else:
            return None

--- a/django_mailbox/runtests.py
+++ b/django_mailbox/runtests.py
@ -3,6 +3,10 @@ import sys

 from os.path import dirname, abspath

+try:
+    from django import setup
+except ImportError:
+    pass
 from django.conf import settings

 if not settings.configured:
@ -27,6 +31,12 @@ def runtests(*test_args):
        test_args = ['django_mailbox']
    parent = dirname(abspath(__file__))
    sys.path.insert(0, parent)
+    try:
+        # ensure that AppRegistry has loaded
+        setup()
+    except NameError:
+        # This version of Django is too old for an app registry.
+        pass
    runner = DjangoTestSuiteRunner(
        verbosity=1,
        interactive=False,
--- a/django_mailbox/south_migrations/0001_initial.py
+++ b/django_mailbox/south_migrations/0001_initial.py
@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Mailbox'
+        db.create_table('django_mailbox_mailbox', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('uri', self.gf('django.db.models.fields.CharField')(max_length=255)),
+        ))
+        db.send_create_signal('django_mailbox', ['Mailbox'])
+
+        # Adding model 'Message'
+        db.create_table('django_mailbox_message', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('mailbox', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['django_mailbox.Mailbox'])),
+            ('subject', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('message_id', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('from_address', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('body', self.gf('django.db.models.fields.TextField')()),
+            ('received', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+        ))
+        db.send_create_signal('django_mailbox', ['Message'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'Mailbox'
+        db.delete_table('django_mailbox_mailbox')
+
+        # Deleting model 'Message'
+        db.delete_table('django_mailbox_message')
+
+
+    models = {
+        'django_mailbox.mailbox': {
+            'Meta': {'object_name': 'Mailbox'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        'django_mailbox.message': {
+            'Meta': {'object_name': 'Message'},
+            'body': ('django.db.models.fields.TextField', [], {}),
+            'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['django_mailbox.Mailbox']"}),
+            'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        }
+    }
+
+    complete_apps = ['django_mailbox']
--- a/django_mailbox/south_migrations/0002_auto__chg_field_mailbox_uri.py
+++ b/django_mailbox/south_migrations/0002_auto__chg_field_mailbox_uri.py
--- a/django_mailbox/south_migrations/0003_auto__add_field_mailbox_active.py
+++ b/django_mailbox/south_migrations/0003_auto__add_field_mailbox_active.py
--- a/django_mailbox/south_migrations/0004_auto__add_field_message_outgoing.py
+++ b/django_mailbox/south_migrations/0004_auto__add_field_message_outgoing.py
--- a/django_mailbox/south_migrations/0005_rename_fields.py
+++ b/django_mailbox/south_migrations/0005_rename_fields.py
--- a/django_mailbox/south_migrations/0006_auto__add_field_message_in_reply_to.py
+++ b/django_mailbox/south_migrations/0006_auto__add_field_message_in_reply_to.py
--- a/django_mailbox/south_migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py
+++ b/django_mailbox/south_migrations/0007_auto__del_field_message_address__add_field_message_from_header__add_fi.py
--- a/django_mailbox/south_migrations/0008_populate_from_to_fields.py
+++ b/django_mailbox/south_migrations/0008_populate_from_to_fields.py
--- a/django_mailbox/south_migrations/0009_remove_references_table.py
+++ b/django_mailbox/south_migrations/0009_remove_references_table.py
--- a/django_mailbox/south_migrations/0010_auto__add_field_mailbox_from_email.py
+++ b/django_mailbox/south_migrations/0010_auto__add_field_mailbox_from_email.py
--- a/django_mailbox/south_migrations/0011_auto__add_field_message_read.py
+++ b/django_mailbox/south_migrations/0011_auto__add_field_message_read.py
--- a/django_mailbox/south_migrations/0012_auto__add_messageattachment.py
+++ b/django_mailbox/south_migrations/0012_auto__add_messageattachment.py
--- a/django_mailbox/south_migrations/0013_auto__add_field_messageattachment_message.py
+++ b/django_mailbox/south_migrations/0013_auto__add_field_messageattachment_message.py
--- a/django_mailbox/south_migrations/0014_migrate_message_attachments.py
+++ b/django_mailbox/south_migrations/0014_migrate_message_attachments.py
--- a/django_mailbox/south_migrations/0015_auto__add_field_messageattachment_headers.py
+++ b/django_mailbox/south_migrations/0015_auto__add_field_messageattachment_headers.py
--- a/django_mailbox/south_migrations/0016_auto__add_field_message_encoded.py
+++ b/django_mailbox/south_migrations/0016_auto__add_field_message_encoded.py
--- a/django_mailbox/south_migrations/init.py
+++ b/django_mailbox/south_migrations/init.py
--- a/django_mailbox/tests/init.py
+++ b/django_mailbox/tests/init.py
@ -1,3 +1,4 @@
 from .test_mailbox import *
 from .test_message_flattening import *
 from .test_process_email import *
+from .test_transports import *
--- a/django_mailbox/tests/base.py
+++ b/django_mailbox/tests/base.py
@ -1,12 +1,23 @@
 import email
 import os.path

+import six
+
 from django.test import TestCase
-import sys

 from django_mailbox import models
 from django_mailbox.models import Mailbox, Message

+def get_email_as_text(name):
+    with open(
+        os.path.join(
+            os.path.dirname(__file__),
+            'messages',
+            name,
+        ),
+        'rb'
+    ) as f:
+        return f.read()

 class EmailMessageTestCase(TestCase):
    ALLOWED_EXTRA_HEADERS = [
@ -22,7 +33,7 @@ class EmailMessageTestCase(TestCase):
        self.mailbox = Mailbox.objects.create()
        super(EmailMessageTestCase, self).setUp()

-    def _get_email_object(self, name):
+    def _get_email_as_text(self, name):
        with open(
            os.path.join(
                os.path.dirname(__file__),
@ -31,10 +42,14 @@ class EmailMessageTestCase(TestCase):
            ),
            'rb'
        ) as f:
-            if sys.version_info < (3, 0):
-                return email.message_from_string(f.read())
+            return f.read()
+
+    def _get_email_object(self, name):
+        copy = self._get_email_as_text(name)
+        if six.PY3:
+            return email.message_from_bytes(copy)
        else:
-                return email.message_from_bytes(f.read())
+            return email.message_from_string(copy)

    def _headers_identical(self, left, right, header=None):
        """ Check if headers are (close enough to) identical.
--- a/django_mailbox/tests/messages/generic_message.eml
+++ b/django_mailbox/tests/messages/generic_message.eml
@ -1,5 +1,4 @@
 MIME-Version: 1.0
-Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 11:53:53 -0800 (PST)
 X-Originating-IP: [24.22.122.177]
 Date: Sun, 20 Jan 2013 11:53:53 -0800
 Delivered-To: test@adamcoddington.net
--- a/django_mailbox/tests/messages/message_with_attachment.eml
+++ b/django_mailbox/tests/messages/message_with_attachment.eml
@ -17,8 +17,8 @@ This message has an attachment.
 --047d7b33dd729737fe04d3bde348
 Content-Type: image/png; name="heart.png"
 Content-Disposition: attachment; filename="heart.png"
-Content-Transfer-Encoding: base64
 X-Attachment-Id: f_hc6mair60
+Content-Transfer-Encoding: base64

 iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAFoTx1HAAAAzUlEQVQoz32RWxXDIBBEr4NIQEIl
 ICESkFAJkRAJSIgEpEQCEqYfu6QUkn7sCcyDGQiSACKSKCAkGwBJwhDwZQNMEiYAIBdQvk7rfaHf
--- a/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml
+++ b/django_mailbox/tests/messages/message_with_invalid_content_for_declared_encoding.eml
@ -1,5 +1,5 @@
 Date: Fri, 30 Jan 2012 11:30:01 PST
-Subject: Tjest
+Subject: Tjest Wrong
 From: "Somebody" <somebody@somewhere.com>
 To: idontknow@somewhere.com
 MIME-Version: 1.0
--- a/django_mailbox/tests/messages/message_with_invalid_encoding.eml
+++ b/django_mailbox/tests/messages/message_with_invalid_encoding.eml
@ -0,0 +1,13 @@
+Reply-To: <private00@qq.com>
+From: "Refinance"<finance@khas.edu.tr>
+Subject: Apply For Loans @ 2% Per Annum
+Date: Sun, 19 Oct 2014 11:48:49 +0100
+MIME-Version: 1.0
+Content-Type: text/plain;
+	charset="_iso-2022-jp$ESC"
+Content-Transfer-Encoding: 7bit
+
+We offer loans to private individuals and corporate organizations at 2% interest rate. Interested serious applicants should apply via email with details of their requirements.
+
+Warm Regards,
+Loan Team
--- a/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml
+++ b/django_mailbox/tests/messages/message_with_single_byte_extended_subject_encoding.eml
@ -21,7 +21,7 @@ Received: from [144.206.32.69] by e.mail.ru with HTTP;
    Mon, 21 Apr 2014 17:01:45 +0400
 From: =?UTF-8?B?dGVzdCB0ZXN0?= <mr.test32@mail.ru>
 To: mr.test32@yandex.ru
-Subject: Óçíàé êàê çàðàáàòûâàòü îò 1000$ â íåäåëþ!
+Subject: Узнай как зарабатывать от 1000$ в неделю!

 Content-Type: text/html;
    charset="iso-8859-1"
--- a/django_mailbox/tests/test_message_flattening.py
+++ b/django_mailbox/tests/test_message_flattening.py
@ -85,3 +85,20 @@ class TestMessageFlattening(EmailMessageTestCase):
            actual_email_object,
            expected_email_object,
        )
+
+    def test_message_processing_unknown_encoding(self):
+        incoming_email_object = self._get_email_object(
+            'message_with_invalid_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(incoming_email_object)
+
+        expected_text = (
+            "We offer loans to private individuals and corporate "
+            "organizations at 2% interest rate. Interested serious "
+            "applicants should apply via email with details of their "
+            "requirements.\n\nWarm Regards,\nLoan Team"
+        )
+        actual_text = msg.text
+
+        self.assertEqual(actual_text, expected_text)
--- a/django_mailbox/tests/test_process_email.py
+++ b/django_mailbox/tests/test_process_email.py
@ -1,4 +1,5 @@
 import os.path
+import sys

 import six

@ -64,7 +65,7 @@ class TestProcessEmail(EmailMessageTestCase):
        msg = mailbox.process_incoming_message(message)

        expected_results = 'Hello there!'
-        actual_results = msg.get_text_body().strip()
+        actual_results = msg.text.strip()

        self.assertEqual(
            expected_results,
@ -79,7 +80,7 @@ class TestProcessEmail(EmailMessageTestCase):

        message.get_email_object = lambda: email_object

-        actual_text = message.get_text_body()
+        actual_text = message.text
        expected_text = (
            'The one of us with a bike pump is far ahead, '
            'but a man stopped to help us and gave us his pump.'
@ -121,11 +122,11 @@ class TestProcessEmail(EmailMessageTestCase):

        msg = self.mailbox.process_incoming_message(email_object)

-        actual_text = msg.get_text_body()
        expected_text = six.u(
            'This message contains funny UTF16 characters like this one: '
            '"\xc2\xa0" and this one "\xe2\x9c\xbf".'
        )
+        actual_text = msg.text

        self.assertEqual(
            expected_text,
@ -150,7 +151,7 @@ class TestProcessEmail(EmailMessageTestCase):

        msg = self.mailbox.process_incoming_message(email_object)

-        msg.get_text_body()
+        msg.text

    def test_message_with_valid_content_in_single_byte_encoding(self):
        email_object = self._get_email_object(
@ -159,8 +160,7 @@ class TestProcessEmail(EmailMessageTestCase):

        msg = self.mailbox.process_incoming_message(email_object)

-        actual_body = msg.get_text_body()
-
+        actual_text = msg.text
        expected_body = six.u(
            '\u042d\u0442\u043e '
            '\u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0435 '
@ -171,6 +171,32 @@ class TestProcessEmail(EmailMessageTestCase):
        )

        self.assertEqual(
-            actual_body,
+            actual_text,
            expected_body,
        )
+
+    def test_message_with_single_byte_subject_encoding(self):
+        email_object = self._get_email_object(
+            'message_with_single_byte_extended_subject_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(email_object)
+
+        expected_subject = six.u(
+            '\u00D3\u00E7\u00ED\u00E0\u00E9 \u00EA\u00E0\u00EA '
+            '\u00E7\u00E0\u00F0\u00E0\u00E1\u00E0\u00F2\u00FB\u00E2'
+            '\u00E0\u00F2\u00FC \u00EE\u00F2 1000$ \u00E2 '
+            '\u00ED\u00E5\u00E4\u00E5\u00EB\u00FE!'
+        )
+        actual_subject = msg.subject
+        self.assertEqual(actual_subject, expected_subject)
+
+        if sys.version_info >= (3, 3):
+            # There were various bugfixes in Py3k's email module,
+            # this is apparently one of them.
+            expected_from = six.u('test test <mr.test32@mail.ru>')
+        else:
+            expected_from = six.u('test test<mr.test32@mail.ru>')
+        actual_from = msg.from_header
+
+        self.assertEqual(expected_from, actual_from)
--- a/django_mailbox/tests/test_transports.py
+++ b/django_mailbox/tests/test_transports.py
@ -0,0 +1,132 @@
+import mock
+
+from django.test.utils import override_settings
+
+from django_mailbox.tests.base import EmailMessageTestCase, get_email_as_text
+from django_mailbox.transports import ImapTransport, Pop3Transport
+
+FAKE_UID_SEARCH_ANSWER = ('OK', ['18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44'])
+FAKE_UID_FETCH_SIZES = ('OK', ['1 (UID 18 RFC822.SIZE 58070000000)', '2 (UID 19 RFC822.SIZE 2593)'])
+FAKE_UID_FETCH_MSG = ('OK', [('1 (UID 18 RFC822 {5807}', get_email_as_text('generic_message.eml') ),])
+FAKE_UID_COPY_MSG = ('OK', ['[COPYUID 1 2 2] (Success)'])
+FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS = ('OK', ['(\\HasNoChildren \\All) "/" "[Gmail]/All Mail"'])
+
+class IMAPTestCase(EmailMessageTestCase):
+    def setUp(self):
+        def imap_server_uid_method(*args):
+            cmd = args[0]
+            arg2 = args[2]
+            if cmd == 'search':
+                return FAKE_UID_SEARCH_ANSWER
+            if cmd == 'copy':
+                return FAKE_UID_COPY_MSG
+            if cmd == 'fetch':
+                if arg2 == '(RFC822.SIZE)':
+                    return FAKE_UID_FETCH_SIZES
+                if arg2 == '(RFC822)':
+                    return FAKE_UID_FETCH_MSG
+        def imap_server_list_method(pattern=None):
+            return FAKE_LIST_ARCHIVE_FOLDERS_ANSWERS
+
+        self.imap_server = mock.Mock()
+        self.imap_server.uid = imap_server_uid_method
+        self.imap_server.list = imap_server_list_method
+        super(IMAPTestCase, self).setUp()
+
+
+class TestImapTransport(IMAPTestCase):
+    def setUp(self):
+        super(TestImapTransport, self).setUp()
+        self.arbitrary_hostname = 'one.two.three'
+        self.arbitrary_port = 100
+        self.ssl = False
+        self.transport = ImapTransport(
+            self.arbitrary_hostname,
+            self.arbitrary_port,
+            self.ssl
+        )
+        self.transport.server = self.imap_server
+
+    def test_get_email_message(self):
+        actual_messages = list(self.transport.get_message())
+        self.assertEqual(len(actual_messages), 27)
+        actual_message = actual_messages[0]
+        expected_message = self._get_email_object('generic_message.eml')
+        self.assertEqual(expected_message, actual_message)
+
+class TestImapArchivedTransport(TestImapTransport):
+    def setUp(self):
+        super(TestImapArchivedTransport, self).setUp()
+        self.archive = 'Archive'
+        self.transport = ImapTransport(
+            self.arbitrary_hostname,
+            self.arbitrary_port,
+            self.ssl,
+            self.archive
+        )
+        self.transport.server = self.imap_server
+
+class TestMaxSizeImapTransport(TestImapTransport):
+
+    @override_settings(DJANGO_MAILBOX_MAX_MESSAGE_SIZE=5807)
+    def setUp(self):
+        super(TestMaxSizeImapTransport, self).setUp()
+
+        self.transport = ImapTransport(
+            self.arbitrary_hostname,
+            self.arbitrary_port,
+            self.ssl,
+        )
+        self.transport.server = self.imap_server
+
+    def test_size_limit(self):
+        all_message_ids = self.transport._get_all_message_ids()
+        small_message_ids = self.transport._get_small_message_ids(all_message_ids)
+        self.assertEqual(len(small_message_ids), 1)
+
+    def test_get_email_message(self):
+        actual_messages = list(self.transport.get_message())
+        self.assertEqual(len(actual_messages), 1)
+        actual_message = actual_messages[0]
+        expected_message = self._get_email_object('generic_message.eml')
+        self.assertEqual(expected_message, actual_message)
+
+
+class TestPop3Transport(EmailMessageTestCase):
+    def setUp(self):
+        self.arbitrary_hostname = 'one.two.three'
+        self.arbitrary_port = 100
+        self.ssl = False
+        self.transport = Pop3Transport(
+            self.arbitrary_hostname,
+            self.arbitrary_port,
+            self.ssl
+        )
+        self.transport.server = None
+        super(TestPop3Transport, self).setUp()
+
+    def test_get_email_message(self):
+        with mock.patch.object(self.transport, 'server') as server:
+            # Consider this value arbitrary, the second parameter
+            # should have one entry per message in the inbox
+            server.list.return_value = [None, ['some_msg']]
+            server.retr.return_value = [
+                '+OK message follows',
+                [
+                    line.encode('ascii')
+                    for line in self._get_email_as_text(
+                        'generic_message.eml'
+                    ).decode('ascii').split('\n')
+                ],
+                10018,  # Some arbitrary size, ideally matching the above
+            ]
+
+            actual_messages = list(self.transport.get_message())
+
+        self.assertEqual(len(actual_messages), 1)
+
+        actual_message = actual_messages[0]
+        expected_message = self._get_email_object('generic_message.eml')
+
+        self.assertEqual(expected_message, actual_message)
+
--- a/django_mailbox/transports/init.py
+++ b/django_mailbox/transports/init.py
@ -5,3 +5,4 @@ from django_mailbox.transports.mbox import MboxTransport
 from django_mailbox.transports.babyl import BabylTransport
 from django_mailbox.transports.mh import MHTransport
 from django_mailbox.transports.mmdf import MMDFTransport
+from django_mailbox.transports.gmail import GmailImapTransport
--- a/django_mailbox/transports/base.py
+++ b/django_mailbox/transports/base.py
@ -0,0 +1,19 @@
+import email
+
+import six
+
+# Do *not* remove this, we need to use this in subclasses of EmailTransport
+if six.PY3:
+    from email.errors import MessageParseError
+else:
+    from email.Errors import MessageParseError
+
+
+class EmailTransport(object):
+    def get_email_from_bytes(self, contents):
+        if six.PY3:
+            message = email.message_from_bytes(contents)
+        else:
+            message = email.message_from_string(contents)
+
+        return message
--- a/django_mailbox/transports/generic.py
+++ b/django_mailbox/transports/generic.py
@ -1,10 +1,17 @@
-class GenericFileMailbox(object):
+import sys
+
+from .base import EmailTransport
+
+
+class GenericFileMailbox(EmailTransport):
    _variant = None
    _path = None

    def __init__(self, path):
        super(GenericFileMailbox, self).__init__()
-        self._path = path
+        self._path = path.encode(
+            sys.getfilesystemencoding()
+        )

    def get_instance(self):
        return self._variant(self._path)
--- a/django_mailbox/transports/gmail.py
+++ b/django_mailbox/transports/gmail.py
@ -0,0 +1,57 @@
+import logging
+
+from django_mailbox.transports.imap import ImapTransport
+
+
+logger = logging.getLogger(__name__)
+
+
+class GmailImapTransport(ImapTransport):
+
+    def connect(self, username, password):
+        # Try to use oauth2 first.  It's much safer
+        try:
+            self._connect_oauth(username)
+        except (TypeError, ValueError) as e:
+            logger.warning("Couldn't do oauth2 because %s" % e)
+            self.server = self.transport(self.hostname, self.port)
+            typ, msg = self.server.login(username, password)
+            self.server.select()
+
+    def _connect_oauth(self, username):
+        # username should be an email address that has already been authorized
+        # for gmail access
+        try:
+            from django_mailbox.google_utils import (
+                get_google_access_token,
+                fetch_user_info,
+                AccessTokenNotFound,
+            )
+        except ImportError:
+            raise ValueError(
+                "Install python-social-auth to use oauth2 auth for gmail"
+            )
+
+        access_token = None
+        while access_token is None:
+            try:
+                access_token = get_google_access_token(username)
+                google_email_address = fetch_user_info(username)['email']
+            except TypeError:
+                # This means that the google process took too long
+                # Trying again is the right thing to do
+                pass
+            except AccessTokenNotFound:
+                raise ValueError(
+                    "No Token available in python-social-auth for %s" % (
+                        username
+                    )
+                )
+
+        auth_string = 'user=%s\1auth=Bearer %s\1\1' % (
+            google_email_address,
+            access_token
+        )
+        self.server = self.transport(self.hostname, self.port)
+        self.server.authenticate('XOAUTH2', lambda x: auth_string)
+        self.server.select()
--- a/django_mailbox/transports/imap.py
+++ b/django_mailbox/transports/imap.py
@ -1,11 +1,24 @@
-import email
 from imaplib import IMAP4, IMAP4_SSL
+import logging
+
+from django.conf import settings
+
+from .base import EmailTransport, MessageParseError


-class ImapTransport(object):
-    def __init__(self, hostname, port=None, ssl=False):
+logger = logging.getLogger(__name__)
+
+
+class ImapTransport(EmailTransport):
+    def __init__(self, hostname, port=None, ssl=False, archive=''):
+        self.max_message_size = getattr(
+            settings,
+            'DJANGO_MAILBOX_MAX_MESSAGE_SIZE',
+            False
+        )
        self.hostname = hostname
        self.port = port
+        self.archive = archive
        if ssl:
            self.transport = IMAP4_SSL
            if not self.port:
@ -20,19 +33,69 @@ class ImapTransport(object):
        typ, msg = self.server.login(username, password)
        self.server.select()

-    def get_message(self):
-        typ, inbox = self.server.search(None, 'ALL')
+    def _get_all_message_ids(self):
+        # Fetch all the message uids
+        response, message_ids = self.server.uid('search', None, 'ALL')
+        message_id_string = message_ids[0].strip()
+        # Usually `message_id_string` will be a list of space-separated
+        # ids; we must make sure that it isn't an empty string before
+        # splitting into individual UIDs.
+        if message_id_string:
+            return message_id_string.split(' ')
+        return []

-        if not inbox[0]:
+    def _get_small_message_ids(self, message_ids):
+        # Using existing message uids, get the sizes and
+        # return only those that are under the size
+        # limit
+        safe_message_ids = []
+
+        status, data = self.server.uid(
+            'fetch',
+            ','.join(message_ids),
+            '(RFC822.SIZE)'
+        )
+
+        for each_msg in data:
+            try:
+                uid = each_msg.split(' ')[2]
+                size = each_msg.split(' ')[4].rstrip(')')
+                if int(size) <= int(self.max_message_size):
+                    safe_message_ids.append(uid)
+            except ValueError as e:
+                logger.warning(
+                    "ValueError: %s working on %s" % (e, each_msg[0])
+                )
+                pass
+        return safe_message_ids
+
+    def get_message(self):
+        message_ids = self._get_all_message_ids()
+
+        if not message_ids:
            return

-        for key in inbox[0].split():
+        # Limit the uids to the small ones if we care about that
+        if self.max_message_size:
+            message_ids = self._get_small_message_ids(message_ids)
+
+        if self.archive:
+            typ, folders = self.server.list(pattern=self.archive)
+            if folders[0] is None:
+                # If the archive folder does not exist, create it
+                self.server.create(self.archive)
+
+        for uid in message_ids:
            try:
-                typ, msg_contents = self.server.fetch(key, '(RFC822)')
-                message = email.message_from_string(msg_contents[0][1])
+                typ, msg_contents = self.server.uid('fetch', uid, '(RFC822)')
+                message = self.get_email_from_bytes(msg_contents[0][1])
                yield message
-            except email.Errors.MessageParseError:
+            except MessageParseError:
                continue
-            self.server.store(key, "+FLAGS", "\\Deleted")
+
+            if self.archive:
+                self.server.uid('copy', uid, self.archive)
+
+            self.server.uid('store', uid, "+FLAGS", "(\\Deleted)")
        self.server.expunge()
        return
--- a/django_mailbox/transports/pop3.py
+++ b/django_mailbox/transports/pop3.py
@ -1,8 +1,11 @@
-import email
+import six
+
 from poplib import POP3, POP3_SSL

+from .base import EmailTransport, MessageParseError

-class Pop3Transport(object):
+
+class Pop3Transport(EmailTransport):
    def __init__(self, hostname, port=None, ssl=False):
        self.hostname = hostname
        self.port = port
@ -20,14 +23,21 @@ class Pop3Transport(object):
        self.server.user(username)
        self.server.pass_(password)

+    def get_message_body(self, message_lines):
+        if six.PY3:
+            return six.binary_type('\r\n', 'ascii').join(message_lines)
+        return '\r\n'.join(message_lines)
+
    def get_message(self):
        message_count = len(self.server.list()[1])
        for i in range(message_count):
            try:
-                msg_contents = "\r\n".join(self.server.retr(i + 1)[1])
-                message = email.message_from_string(msg_contents)
+                msg_contents = self.get_message_body(
+                    self.server.retr(i + 1)[1]
+                )
+                message = self.get_email_from_bytes(msg_contents)
                yield message
-            except email.Errors.MessageParseError:
+            except MessageParseError:
                continue
            self.server.dele(i + 1)
        self.server.quit()
--- a/django_mailbox/utils.py
+++ b/django_mailbox/utils.py
@ -1,6 +1,8 @@
 import email.header
 import logging

+import six
+
 from django.conf import settings


@ -10,16 +12,24 @@ logger = logging.getLogger(__name__)
 DEFAULT_CHARSET = getattr(
    settings,
    'DJANGO_MAILBOX_DEFAULT_CHARSET',
-    'ascii',
+    'iso8859-1',
 )


-def decode_header(header):
+def convert_header_to_unicode(header):
+    def _decode(value, encoding):
+        if isinstance(value, six.text_type):
+            return value
+        if not encoding or encoding == 'unknown-8bit':
+            encoding = DEFAULT_CHARSET
+        return value.decode(encoding, 'REPLACE')
+
    try:
        return ''.join(
            [
-                unicode(t[0], t[1] or DEFAULT_CHARSET)
-                for t in email.header.decode_header(header)
+                (
+                    _decode(bytestr, encoding)
+                ) for bytestr, encoding in email.header.decode_header(header)
            ]
        )
    except UnicodeDecodeError:
@ -29,3 +39,42 @@ def decode_header(header):
            DEFAULT_CHARSET,
        )
        return unicode(header, DEFAULT_CHARSET, 'replace')
+
+
+def get_body_from_message(message, maintype, subtype):
+    """
+    Fetchs the body message matching main/sub content type.
+    """
+    body = six.text_type('')
+    for part in message.walk():
+        if part.get_content_maintype() == maintype and \
+                part.get_content_subtype() == subtype:
+            charset = part.get_content_charset()
+            this_part = part.get_payload(decode=True)
+            if charset:
+                try:
+                    this_part = this_part.decode(charset, 'replace')
+                except LookupError:
+                    this_part = this_part.decode('ascii', 'replace')
+                    logger.warning(
+                        'Unknown encoding %s encountered while decoding '
+                        'text payload.  Interpreting as ASCII with '
+                        'replacement, but some data may not be '
+                        'represented as the sender intended.',
+                        charset
+                    )
+                except ValueError:
+                    this_part = this_part.decode('ascii', 'replace')
+                    logger.warning(
+                        'Error encountered while decoding text '
+                        'payload from an incorrectly-constructed '
+                        'e-mail; payload was converted to ASCII with '
+                        'replacement, but some data may not be '
+                        'represented as the sender intended.'
+                    )
+            else:
+                this_part = this_part.decode('ascii', 'replace')
+
+            body += this_part
+
+    return body
--- a/docs/conf.py
+++ b/docs/conf.py
@ -41,16 +41,16 @@ master_doc = 'index'

 # General information about the project.
 project = u'django-mailbox'
-copyright = u'2013, Adam Coddington'
+copyright = u'2014, Adam Coddington'

 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '3.0'
+version = '3.3'
 # The full version, including alpha/beta/rc tags.
-release = '3.0'
+release = '3.3'

 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
--- a/docs/index.rst
+++ b/docs/index.rst
@ -6,8 +6,8 @@
 Django-mailbox
 ==============

-.. image:: https://travis-ci.org/latestrevision/django-mailbox.png?branch=master
-   :target: https://travis-ci.org/latestrevision/django-mailbox
+.. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master
+   :target: https://travis-ci.org/coddingtonbear/django-mailbox

 How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, 
 or had to otherwise construct an application driven by e-mail?
@ -20,9 +20,12 @@ Contents:

 .. toctree::
   :maxdepth: 3
-   :glob:

-   topics/*
+   topics/installation
+   topics/mailbox_types
+   topics/polling
+   topics/signal
+   topics/appendix


 Indices and tables
--- a/docs/topics/appendix.rst
+++ b/docs/topics/appendix.rst
@ -0,0 +1,8 @@
+Appendix
+========
+
+.. toctree::
+   :maxdepth: 3
+   :glob:
+
+   appendix/*
--- a/docs/topics/appendix/message-storage.rst
+++ b/docs/topics/appendix/message-storage.rst
--- a/docs/topics/appendix/settings.rst
+++ b/docs/topics/appendix/settings.rst
@ -58,3 +58,11 @@ Settings
    a message instance's ``get_email_object`` method being called.  Value of
    this field is the primary key of the ``django_mailbox.MessageAttachment``
    instance currently storing this payload component's contents.
+
+* ``DJANGO_MAILBOX_MAX_MESSAGE_SIZE``
+
+  * Default: ``False``
+  * Type: ``integer``
+  * If this is set, it will be read as a number of
+    bytes.  Any messages above that size will not be
+    downloaded.  ``2000000`` is 2 Megabytes.
--- a/docs/topics/installation.rst
+++ b/docs/topics/installation.rst
@ -2,16 +2,36 @@
 Installation
 ============

-You can either install from pip::
+1. You can either install from pip::

       pip install django-mailbox
   
-*or* checkout and install the source from the `github repository <https://github.com/latestrevision/django-mailbox/>`_::
+   *or* checkout and install the source from the `github repository <https://github.com/coddingtonbear/django-mailbox/>`_::
   
-    git clone https://github.com/latestrevision/django-mailbox.git
+       git clone https://github.com/coddingtonbear/django-mailbox.git
       cd django-mailbox
       python setup.py install

-After you have installed the package, 
-you should add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in your project's ``settings.py`` file.
+2. After you have installed the package, 
+   add ``django_mailbox`` to the ``INSTALLED_APPS`` setting in
+   your project's ``settings.py`` file.
+
+3. From your project folder, run ``python manage.py migrate django_mailbox`` to
+   create the required database tables.
+
+4. Head to your project's Django Admin and create a mailbox to consume.
+
+
+.. note::
+
+   Once you have entered a mailbox to consume, you can easily verify that you
+   have properly configured your mailbox by either:
+
+   * From the Django Admin, using the 'Get New Mail' action from the action
+     dropdown on the Mailbox changelist 
+     (http://yourproject.com/admin/django_mailbox/mailbox/).
+   * *Or* from a shell opened to your project's directory, using the
+     ``getmail`` management command by running::
+
+       python manage.py getmail

--- a/docs/topics/mailbox_types.rst
+++ b/docs/topics/mailbox_types.rst
@ -7,22 +7,25 @@ POP3 and IMAP as well as local file-based mailboxes.

 .. table:: 'Protocol' Options

-  ============ ============== ===============================================
+  ============ ================ =================================================================================================================================================================
  Mailbox Type 'Protocol'://    Notes
-  ============ ============== ===============================================
+  ============ ================ =================================================================================================================================================================
  POP3         ``pop3://``      Can also specify SSL with ``pop3+ssl://``
-  IMAP         ``imap://``    Can also specify SSL with ``imap+ssl://``
+  IMAP         ``imap://``      Can also specify SSL with ``imap+ssl://``, or specify a folder to save processed messages into by appending ``?archive=my_archive_folder`` to the end of the URI.
+  Gmail IMAP   ``gmail+ssl://`` Uses OAuth authentication for  Gmail's IMAP transport.  See :ref:`gmail-oauth` for details.
  Maildir      ``maildir://``
  Mbox         ``mbox://``
  Babyl        ``babyl://``
  MH           ``mh://``
  MMDF         ``mmdf://``
  Piped Mail   *empty*          See :ref:`receiving-mail-from-exim4-or-postfix`
-  ============ ============== ===============================================
+  ============ ================ =================================================================================================================================================================
+

 .. warning::

-   This will delete any messages it can find in the inbox you specify; 
+   Unless you are using IMAP's 'Archive' feature,
+   this will delete any messages it can find in the inbox you specify;
   do not use an e-mail inbox that you would like to share between
   applications.

@ -39,16 +42,44 @@ Basic IMAP Example: ``imap://username:password@server``
 Basic POP3 Example: ``pop3://username:password@server``

 Most mailboxes these days are SSL-enabled; 
-if yours is, add ``+ssl`` to your URI.  
+if yours is, add ``+ssl`` to the protocol section of your URI.  
 Also, if your username or password include any non-ascii characters,
 they should be URL-encoded  (for example, if your username includes an
 ``@``, it should be changed to ``%40`` in your URI).

-If you have an account named ``youremailaddress@gmail.com`` with a password
-of ``1234`` on GMail, which uses a POP3 server of 'pop.gmail.com' and requires
-SSL, you would enter the following as your URI::
+If you are using an IMAP Mailbox, you can archive all messages before they
+are deleted from the inbox. To archive emails, add the archive folder
+name as a query paremeter to the uri.  For example, if your mailbox has a
+folder named ``myarchivefolder`` that you would like to copy messages to
+after processing, add ``?archive=myarchivefolder`` to the end of the URI.

-    pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com
+For a verbose example, if you have an account named
+``youremailaddress@gmail.com`` with a password
+of ``1234`` on GMail, which uses a IMAP server of ``imap.gmail.com`` (requiring
+SSL) and you would like to archive all processed emails
+into a folder named ``Archived``, you
+would enter the following as your URI::
+
+    imap+ssl://youremailaddress%40gmail.com:1234@imap.gmail.com?archive=Archived
+
+.. _gmail-oauth:
+
+Gmail IMAP with Oauth2 authentication
+-------------------------------------
+
+For added security, Gmail supports using OAuth2 for authentication_.
+To handle the handshake and storing the credentials, use python-social-auth_.
+
+.. _authentication: https://developers.google.com/gmail/xoauth2_protocol
+.. _python-social-auth: http://psa.matiasaguirre.net/
+
+The Gmail Mailbox is also a regular IMAP mailbox,
+but the password you specify will be ignored if OAuth2 authentication succeeds.
+It will fall back to use your specified password as needed.
+
+Build your URI accordingly::
+
+    gmail+ssl://youremailaddress%40gmail.com:oauth2@imap.gmail.com?archive=Archived


 Local File-based Mailboxes
--- a/docs/topics/polling.rst
+++ b/docs/topics/polling.rst
@ -18,8 +18,10 @@ this method will gather new messages from the server.
 Using the Django Admin
 ----------------------

-Check the box next to each of the mailboxes you'd like to fetch e-mail from, 
-and select the 'Get new mail' option.
+From the 'Mailboxes' page in the Django Admin,
+check the box next to each of the mailboxes you'd like to fetch e-mail from, 
+select 'Get new mail' from the action selector at the top of the list
+of mailboxes, then click 'Go'.

 Using a cron job
 ----------------
@ -58,8 +60,8 @@ start by adding a new router configuration to your Exim4 configuration like::
  django_mailbox:
    debug_print = 'R: django_mailbox for $localpart@$domain'
    driver = accept
-    domains = +local_domains
    transport = send_to_django_mailbox
+    domains = mydomain.com
    local_parts = emailusernameone : emailusernametwo

 Make sure that the e-mail addresses you would like handled by Django Mailbox
@ -72,6 +74,14 @@ For example, if one of the e-mail addresses targeted at this machine is
 ``jane@example.com``, 
 the contents of ``local_parts`` would be, simply ``jane``.

+.. note::
+
+   If you would like messages addressed to *any* account *@mydomain.com*
+   to be delivered to django_mailbox, simply omit the above ``local_parts``
+   setting.  In the same vein, if you would like messages addressed to
+   any domain or any local domains, you can omit the ``domains`` setting
+   or set it to ``+local_domains`` respectively.
+
 Next, a new transport configuration to your Exim4 configuration::

  send_to_django_mailbox:
--- a/readme.rst
+++ b/readme.rst
@ -1,12 +1,20 @@
 .. image:: https://travis-ci.org/coddingtonbear/django-mailbox.png?branch=master
   :target: https://travis-ci.org/coddingtonbear/django-mailbox

-How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, 
-or had to otherwise construct an application driven by e-mail?
-One too many times, I'm sure.
+.. image:: https://badge.fury.io/py/django-mailbox.png
+    :target: http://badge.fury.io/py/django-mailbox

-This small Django application will allow you to specify mailboxes that you would like consumed for incoming content; 
-the e-mail will be stored, and you can process it at will (or, if you're in a hurry, by subscribing to a signal).
+.. image:: https://pypip.in/d/django-mailbox/badge.png
+    :target: https://pypi.python.org/pypi/django-mailbox
+
+
+Easily ingest messages from POP3, IMAP, or local mailboxes into your Django application.
+
+This app allows you to either ingest e-mail content from common e-mail services (as long as the service provides POP3 or IMAP support),
+or directly recieve e-mail messages from ``stdin`` (for locally processing messages from Postfix or Exim4).
+
+These ingested messages will be stored in the database in Django models and you can process their content at will,
+or -- if you're in a hurry -- by using a signal receiver.

 - Documentation for django-mailbox is available on
  `ReadTheDocs <http://django-mailbox.readthedocs.org/>`_.
@ -14,10 +22,3 @@ the e-mail will be stored, and you can process it at will (or, if you're in a hu
  `Github <http://github.com/coddingtonbear/django-mailbox/issues>`_.
 - Test status available on
  `Travis-CI <https://travis-ci.org/coddingtonbear/django-mailbox>`_.
-
-
-
-.. image:: https://d2weczhvl823v0.cloudfront.net/coddingtonbear/django-mailbox/trend.png
-   :alt: Bitdeli badge
-   :target: https://bitdeli.com/free
-
--- a/setup.py
+++ b/setup.py
@ -1,13 +1,20 @@
-from setuptools import setup
+from setuptools import find_packages, setup
+
+from django_mailbox import __version__ as version_string

 tests_require = [
    'django',
+    'mock',
+]
+
+gmail_oauth2_require = [
+    'python-social-auth',
 ]

 setup(
    name='django-mailbox',
-    version='3.0.3',
-    url='http://github.com/latestrevision/django-mailbox/',
+    version=version_string,
+    url='http://github.com/coddingtonbear/django-mailbox/',
    description=(
        'Import mail from POP3, IMAP, local mailboxes or directly from '
        'Postfix or Exim4 into your Django application automatically.'
@ -15,7 +22,10 @@ setup(
    author='Adam Coddington',
    author_email='me@adamcoddington.net',
    tests_require=tests_require,
-    extras_require={'test': tests_require},
+    extras_require={
+        'test': tests_require,
+        'gmail-oauth2': gmail_oauth2_require
+    },
    test_suite='django_mailbox.runtests.runtests',
    classifiers=[
        'Framework :: Django',
@ -32,15 +42,8 @@ setup(
        'Topic :: Communications :: Email :: Post-Office :: POP3',
        'Topic :: Communications :: Email :: Email Clients (MUA)',
    ],
-    packages=[
-        'django_mailbox',
-        'django_mailbox.management',
-        'django_mailbox.management.commands',
-        'django_mailbox.migrations',
-        'django_mailbox.transports',
-        'django_mailbox.tests',
-    ],
+    packages=find_packages(),
    install_requires=[
-        'six'
+        'six>=1.6.1'
    ]
 )