Base64-encode message body when storing in DB.

* Since Django requires that the content stored in a TextField be unicode, we
  need some way to encode the message safely.  In situations before where
  a single-byte non-7-bit-clean encoding were used, django-mailbox could
  explode when writing the message copy to the database; this should ameliorate
  that problem.
* Reverts earlier changesets' encoding normalization.

django_mailbox/migrations/0016_auto__add_field_message_encoded.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'Message.encoded'
+        db.add_column(u'django_mailbox_message', 'encoded',
+                      self.gf('django.db.models.fields.BooleanField')(default=False),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'Message.encoded'
+        db.delete_column(u'django_mailbox_message', 'encoded')
+
+
+    models = {
+        u'django_mailbox.mailbox': {
+            'Meta': {'object_name': 'Mailbox'},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'from_email': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'uri': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'})
+        },
+        u'django_mailbox.message': {
+            'Meta': {'object_name': 'Message'},
+            'body': ('django.db.models.fields.TextField', [], {}),
+            'encoded': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'from_header': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'in_reply_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'replies'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}),
+            'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'messages'", 'to': u"orm['django_mailbox.Mailbox']"}),
+            'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'outgoing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'processed': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'read': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
+            'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'to_header': ('django.db.models.fields.TextField', [], {})
+        },
+        u'django_mailbox.messageattachment': {
+            'Meta': {'object_name': 'MessageAttachment'},
+            'document': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+            'headers': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'message': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'attachments'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"})
+        }
+    }
+
+    complete_apps = ['django_mailbox']

django_mailbox/models.py

@@ -1,8 +1,10 @@
+import base64
 import email
 from email.header import decode_header
 from email.message import Message as EmailMessage
 from email.utils import formatdate, parseaddr
 from email.encoders import encode_base64
+import logging
 import urllib
 import mimetypes
 import os.path
@@ -17,7 +19,7 @@ except ImportError:
 
 from django.conf import settings
 from django.core.mail.message import make_msgid
-from django.core.files.base import File, ContentFile
+from django.core.files.base import ContentFile
 from django.db import models
 from django_mailbox.transports import Pop3Transport, ImapTransport,\
     MaildirTransport, MboxTransport, BabylTransport, MHTransport, \
@@ -26,6 +28,9 @@ from django_mailbox.signals import message_received
 import six
 
 
+logger = logging.getLogger(__name__)
+
+
 STRIP_UNALLOWED_MIMETYPES = getattr(
     settings,
     'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES',
@@ -57,11 +62,6 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr(
     'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER',
     'X-Django-Mailbox-Interpolate-Attachment'
 )
-ORIGINAL_CHARSET_HEADER = getattr(
-    settings,
-    'DJANGO_MAILBOX_ORIGINAL_CHARSET_HEADER',
-    'X-Django-Mailbox-Original-Charset'
-)
 
 
 class ActiveMailboxManager(models.Manager):
@@ -230,10 +230,14 @@ class Mailbox(models.Model):
                 extension = '.bin'
 
             attachment = MessageAttachment()
-            
+
             attachment.document.save(
                 uuid.uuid4().hex + extension,
-                ContentFile(six.BytesIO(msg.get_payload(decode=True)).getvalue())
+                ContentFile(
+                    six.BytesIO(
+                        msg.get_payload(decode=True)
+                    ).getvalue()
+                )
             )
             attachment.message = record
             for key, value in msg.items():
@@ -244,26 +248,7 @@ class Mailbox(models.Model):
             placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk)
             new = placeholder
         else:
-            payload = msg.get_payload()
+            new = msg
-            for header, value in msg.items():
-                new[header] = value
-            charset = msg.get_content_charset()
-            if charset:
-                try:
-                    payload = payload.decode(charset, 'replace')
-                    new[ORIGINAL_CHARSET_HEADER] = charset
-                    new.set_charset('utf-8')
-                except LookupError:
-                    payload = payload.decode('ascii', 'replace')
-                    new.set_charset('utf-8')
-                    new[ALTERED_MESSAGE_HEADER] = (
-                        'Message declared unknown charset \'%s\'' % (
-                            charset
-                        )
-                    )
-            # Remove any characters that are invalid in the
-            # encoding that this part is supposed to be in.
-            new.set_payload(payload.encode('utf-8'))
         return new
 
     def _process_message(self, message):
@@ -279,7 +264,7 @@ class Mailbox(models.Model):
             msg.to_header = message['to']
         msg.save()
         message = self._get_dehydrated_message(message, msg)
-        msg.body = message.as_string().decode('utf-8')
+        msg.set_body(message.as_string())
         if message['in-reply-to']:
             try:
                 msg.in_reply_to = Message.objects.filter(
@@ -348,6 +333,10 @@ class Message(models.Model):
     )
 
     body = models.TextField()
+    encoded = models.BooleanField(
+        default=False,
+        help_text='True if the e-mail body is Base64 encoded'
+    )
 
     processed = models.DateTimeField(
         auto_now_add=True
@@ -375,7 +364,7 @@ class Message(models.Model):
 
         """
         addresses = []
-        addresses = self.to_addresses +  self.from_address
+        addresses = self.to_addresses + self.from_address
         return addresses
 
     @property
@@ -434,13 +423,9 @@ class Message(models.Model):
                 ):
                     charset = part.get_content_charset()
                     this_part = part.get_payload(decode=True)
-                    if isinstance(this_part, six.text_type):
-                        # For some unknown reason, get_payload() sometimes
-                        # returns a unicode object -- it should always be bytes
-                        this_part = this_part.encode('unicode-escape')
-                        this_part = this_part.decode('string-escape')
                     if charset:
                         this_part = this_part.decode(charset, 'replace')
+
                     body += this_part
             return body
 
@@ -496,48 +481,32 @@ class Message(models.Model):
                 )
                 new.set_payload('')
         else:
-            payload = msg.get_payload().decode('utf-8')
-            charset = msg.get_content_charset()
-            if not charset:
-                charset = 'utf-8'
             for header, value in msg.items():
-                if header == ORIGINAL_CHARSET_HEADER:
-                    # If we process ORIGINAL_CHARSET_HEADER before other
-                    # headers, due to some idiosyncrasies of python's email
-                    # module, we'll end up with duplicated headers --
-                    # set_charset sets a handful of headers if they do not
-                    # already exist. We'll store this value and re-encode
-                    # the payload afterward.
-                    charset = value
-                    # We do not want to preserve this header.
-                    continue
                 new[header] = value
-            payload = payload.encode(charset, 'replace')
+            new.set_payload(
-            new.set_charset(charset)
+                msg.get_payload()
-            new.set_payload(payload)
+            )
         return new
 
-    def get_email_object(self):
+    def get_body(self):
-        """ Returns an `email.message.Message` instance for this message.
+        if self.encoded:
+            return base64.b64decode(self.body.encode('ascii'))
+        return self.body.encode('utf-8')
 
-        Note: Python 2's version of email.message.fromstring on some platforms
+    def set_body(self, body):
-        requires bytes and will incorrectly create an e-mail object if a
+        if sys.version_info >= (3, 0):
-        unicode object is passed-in.
-
-        Note that in reality 7-bit ascii *should* be an acceptable encoding
-        here per RFC-2822 (2.1), but given that `message_from_string` really
-        only cares that bytes are incoming rather than a unicode object, and
-        django will dutifully, if you were to edit the message in code or
-        via the admin, allow you to store unicode characters, returning UTF-8
-        encoded bytes is probably the safest answer.
-
-        """
-        body = self.body
-        if sys.version_info < (3, 0):
             body = body.encode('utf-8')
-        return self._rehydrate(
+        self.encoded = True
-            email.message_from_string(body)
+        self.body = base64.b64encode(body).decode('ascii')
-        )
+
+    def get_email_object(self):
+        """ Returns an `email.message.Message` instance for this message."""
+        body = self.get_body()
+        if sys.version_info < (3, 0):
+            flat = email.message_from_string(body)
+        else:
+            flat = email.message_from_bytes(body)
+        return self._rehydrate(flat)
 
     def delete(self, *args, **kwargs):
         for attachment in self.attachments.all():

django_mailbox/tests/base.py

@@ -2,6 +2,7 @@ import email
 import os.path
 
 from django.test import TestCase
+import sys
 
 from django_mailbox import models
 from django_mailbox.models import Mailbox, Message
@@ -28,11 +29,12 @@ class EmailMessageTestCase(TestCase):
                 'messages',
                 name,
             ),
-            'r'
+            'rb'
         ) as f:
-            return email.message_from_string(
+            if sys.version_info < (3, 0):
-                f.read()
+                return email.message_from_string(f.read())
-            )
+            else:
+                return email.message_from_bytes(f.read())
 
     def _headers_identical(self, left, right, header=None):
         """ Check if headers are (close enough to) identical.

django_mailbox/tests/messages/message_with_single_byte_encoding.eml

@@ -3,7 +3,7 @@ Subject: Tjest
 From: "Somebody" <somebody@somewhere.com>
 To: idontknow@somewhere.com
 MIME-Version: 1.0
-Content-Type: text/plain; charset="raspberries"
+Content-Type: text/plain; charset="cp1251"
 Content-Transfer-Encoding: 7bit
 
-Это сообщение имеет неверная кодировка.
+Это сообщение имеет неправильную кодировка.

django_mailbox/tests/test_process_email.py

@@ -132,48 +132,16 @@ class TestProcessEmail(EmailMessageTestCase):
             actual_text,
         )
 
-    def test_message_with_unknown_charset(self):
-        """ Ensure that we gracefully fail at handling unknown encodings.
-
-        Should an unknown encoding be used, we should:
-
-        - Add a X-Django-Mailbox-Altered-Message header
-        - Use UTF-8 (with replacement) as the de-facto (and probably incorrect)
-          encoding for this payload part.
-
-        """
-        email_object = self._get_email_object(
-            'message_with_invalid_payload_encoding.eml'
-        )
-
-        msg = self.mailbox.process_incoming_message(email_object)
-
-        actual_body = msg.get_text_body()
-
-        expected_body = six.u(
-            '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
-            '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
-            '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
-            '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
-            '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
-            '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
-            '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
-            '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd.'
-        )
-
-        self.assertEqual(
-            expected_body,
-            actual_body,
-        )
-
     def test_message_with_invalid_content_for_declared_encoding(self):
         """ Ensure that we gracefully handle mis-encoded bodies.
 
         Should a payload body be misencoded, we should:
 
-        - Decode the message (with replacement) using the declared encoding.
+        - Not explode
-        - Always return a payload body in the declared encoding, using python's
+
-          usual replacement mechanisms.
+        Note: there is (intentionally) no assertion below; the only guarantee
+        we make via this library is that processing this e-mail message will
+        not cause an exception to be raised.
 
         """
         email_object = self._get_email_object(
@@ -182,8 +150,25 @@ class TestProcessEmail(EmailMessageTestCase):
 
         msg = self.mailbox.process_incoming_message(email_object)
 
+        msg.get_text_body()
+
+    def test_message_with_valid_content_in_single_byte_encoding(self):
+        email_object = self._get_email_object(
+            'message_with_single_byte_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(email_object)
+
         actual_body = msg.get_text_body()
-        expected_body = '??? ????????? ????? ???????????? ?????????.'
+
+        expected_body = six.u(
+            '\u042d\u0442\u043e '
+            '\u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0435 '
+            '\u0438\u043c\u0435\u0435\u0442 '
+            '\u043d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d'
+            '\u0443\u044e '
+            '\u043a\u043e\u0434\u0438\u0440\u043e\u0432\u043a\u0430.'
+        )
 
         self.assertEqual(
             actual_body,