federico/django-mailbox
1
0
Fork 0
forked from mirror/django-mailbox
Code Activity

Base64-encode message body when storing in DB.

Browse source 
* Since Django requires that the content stored in a TextField be unicode, we
  need some way to encode the message safely.  In situations before where
  a single-byte non-7-bit-clean encoding were used, django-mailbox could
  explode when writing the message copy to the database; this should ameliorate
  that problem.
* Reverts earlier changesets' encoding normalization.
This commit is contained in:
Adam Coddington 2013-07-26 22:13:29 -07:00
parent 4c3af900c6
commit 88d122c421
5 changed files with 126 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

55
django_mailbox/migrations/0016_auto__add_field_message_encoded.py Normal file
View file

@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'Message.encoded'
db.add_column(u'django_mailbox_message', 'encoded',
self.gf('django.db.models.fields.BooleanField')(default=False),
keep_default=False)
def backwards(self, orm):
# Deleting field 'Message.encoded'
db.delete_column(u'django_mailbox_message', 'encoded')
models = {
u'django_mailbox.mailbox': {
'Meta': {'object_name': 'Mailbox'},
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'from_email': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'uri': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'})
},
u'django_mailbox.message': {
'Meta': {'object_name': 'Message'},
'body': ('django.db.models.fields.TextField', [], {}),
'encoded': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'from_header': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'in_reply_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'replies'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}),
'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'messages'", 'to': u"orm['django_mailbox.Mailbox']"}),
'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'outgoing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'processed': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'read': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'to_header': ('django.db.models.fields.TextField', [], {})
},
u'django_mailbox.messageattachment': {
'Meta': {'object_name': 'MessageAttachment'},
'document': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
'headers': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'message': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'attachments'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"})
}
}
complete_apps = ['django_mailbox']

111
django_mailbox/models.py
View file

@ -1,8 +1,10 @@
import base64
import email
from email.header import decode_header
from email.message import Message as EmailMessage
from email.utils import formatdate, parseaddr
from email.encoders import encode_base64
import logging
import urllib
import mimetypes
import os.path
@ -17,7 +19,7 @@ except ImportError:
from django.conf import settings
from django.core.mail.message import make_msgid
from django.core.files.base import File, ContentFile
from django.core.files.base import ContentFile
from django.db import models
from django_mailbox.transports import Pop3Transport, ImapTransport,\
MaildirTransport, MboxTransport, BabylTransport, MHTransport, \
@ -26,6 +28,9 @@ from django_mailbox.signals import message_received
import six
logger = logging.getLogger(__name__)
STRIP_UNALLOWED_MIMETYPES = getattr(
settings,
'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES',
@ -57,11 +62,6 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr(
'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER',
'X-Django-Mailbox-Interpolate-Attachment'
)
ORIGINAL_CHARSET_HEADER = getattr(
settings,
'DJANGO_MAILBOX_ORIGINAL_CHARSET_HEADER',
'X-Django-Mailbox-Original-Charset'
)
class ActiveMailboxManager(models.Manager):
@ -230,10 +230,14 @@ class Mailbox(models.Model):
extension = '.bin'
attachment = MessageAttachment()
attachment.document.save(
uuid.uuid4().hex + extension,
ContentFile(six.BytesIO(msg.get_payload(decode=True)).getvalue())
ContentFile(
six.BytesIO(
msg.get_payload(decode=True)
).getvalue()
)
)
attachment.message = record
for key, value in msg.items():
@ -244,26 +248,7 @@ class Mailbox(models.Model):
placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk)
new = placeholder
else:
payload = msg.get_payload()
for header, value in msg.items():
new[header] = value
charset = msg.get_content_charset()
if charset:
try:
payload = payload.decode(charset, 'replace')
new[ORIGINAL_CHARSET_HEADER] = charset
new.set_charset('utf-8')
except LookupError:
payload = payload.decode('ascii', 'replace')
new.set_charset('utf-8')
new[ALTERED_MESSAGE_HEADER] = (
'Message declared unknown charset \'%s\'' % (
charset
)
)
# Remove any characters that are invalid in the
# encoding that this part is supposed to be in.
new.set_payload(payload.encode('utf-8'))
new = msg
return new
def _process_message(self, message):
@ -279,7 +264,7 @@ class Mailbox(models.Model):
msg.to_header = message['to']
msg.save()
message = self._get_dehydrated_message(message, msg)
msg.body = message.as_string().decode('utf-8')
msg.set_body(message.as_string())
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
@ -348,6 +333,10 @@ class Message(models.Model):
)
body = models.TextField()
encoded = models.BooleanField(
default=False,
help_text='True if the e-mail body is Base64 encoded'
)
processed = models.DateTimeField(
auto_now_add=True
@ -375,7 +364,7 @@ class Message(models.Model):
"""
addresses = []
addresses = self.to_addresses + self.from_address
addresses = self.to_addresses + self.from_address
return addresses
@property
@ -434,13 +423,9 @@ class Message(models.Model):
):
charset = part.get_content_charset()
this_part = part.get_payload(decode=True)
if isinstance(this_part, six.text_type):
# For some unknown reason, get_payload() sometimes
# returns a unicode object -- it should always be bytes
this_part = this_part.encode('unicode-escape')
this_part = this_part.decode('string-escape')
if charset:
this_part = this_part.decode(charset, 'replace')
body += this_part
return body
@ -496,48 +481,32 @@ class Message(models.Model):
)
new.set_payload('')
else:
payload = msg.get_payload().decode('utf-8')
charset = msg.get_content_charset()
if not charset:
charset = 'utf-8'
for header, value in msg.items():
if header == ORIGINAL_CHARSET_HEADER:
# If we process ORIGINAL_CHARSET_HEADER before other
# headers, due to some idiosyncrasies of python's email
# module, we'll end up with duplicated headers --
# set_charset sets a handful of headers if they do not
# already exist. We'll store this value and re-encode
# the payload afterward.
charset = value
# We do not want to preserve this header.
continue
new[header] = value
payload = payload.encode(charset, 'replace')
new.set_charset(charset)
new.set_payload(payload)
new.set_payload(
msg.get_payload()
)
return new
def get_email_object(self):
""" Returns an `email.message.Message` instance for this message.
def get_body(self):
if self.encoded:
return base64.b64decode(self.body.encode('ascii'))
return self.body.encode('utf-8')
Note: Python 2's version of email.message.fromstring on some platforms
requires bytes and will incorrectly create an e-mail object if a
unicode object is passed-in.
Note that in reality 7-bit ascii *should* be an acceptable encoding
here per RFC-2822 (2.1), but given that `message_from_string` really
only cares that bytes are incoming rather than a unicode object, and
django will dutifully, if you were to edit the message in code or
via the admin, allow you to store unicode characters, returning UTF-8
encoded bytes is probably the safest answer.
"""
body = self.body
if sys.version_info < (3, 0):
def set_body(self, body):
if sys.version_info >= (3, 0):
body = body.encode('utf-8')
return self._rehydrate(
email.message_from_string(body)
)
self.encoded = True
self.body = base64.b64encode(body).decode('ascii')
def get_email_object(self):
""" Returns an `email.message.Message` instance for this message."""
body = self.get_body()
if sys.version_info < (3, 0):
flat = email.message_from_string(body)
else:
flat = email.message_from_bytes(body)
return self._rehydrate(flat)
def delete(self, *args, **kwargs):
for attachment in self.attachments.all():

10
django_mailbox/tests/base.py
View file

@ -2,6 +2,7 @@ import email
import os.path
from django.test import TestCase
import sys
from django_mailbox import models
from django_mailbox.models import Mailbox, Message
@ -28,11 +29,12 @@ class EmailMessageTestCase(TestCase):
'messages',
name,
),
'r'
'rb'
) as f:
return email.message_from_string(
f.read()
)
if sys.version_info < (3, 0):
return email.message_from_string(f.read())
else:
return email.message_from_bytes(f.read())
def _headers_identical(self, left, right, header=None):
""" Check if headers are (close enough to) identical.

4
django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml → django_mailbox/tests/messages/message_with_single_byte_encoding.eml
View file

@ -3,7 +3,7 @@ Subject: Tjest
From: "Somebody" <somebody@somewhere.com>
To: idontknow@somewhere.com
MIME-Version: 1.0
Content-Type: text/plain; charset="raspberries"
Content-Type: text/plain; charset="cp1251"
Content-Transfer-Encoding: 7bit
Это сообщение имеет неверная кодировка.
Это сообщение имеет неправильную кодировка.

61
django_mailbox/tests/test_process_email.py
View file

@ -132,48 +132,16 @@ class TestProcessEmail(EmailMessageTestCase):
actual_text,
)
def test_message_with_unknown_charset(self):
""" Ensure that we gracefully fail at handling unknown encodings.
Should an unknown encoding be used, we should:
- Add a X-Django-Mailbox-Altered-Message header
- Use UTF-8 (with replacement) as the de-facto (and probably incorrect)
encoding for this payload part.
"""
email_object = self._get_email_object(
'message_with_invalid_payload_encoding.eml'
)
msg = self.mailbox.process_incoming_message(email_object)
actual_body = msg.get_text_body()
expected_body = six.u(
'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd '
'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd'
'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd.'
)
self.assertEqual(
expected_body,
actual_body,
)
def test_message_with_invalid_content_for_declared_encoding(self):
""" Ensure that we gracefully handle mis-encoded bodies.
Should a payload body be misencoded, we should:
- Decode the message (with replacement) using the declared encoding.
- Always return a payload body in the declared encoding, using python's
usual replacement mechanisms.
- Not explode
Note: there is (intentionally) no assertion below; the only guarantee
we make via this library is that processing this e-mail message will
not cause an exception to be raised.
"""
email_object = self._get_email_object(
@ -182,8 +150,25 @@ class TestProcessEmail(EmailMessageTestCase):
msg = self.mailbox.process_incoming_message(email_object)
msg.get_text_body()
def test_message_with_valid_content_in_single_byte_encoding(self):
email_object = self._get_email_object(
'message_with_single_byte_encoding.eml',
)
msg = self.mailbox.process_incoming_message(email_object)
actual_body = msg.get_text_body()
expected_body = '??? ????????? ????? ???????????? ?????????.'
expected_body = six.u(
'\u042d\u0442\u043e '
'\u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0435 '
'\u0438\u043c\u0435\u0435\u0442 '
'\u043d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d'
'\u0443\u044e '
'\u043a\u043e\u0434\u0438\u0440\u043e\u0432\u043a\u0430.'
)
self.assertEqual(
actual_body,