From 88d122c421b181d246157da00788ada4cd9969e9 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Fri, 26 Jul 2013 22:13:29 -0700 Subject: [PATCH] Base64-encode message body when storing in DB. * Since Django requires that the content stored in a TextField be unicode, we need some way to encode the message safely. In situations before where a single-byte non-7-bit-clean encoding were used, django-mailbox could explode when writing the message copy to the database; this should ameliorate that problem. * Reverts earlier changesets' encoding normalization. --- .../0016_auto__add_field_message_encoded.py | 55 +++++++++ django_mailbox/models.py | 111 +++++++----------- django_mailbox/tests/base.py | 10 +- ... => message_with_single_byte_encoding.eml} | 4 +- django_mailbox/tests/test_process_email.py | 61 ++++------ 5 files changed, 126 insertions(+), 115 deletions(-) create mode 100644 django_mailbox/migrations/0016_auto__add_field_message_encoded.py rename django_mailbox/tests/messages/{message_with_invalid_payload_encoding.eml => message_with_single_byte_encoding.eml} (58%) diff --git a/django_mailbox/migrations/0016_auto__add_field_message_encoded.py b/django_mailbox/migrations/0016_auto__add_field_message_encoded.py new file mode 100644 index 0000000..8fa5e9d --- /dev/null +++ b/django_mailbox/migrations/0016_auto__add_field_message_encoded.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Message.encoded' + db.add_column(u'django_mailbox_message', 'encoded', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Message.encoded' + db.delete_column(u'django_mailbox_message', 'encoded') + + + models = { + u'django_mailbox.mailbox': { + 'Meta': {'object_name': 'Mailbox'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'from_email': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'uri': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}) + }, + u'django_mailbox.message': { + 'Meta': {'object_name': 'Message'}, + 'body': ('django.db.models.fields.TextField', [], {}), + 'encoded': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'from_header': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'in_reply_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'replies'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}), + 'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'messages'", 'to': u"orm['django_mailbox.Mailbox']"}), + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'outgoing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'processed': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'read': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'to_header': ('django.db.models.fields.TextField', [], {}) + }, + u'django_mailbox.messageattachment': { + 'Meta': {'object_name': 'MessageAttachment'}, + 'document': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), + 'headers': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'attachments'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}) + } + } + + complete_apps = ['django_mailbox'] \ No newline at end of file diff --git a/django_mailbox/models.py b/django_mailbox/models.py index fe6e295..e9f1bac 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -1,8 +1,10 @@ +import base64 import email from email.header import decode_header from email.message import Message as EmailMessage from email.utils import formatdate, parseaddr from email.encoders import encode_base64 +import logging import urllib import mimetypes import os.path @@ -17,7 +19,7 @@ except ImportError: from django.conf import settings from django.core.mail.message import make_msgid -from django.core.files.base import File, ContentFile +from django.core.files.base import ContentFile from django.db import models from django_mailbox.transports import Pop3Transport, ImapTransport,\ MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ @@ -26,6 +28,9 @@ from django_mailbox.signals import message_received import six +logger = logging.getLogger(__name__) + + STRIP_UNALLOWED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES', @@ -57,11 +62,6 @@ ATTACHMENT_INTERPOLATION_HEADER = getattr( 'DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER', 'X-Django-Mailbox-Interpolate-Attachment' ) -ORIGINAL_CHARSET_HEADER = getattr( - settings, - 'DJANGO_MAILBOX_ORIGINAL_CHARSET_HEADER', - 'X-Django-Mailbox-Original-Charset' -) class ActiveMailboxManager(models.Manager): @@ -230,10 +230,14 @@ class Mailbox(models.Model): extension = '.bin' attachment = MessageAttachment() - + attachment.document.save( uuid.uuid4().hex + extension, - ContentFile(six.BytesIO(msg.get_payload(decode=True)).getvalue()) + ContentFile( + six.BytesIO( + msg.get_payload(decode=True) + ).getvalue() + ) ) attachment.message = record for key, value in msg.items(): @@ -244,26 +248,7 @@ class Mailbox(models.Model): placeholder[ATTACHMENT_INTERPOLATION_HEADER] = str(attachment.pk) new = placeholder else: - payload = msg.get_payload() - for header, value in msg.items(): - new[header] = value - charset = msg.get_content_charset() - if charset: - try: - payload = payload.decode(charset, 'replace') - new[ORIGINAL_CHARSET_HEADER] = charset - new.set_charset('utf-8') - except LookupError: - payload = payload.decode('ascii', 'replace') - new.set_charset('utf-8') - new[ALTERED_MESSAGE_HEADER] = ( - 'Message declared unknown charset \'%s\'' % ( - charset - ) - ) - # Remove any characters that are invalid in the - # encoding that this part is supposed to be in. - new.set_payload(payload.encode('utf-8')) + new = msg return new def _process_message(self, message): @@ -279,7 +264,7 @@ class Mailbox(models.Model): msg.to_header = message['to'] msg.save() message = self._get_dehydrated_message(message, msg) - msg.body = message.as_string().decode('utf-8') + msg.set_body(message.as_string()) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( @@ -348,6 +333,10 @@ class Message(models.Model): ) body = models.TextField() + encoded = models.BooleanField( + default=False, + help_text='True if the e-mail body is Base64 encoded' + ) processed = models.DateTimeField( auto_now_add=True @@ -375,7 +364,7 @@ class Message(models.Model): """ addresses = [] - addresses = self.to_addresses + self.from_address + addresses = self.to_addresses + self.from_address return addresses @property @@ -434,13 +423,9 @@ class Message(models.Model): ): charset = part.get_content_charset() this_part = part.get_payload(decode=True) - if isinstance(this_part, six.text_type): - # For some unknown reason, get_payload() sometimes - # returns a unicode object -- it should always be bytes - this_part = this_part.encode('unicode-escape') - this_part = this_part.decode('string-escape') if charset: this_part = this_part.decode(charset, 'replace') + body += this_part return body @@ -496,48 +481,32 @@ class Message(models.Model): ) new.set_payload('') else: - payload = msg.get_payload().decode('utf-8') - charset = msg.get_content_charset() - if not charset: - charset = 'utf-8' for header, value in msg.items(): - if header == ORIGINAL_CHARSET_HEADER: - # If we process ORIGINAL_CHARSET_HEADER before other - # headers, due to some idiosyncrasies of python's email - # module, we'll end up with duplicated headers -- - # set_charset sets a handful of headers if they do not - # already exist. We'll store this value and re-encode - # the payload afterward. - charset = value - # We do not want to preserve this header. - continue new[header] = value - payload = payload.encode(charset, 'replace') - new.set_charset(charset) - new.set_payload(payload) + new.set_payload( + msg.get_payload() + ) return new - def get_email_object(self): - """ Returns an `email.message.Message` instance for this message. + def get_body(self): + if self.encoded: + return base64.b64decode(self.body.encode('ascii')) + return self.body.encode('utf-8') - Note: Python 2's version of email.message.fromstring on some platforms - requires bytes and will incorrectly create an e-mail object if a - unicode object is passed-in. - - Note that in reality 7-bit ascii *should* be an acceptable encoding - here per RFC-2822 (2.1), but given that `message_from_string` really - only cares that bytes are incoming rather than a unicode object, and - django will dutifully, if you were to edit the message in code or - via the admin, allow you to store unicode characters, returning UTF-8 - encoded bytes is probably the safest answer. - - """ - body = self.body - if sys.version_info < (3, 0): + def set_body(self, body): + if sys.version_info >= (3, 0): body = body.encode('utf-8') - return self._rehydrate( - email.message_from_string(body) - ) + self.encoded = True + self.body = base64.b64encode(body).decode('ascii') + + def get_email_object(self): + """ Returns an `email.message.Message` instance for this message.""" + body = self.get_body() + if sys.version_info < (3, 0): + flat = email.message_from_string(body) + else: + flat = email.message_from_bytes(body) + return self._rehydrate(flat) def delete(self, *args, **kwargs): for attachment in self.attachments.all(): diff --git a/django_mailbox/tests/base.py b/django_mailbox/tests/base.py index 2c9bb90..b062797 100644 --- a/django_mailbox/tests/base.py +++ b/django_mailbox/tests/base.py @@ -2,6 +2,7 @@ import email import os.path from django.test import TestCase +import sys from django_mailbox import models from django_mailbox.models import Mailbox, Message @@ -28,11 +29,12 @@ class EmailMessageTestCase(TestCase): 'messages', name, ), - 'r' + 'rb' ) as f: - return email.message_from_string( - f.read() - ) + if sys.version_info < (3, 0): + return email.message_from_string(f.read()) + else: + return email.message_from_bytes(f.read()) def _headers_identical(self, left, right, header=None): """ Check if headers are (close enough to) identical. diff --git a/django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml b/django_mailbox/tests/messages/message_with_single_byte_encoding.eml similarity index 58% rename from django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml rename to django_mailbox/tests/messages/message_with_single_byte_encoding.eml index 3cf61cd..209e15e 100644 --- a/django_mailbox/tests/messages/message_with_invalid_payload_encoding.eml +++ b/django_mailbox/tests/messages/message_with_single_byte_encoding.eml @@ -3,7 +3,7 @@ Subject: Tjest From: "Somebody" To: idontknow@somewhere.com MIME-Version: 1.0 -Content-Type: text/plain; charset="raspberries" +Content-Type: text/plain; charset="cp1251" Content-Transfer-Encoding: 7bit -Это сообщение имеет неверная кодировка. + . diff --git a/django_mailbox/tests/test_process_email.py b/django_mailbox/tests/test_process_email.py index 6e5dab7..0dea582 100644 --- a/django_mailbox/tests/test_process_email.py +++ b/django_mailbox/tests/test_process_email.py @@ -132,48 +132,16 @@ class TestProcessEmail(EmailMessageTestCase): actual_text, ) - def test_message_with_unknown_charset(self): - """ Ensure that we gracefully fail at handling unknown encodings. - - Should an unknown encoding be used, we should: - - - Add a X-Django-Mailbox-Altered-Message header - - Use UTF-8 (with replacement) as the de-facto (and probably incorrect) - encoding for this payload part. - - """ - email_object = self._get_email_object( - 'message_with_invalid_payload_encoding.eml' - ) - - msg = self.mailbox.process_incoming_message(email_object) - - actual_body = msg.get_text_body() - - expected_body = six.u( - '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd ' - '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd' - '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd ' - '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd ' - '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd' - '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd ' - '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd' - '\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd.' - ) - - self.assertEqual( - expected_body, - actual_body, - ) - def test_message_with_invalid_content_for_declared_encoding(self): """ Ensure that we gracefully handle mis-encoded bodies. Should a payload body be misencoded, we should: - - Decode the message (with replacement) using the declared encoding. - - Always return a payload body in the declared encoding, using python's - usual replacement mechanisms. + - Not explode + + Note: there is (intentionally) no assertion below; the only guarantee + we make via this library is that processing this e-mail message will + not cause an exception to be raised. """ email_object = self._get_email_object( @@ -182,8 +150,25 @@ class TestProcessEmail(EmailMessageTestCase): msg = self.mailbox.process_incoming_message(email_object) + msg.get_text_body() + + def test_message_with_valid_content_in_single_byte_encoding(self): + email_object = self._get_email_object( + 'message_with_single_byte_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(email_object) + actual_body = msg.get_text_body() - expected_body = '??? ????????? ????? ???????????? ?????????.' + + expected_body = six.u( + '\u042d\u0442\u043e ' + '\u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0435 ' + '\u0438\u043c\u0435\u0435\u0442 ' + '\u043d\u0435\u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d' + '\u0443\u044e ' + '\u043a\u043e\u0434\u0438\u0440\u043e\u0432\u043a\u0430.' + ) self.assertEqual( actual_body,