From 4b70cf6f216cd7c828607dc922f2c34c1efb47a8 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Tue, 11 Jun 2013 22:23:27 -0700 Subject: [PATCH] Fixing bad migration; keeping migration for rebuilding associations as a management command. --- .../commands/rebuildmessageattachments.py | 71 ++++++++++++++ ...to__add_field_messageattachment_message.py | 4 +- .../0014_migrate_message_attachments.py | 8 +- .../0015_rebuild_message_associations.py | 97 ------------------- setup.py | 2 +- 5 files changed, 78 insertions(+), 104 deletions(-) create mode 100644 django_mailbox/management/commands/rebuildmessageattachments.py delete mode 100644 django_mailbox/migrations/0015_rebuild_message_associations.py diff --git a/django_mailbox/management/commands/rebuildmessageattachments.py b/django_mailbox/management/commands/rebuildmessageattachments.py new file mode 100644 index 0000000..3bd2eb3 --- /dev/null +++ b/django_mailbox/management/commands/rebuildmessageattachments.py @@ -0,0 +1,71 @@ +import email +import hashlib +import logging + +from django.core.management.base import BaseCommand + +from django_mailbox.models import MessageAttachment, Message + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +class Command(BaseCommand): + """ Briefly, a bug existed in a migration that may have caused message + attachments to become disassociated with their messages. This management + command will read through existing message attachments and attempt to + re-associate them with their original message. + + This isn't foolproof, I'm afraid. If an attachment exists twice, it will + be associated only with the most recent e-mail message. That said, + I'm quite sure that the bug in the migration is gone (and you'd have to + have been quite unlucky to have ran the bad migration). + + """ + def handle(self, *args, **options): + ATTACHMENT_HASH_MAP = {} + + attachments_without_messages = MessageAttachment.objects.filter( + message=None + ).order_by( + 'id' + ) + + if attachments_without_messages.count() < 1: + return + + for attachment in attachments_without_messages: + md5 = hashlib.md5() + for chunk in attachment.document.file.chunks(): + md5.update(chunk) + ATTACHMENT_HASH_MAP[md5.hexdigest()] = attachment.pk + + for message_record in Message.objects.all().order_by('id'): + message = email.message_from_string(message_record.body) + if message.is_multipart(): + for part in message.walk(): + if part.get_content_maintype() == 'multipart': + continue + if part.get('Content-Disposition') is None: + continue + md5 = hashlib.md5() + md5.update(part.get_payload(decode=True)) + digest = md5.hexdigest() + if digest in ATTACHMENT_HASH_MAP: + attachment = MessageAttachment.objects.get( + pk=ATTACHMENT_HASH_MAP[digest] + ) + attachment.message = message_record + attachment.save() + logger.info( + "Associated message %s with attachment %s (%s)", + message_record.pk, + attachment.pk, + digest + ) + else: + logger.info( + "%s(%s) not found in currently-stored attachments", + part.get_filename(), + digest + ) diff --git a/django_mailbox/migrations/0013_auto__add_field_messageattachment_message.py b/django_mailbox/migrations/0013_auto__add_field_messageattachment_message.py index 05822bb..d1b9b0d 100644 --- a/django_mailbox/migrations/0013_auto__add_field_messageattachment_message.py +++ b/django_mailbox/migrations/0013_auto__add_field_messageattachment_message.py @@ -30,7 +30,7 @@ class Migration(SchemaMigration): }, u'django_mailbox.message': { 'Meta': {'object_name': 'Message'}, - 'attachments_old': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'messages_old'", 'blank': 'True', 'to': u"orm['django_mailbox.MessageAttachment']"}), + 'attachments': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'messages_old'", 'blank': 'True', 'to': u"orm['django_mailbox.MessageAttachment']"}), 'body': ('django.db.models.fields.TextField', [], {}), 'from_header': ('django.db.models.fields.CharField', [], {'max_length': '255'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), @@ -47,7 +47,7 @@ class Migration(SchemaMigration): 'Meta': {'object_name': 'MessageAttachment'}, 'document': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'message': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'attachments'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}) + 'message': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'attachments_new'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}) } } diff --git a/django_mailbox/migrations/0014_migrate_message_attachments.py b/django_mailbox/migrations/0014_migrate_message_attachments.py index 7f704f8..d2391c9 100644 --- a/django_mailbox/migrations/0014_migrate_message_attachments.py +++ b/django_mailbox/migrations/0014_migrate_message_attachments.py @@ -9,14 +9,14 @@ class Migration(SchemaMigration): def forwards(self, orm): for message in orm['django_mailbox.Message'].objects.all(): - for attachment in message.attachments_old.all(): + for attachment in message.attachments.all(): attachment.message = message attachment.save() def backwards(self, orm): for attachment in orm['django_mailbox.MessageAttachment'].objects.all(): if attachment.message: - attachment.message.attachments_old.add( + attachment.message.attachments.add( attachment ) @@ -31,7 +31,7 @@ class Migration(SchemaMigration): }, u'django_mailbox.message': { 'Meta': {'object_name': 'Message'}, - 'attachments_old': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'messages_old'", 'blank': 'True', 'to': u"orm['django_mailbox.MessageAttachment']"}), + 'attachments': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'messages_old'", 'blank': 'True', 'to': u"orm['django_mailbox.MessageAttachment']"}), 'body': ('django.db.models.fields.TextField', [], {}), 'from_header': ('django.db.models.fields.CharField', [], {'max_length': '255'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), @@ -48,7 +48,7 @@ class Migration(SchemaMigration): 'Meta': {'object_name': 'MessageAttachment'}, 'document': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'message': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'attachments'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}) + 'message': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'attachments_new'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}) } } diff --git a/django_mailbox/migrations/0015_rebuild_message_associations.py b/django_mailbox/migrations/0015_rebuild_message_associations.py deleted file mode 100644 index 783cd9e..0000000 --- a/django_mailbox/migrations/0015_rebuild_message_associations.py +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- -import datetime -import email -import hashlib -import logging -from south.db import db -from south.v2 import SchemaMigration -from django.db import models - - -logger = logging.getLogger(__name__) - - -class Migration(SchemaMigration): - - def forwards(self, orm): - ATTACHMENT_HASH_MAP = {} - - attachments_without_messages = orm[ - 'django_mailbox.messageattachment' - ].objects.filter(message=None).order_by('id') - - if attachments_without_messages.count() < 1: - return - - for attachment in attachments_without_messages: - md5 = hashlib.md5() - for chunk in attachment.document.file.chunks(): - md5.update(chunk) - ATTACHMENT_HASH_MAP[md5.hexdigest()] = attachment.pk - - for message_record in orm['django_mailbox.message'].objects.all().order_by('id'): - message = email.message_from_string(message_record.body) - if message.is_multipart(): - for part in message.walk(): - if part.get_content_maintype() == 'multipart': - continue - if part.get('Content-Disposition') is None: - continue - md5 = hashlib.md5() - md5.update(part.get_payload(decode=True)) - digest = md5.hexdigest() - if digest in ATTACHMENT_HASH_MAP: - attachment = orm[ - 'django_mailbox.messageattachment' - ].objects.get(pk=ATTACHMENT_HASH_MAP[digest]) - attachment.message = message_record - attachment.save() - logger.info( - "Associated message %s with attachment %s (%s)", - message_record.pk, - attachment.pk, - digest - ) - else: - logger.info( - "%s(%s) not found in currently-stored attachments", - part.get_filename(), - digest - ) - - def backwards(self, orm): - # No backward migration necessary - pass - - models = { - u'django_mailbox.mailbox': { - 'Meta': {'object_name': 'Mailbox'}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'from_email': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'uri': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}) - }, - u'django_mailbox.message': { - 'Meta': {'object_name': 'Message'}, - 'body': ('django.db.models.fields.TextField', [], {}), - 'from_header': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'in_reply_to': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'replies'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}), - 'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'messages'", 'to': u"orm['django_mailbox.Mailbox']"}), - 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'outgoing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'processed': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'read': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}), - 'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}), - 'to_header': ('django.db.models.fields.TextField', [], {}) - }, - u'django_mailbox.messageattachment': { - 'Meta': {'object_name': 'MessageAttachment'}, - 'document': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'message': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'attachments'", 'null': 'True', 'to': u"orm['django_mailbox.Message']"}) - } - } - - complete_apps = ['django_mailbox'] diff --git a/setup.py b/setup.py index 2d76937..5d7316e 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ tests_require=[ setup( name='django-mailbox', - version='1.10.3', + version='1.10.4', url='http://bitbucket.org/latestrevision/django-mailbox/', description='Import mail from POP3, IMAP, local mailboxes or directly from Postfix or Exim4 into your Django application automatically.', author='Adam Coddington',