From eca0b2caa456316b9d506a6f9c800a6d74f87a7c Mon Sep 17 00:00:00 2001 From: "Anders.Du" Date: Fri, 19 Jul 2013 23:39:41 +0800 Subject: [PATCH 1/5] * Add decoding for some encoded messages parts, i.e. attachments' name, message header, etc. * Remove overriding for MessageAttachment's __getitem__ method, so we can call the members directly in django templates. Signed-off-by: Anders.Du --- django_mailbox/models.py | 105 +++++++++++++++++++++++++++------------ 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index ddc7344..6504d1a 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -6,6 +6,7 @@ import urllib import mimetypes import os.path from quopri import encode as encode_quopri +from quopri import decodestring as decodestring_quopri import sys import uuid @@ -24,6 +25,12 @@ from django_mailbox.transports import Pop3Transport, ImapTransport,\ from django_mailbox.signals import message_received import six +from dateutil import parser +from django.db import IntegrityError, DatabaseError +from django.utils.encoding import smart_unicode, smart_str +from tickets.models import Ticket +import re +import traceback STRIP_UNALLOWED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES', @@ -243,23 +250,35 @@ class Mailbox(models.Model): return new def _process_message(self, message): - msg = Message() - msg.mailbox = self - msg.subject = message['subject'][0:255] - msg.message_id = message['message-id'][0:255] - msg.from_header = message['from'] - msg.to_header = message['to'] - msg.save() - message = self._get_dehydrated_message(message, msg) - msg.body = message.as_string() - if message['in-reply-to']: - try: - msg.in_reply_to = Message.objects.filter( - message_id=message['in-reply-to'] - )[0] - except IndexError: - pass - msg.save() + try: + msg = Message() + msg.mailbox = self + if 'subject' in message: + msg.subject = message['subject'][0:255] + if 'message-id' in message: + msg.message_id = message['message-id'][0:255] + if 'from' in message: + msg.from_header = message['from'] + if 'to' in message: + msg.to_header = message['to'] + msg.save() + message = self._get_dehydrated_message(message, msg) + msg.body = unicode(e_message.as_string(),errors='ignore') + if message['in-reply-to']: + try: + msg.in_reply_to = Message.objects.filter( + message_id=message['in-reply-to'] + )[0] + except IndexError: + pass + msg.save() + except IntegrityError,e: + traceback.print_exc() + pass + except DatabaseError, e: + traceback.print_exc() + pass + return msg def get_new_mail(self): @@ -346,24 +365,27 @@ class Message(models.Model): separate fields. """ - if self.outgoing: - return self.to_addresses() - else: - return self.from_addresses() + addresses = [] + addresses = self.to_addresses + self.from_address + return addresses @property def from_address(self): - return parseaddr(self.from_header)[1] + if self.from_header: + return [parseaddr(self.from_header)[1].lower()] + else: + return [] @property def to_addresses(self): addresses = [] for address in self.to_header.split(','): - addresses.append( - parseaddr( - address - )[1] - ) + if address: + addresses.append( + parseaddr( + address + )[1].lower() + ) return addresses def reply(self, message): @@ -396,12 +418,21 @@ class Message(models.Model): def get_text_body(self): def get_body_from_message(message): body = '' + charset = 'ascii' for part in message.walk(): if ( part.get_content_maintype() == 'text' and part.get_content_subtype() == 'plain' ): - body = body + part.get_payload() + # body = body + part.get_payload() + if part.get_content_charset(): + charset = part.get_content_charset() + encoding = part['Content-Transfer-Encoding'] + # print encoding + if encoding and encoding.lower() == 'base64': + body = body + unicode(part.get_payload().decode('base64'), charset) + else: + body = body + unicode(decodestring_quopri(part.get_payload()), charset) return body return get_body_from_message( @@ -531,13 +562,25 @@ class MessageAttachment(models.Model): self._set_dehydrated_headers(rehydrated) def get_filename(self): - return self._get_rehydrated_headers().get_filename() + file_name = self._get_rehydrated_headers().get_filename() + if file_name: + try: + d_subject = email.Header.decode_header(file_name) + except UnicodeEncodeError, e: + print e + return file_name + if d_subject[0][1]: + return unicode(d_subject[0][0], d_subject[0][1]) + else: + return d_subject[0][0] + else: + return '' def items(self): return self._get_rehydrated_headers().items() - def __getitem__(self, name): - return self._get_rehydrated_headers()[name] + # def __getitem__(self, name): + # return self._get_rehydrated_headers()[name] def __unicode__(self): return self.document.url From 491b489867dd8b29458b376109964e29526924e7 Mon Sep 17 00:00:00 2001 From: "Anders.Du" Date: Sun, 21 Jul 2013 17:31:44 +0800 Subject: [PATCH 2/5] Fix the IndentationError in last commit. --- django_mailbox/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 6504d1a..1c9d3dd 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -263,7 +263,7 @@ class Mailbox(models.Model): msg.to_header = message['to'] msg.save() message = self._get_dehydrated_message(message, msg) - msg.body = unicode(e_message.as_string(),errors='ignore') + msg.body = unicode(e_message.as_string(),errors='ignore') if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( From 6697e848c8fb5df719135616e1861e442510fd0e Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Mon, 22 Jul 2013 18:16:28 +0100 Subject: [PATCH 3/5] Support for Django 1.4 with python 2.5, copied from https://github.com/jhmaddox/django-precompressed/issues/1 ("AttributeError: StringIO instance has no attribute 'name'") --- django_mailbox/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 1c9d3dd..c4c847f 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -17,7 +17,7 @@ except ImportError: from django.conf import settings from django.core.mail.message import make_msgid -from django.core.files import File +from django.core.files.base import File, ContentFile from django.db import models from django_mailbox.transports import Pop3Transport, ImapTransport,\ MaildirTransport, MboxTransport, BabylTransport, MHTransport, \ @@ -229,9 +229,10 @@ class Mailbox(models.Model): _, extension = os.path.splitext(filename) attachment = MessageAttachment() + attachment.document.save( uuid.uuid4().hex + extension, - File(six.BytesIO(msg.get_payload(decode=True))) + ContentFile(six.BytesIO(msg.get_payload(decode=True)).getvalue()) ) attachment.message = record for key, value in msg.items(): From ac0f2153c0e1e9902162ed7b577850b96049e134 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Thu, 25 Jul 2013 18:47:49 +0100 Subject: [PATCH 4/5] Applied cleaned up fixes from Anders.Du (duyu) --- django_mailbox/models.py | 57 ++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/django_mailbox/models.py b/django_mailbox/models.py index c4c847f..0714e09 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -25,12 +25,7 @@ from django_mailbox.transports import Pop3Transport, ImapTransport,\ from django_mailbox.signals import message_received import six -from dateutil import parser -from django.db import IntegrityError, DatabaseError -from django.utils.encoding import smart_unicode, smart_str -from tickets.models import Ticket -import re -import traceback + STRIP_UNALLOWED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES', @@ -251,35 +246,27 @@ class Mailbox(models.Model): return new def _process_message(self, message): - try: - msg = Message() - msg.mailbox = self - if 'subject' in message: - msg.subject = message['subject'][0:255] - if 'message-id' in message: - msg.message_id = message['message-id'][0:255] - if 'from' in message: - msg.from_header = message['from'] - if 'to' in message: - msg.to_header = message['to'] - msg.save() - message = self._get_dehydrated_message(message, msg) - msg.body = unicode(e_message.as_string(),errors='ignore') - if message['in-reply-to']: - try: - msg.in_reply_to = Message.objects.filter( - message_id=message['in-reply-to'] - )[0] - except IndexError: - pass - msg.save() - except IntegrityError,e: - traceback.print_exc() - pass - except DatabaseError, e: - traceback.print_exc() - pass - + msg = Message() + msg.mailbox = self + if 'subject' in message: + msg.subject = message['subject'][0:255] + if 'message-id' in message: + msg.message_id = message['message-id'][0:255] + if 'from' in message: + msg.from_header = message['from'] + if 'to' in message: + msg.to_header = message['to'] + msg.save() + message = self._get_dehydrated_message(message, msg) + msg.body = unicode(message.as_string(),errors='ignore') + if message['in-reply-to']: + try: + msg.in_reply_to = Message.objects.filter( + message_id=message['in-reply-to'] + )[0] + except IndexError: + pass + msg.save() return msg def get_new_mail(self): From d06c64a4872be0f65134f479694986b3ae3118c5 Mon Sep 17 00:00:00 2001 From: Yuri Baburov Date: Thu, 25 Jul 2013 17:49:27 +0000 Subject: [PATCH 5/5] Added subject decode --- django_mailbox/admin.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/django_mailbox/admin.py b/django_mailbox/admin.py index 896bf6b..84bbba1 100644 --- a/django_mailbox/admin.py +++ b/django_mailbox/admin.py @@ -44,16 +44,23 @@ class MessageAttachmentInline(admin.TabularInline): model = MessageAttachment extra = 0 +default_charset = 'ASCII' +import email.header +def decode(m): + return ''.join([ unicode(t[0], t[1] or default_charset) for t in email.header.decode_header(m) ]) class MessageAdmin(admin.ModelAdmin): def attachment_count(self, msg): return msg.attachments.count() + def Subject(self, msg): + return decode(msg.subject) + inlines = [ MessageAttachmentInline, ] list_display = ( - 'subject', + 'Subject', 'processed', 'read', 'mailbox',