From b7192e914951ba7b40d36488d09b664e13b519e2 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Sun, 7 Oct 2012 19:43:38 -0700 Subject: [PATCH 01/11] Model modifications for allowing empty source inputs; adding processincomingmessage management command. --HG-- branch : exim4_pipe --- .../management/commands/processincomingmessage.py | 14 ++++++++++++++ django_mailbox/models.py | 4 +++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 django_mailbox/management/commands/processincomingmessage.py diff --git a/django_mailbox/management/commands/processincomingmessage.py b/django_mailbox/management/commands/processincomingmessage.py new file mode 100644 index 0000000..15d8b7b --- /dev/null +++ b/django_mailbox/management/commands/processincomingmessage.py @@ -0,0 +1,14 @@ +import email +import ipdb +import sys + +from django.core.managemet.base import BaseCommand + +from django_mailbox.models import Mailbox, Message + +class Command(BaseCommand): + def handle(self, *args, **options): + message_string = open(sys.stdin, 'r').read() + ipdb.set_trace() + #message = email.message_from_string(message_string) + diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 50ea5d8..f293850 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -22,7 +22,9 @@ class Mailbox(models.Model):
Be sure to urlencode your username and password should they contain illegal characters (like @, :, etc). - """ + """, + blank=True, + default=None, ) @property From 5cf07a1c7b1dbe4b9ad32086e9621dcd7eb72aec Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 03:07:30 +0000 Subject: [PATCH 02/11] Modifying management command to accept incoming mail from STDIN. --HG-- branch : exim4_pipe --- .../commands/processincomingmessage.py | 26 ++++++++++++++----- django_mailbox/models.py | 20 ++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/django_mailbox/management/commands/processincomingmessage.py b/django_mailbox/management/commands/processincomingmessage.py index 15d8b7b..77c9788 100644 --- a/django_mailbox/management/commands/processincomingmessage.py +++ b/django_mailbox/management/commands/processincomingmessage.py @@ -1,14 +1,28 @@ import email -import ipdb +import logging +import rfc822 import sys -from django.core.managemet.base import BaseCommand +from django.core.management.base import BaseCommand -from django_mailbox.models import Mailbox, Message +from django_mailbox.models import Mailbox + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) class Command(BaseCommand): def handle(self, *args, **options): - message_string = open(sys.stdin, 'r').read() - ipdb.set_trace() - #message = email.message_from_string(message_string) + message = email.message_from_string(sys.stdin.read()) + if message: + mailbox = self.get_mailbox_for_message(message) + mailbox.process_incoming_message(message) + logger.info("Message received from %s" % message['from']) + else: + logger.warning("Message not processable.") + def get_mailbox_for_message(self, message): + email_address = rfc822.parseaddr(message['from'][1][0:255]) + mailbox, created = Mailbox.objects.get_or_create( + name=email_address, + ) + return mailbox diff --git a/django_mailbox/models.py b/django_mailbox/models.py index f293850..30d697b 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -89,19 +89,23 @@ class Mailbox(models.Model): conn = MMDFTransport(self.location) return conn + def process_incoming_message(self, message): + msg = Message() + msg.mailbox = self + msg.subject = message['subject'][0:255] + msg.message_id = message['message-id'][0:255] + msg.from_address = rfc822.parseaddr(message['from'])[1][0:255] + msg.body = message.as_string() + msg.save() + message_received.send(sender=self, message=msg) + return msg + def get_new_mail(self): connection = self.get_connection() new_mail = [] for message in connection.get_message(): - msg = Message() - msg.mailbox = self - msg.subject = message['subject'][0:255] - msg.message_id = message['message-id'][0:255] - msg.from_address = rfc822.parseaddr(message['from'])[1][0:255] - msg.body = message.as_string() - msg.save() + msg = self.process_incoming_message(message) new_mail.append(msg) - message_received.send(sender=self, message=msg) return new_mail def __unicode__(self): From 24c2a98d81c3fd7e52baaff6dc0dab9a8e867904 Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 15:24:42 +0000 Subject: [PATCH 03/11] Adding migration for handling incoming messages; adding exim4 scripts for routing messages to django_messages. --HG-- branch : exim4_pipe --- .../0002_auto__chg_field_mailbox_uri.py | 39 +++++++++++++++++++ django_mailbox/models.py | 1 + scripts/router | 6 +++ scripts/transport | 8 ++++ 4 files changed, 54 insertions(+) create mode 100644 django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py create mode 100644 scripts/router create mode 100644 scripts/transport diff --git a/django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py b/django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py new file mode 100644 index 0000000..3ff3098 --- /dev/null +++ b/django_mailbox/migrations/0002_auto__chg_field_mailbox_uri.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Changing field 'Mailbox.uri' + db.alter_column('django_mailbox_mailbox', 'uri', self.gf('django.db.models.fields.CharField')(max_length=255, null=True)) + + def backwards(self, orm): + + # User chose to not deal with backwards NULL issues for 'Mailbox.uri' + raise RuntimeError("Cannot reverse this migration. 'Mailbox.uri' and its values cannot be restored.") + + models = { + 'django_mailbox.mailbox': { + 'Meta': {'object_name': 'Mailbox'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'uri': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}) + }, + 'django_mailbox.message': { + 'Meta': {'object_name': 'Message'}, + 'body': ('django.db.models.fields.TextField', [], {}), + 'from_address': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mailbox': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'messages'", 'to': "orm['django_mailbox.Mailbox']"}), + 'message_id': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'received': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'subject': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['django_mailbox'] \ No newline at end of file diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 30d697b..5e91395 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -24,6 +24,7 @@ class Mailbox(models.Model): contain illegal characters (like @, :, etc). """, blank=True, + null=True, default=None, ) diff --git a/scripts/router b/scripts/router new file mode 100644 index 0000000..3980d16 --- /dev/null +++ b/scripts/router @@ -0,0 +1,6 @@ +django_mailbox: + debug_print = 'R: django_mailbox for $localpart@$domain" + driver = accept + domains = +local_domains + transport = send_to_django_mailbox + local_parts = emailaddressusername : anotheremailaddressusername diff --git a/scripts/transport b/scripts/transport new file mode 100644 index 0000000..21a2d1a --- /dev/null +++ b/scripts/transport @@ -0,0 +1,8 @@ + +send_to_django_mailbox: + driver = pipe + command = /path/to/python /path/to/manage.py processincomingmessage + user = www-data + group = www-data + return_path_add + delivery_date_add From e165d16660cb10a7008be2b7566a6db471dafde0 Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 15:27:46 +0000 Subject: [PATCH 04/11] Fixing a typo that handles generating new mailbox names. --HG-- branch : exim4_pipe --- django_mailbox/management/commands/processincomingmessage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/management/commands/processincomingmessage.py b/django_mailbox/management/commands/processincomingmessage.py index 77c9788..b286ae7 100644 --- a/django_mailbox/management/commands/processincomingmessage.py +++ b/django_mailbox/management/commands/processincomingmessage.py @@ -21,7 +21,7 @@ class Command(BaseCommand): logger.warning("Message not processable.") def get_mailbox_for_message(self, message): - email_address = rfc822.parseaddr(message['from'][1][0:255]) + email_address = rfc822.parseaddr(message['from'])[1][0:255] mailbox, created = Mailbox.objects.get_or_create( name=email_address, ) From 0515cf7f7495f3297f6c34515da7dfba381001f4 Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 15:28:37 +0000 Subject: [PATCH 05/11] Do not attempt to fetch mail for mailbox not having a URI. --HG-- branch : exim4_pipe --- django_mailbox/management/commands/getmail.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_mailbox/management/commands/getmail.py b/django_mailbox/management/commands/getmail.py index 5f1aad3..e74f683 100644 --- a/django_mailbox/management/commands/getmail.py +++ b/django_mailbox/management/commands/getmail.py @@ -8,6 +8,8 @@ class Command(BaseCommand): if args: mailboxes = mailboxes.filter(name = ' '.join(args)) for mailbox in mailboxes: + if not mailbox.uri: + continue self.stdout.write('Gathering messages for %s\n' % mailbox.name) messages = mailbox.get_new_mail() for message in messages: From ffbf4ace6b638085de260003eddddd28e89e32b2 Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 15:30:32 +0000 Subject: [PATCH 06/11] Return None for connection if no URI is specified in mailbox settings. --HG-- branch : exim4_pipe --- django_mailbox/management/commands/getmail.py | 2 -- django_mailbox/models.py | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/django_mailbox/management/commands/getmail.py b/django_mailbox/management/commands/getmail.py index e74f683..5f1aad3 100644 --- a/django_mailbox/management/commands/getmail.py +++ b/django_mailbox/management/commands/getmail.py @@ -8,8 +8,6 @@ class Command(BaseCommand): if args: mailboxes = mailboxes.filter(name = ' '.join(args)) for mailbox in mailboxes: - if not mailbox.uri: - continue self.stdout.write('Gathering messages for %s\n' % mailbox.name) messages = mailbox.get_new_mail() for message in messages: diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 5e91395..41be837 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -64,7 +64,9 @@ class Mailbox(models.Model): return '+ssl' in self._protocol_info.scheme.lower() def get_connection(self): - if self.type == 'imap': + if not self.uri: + return None + elif self.type == 'imap': conn = ImapTransport( self.location, port=self.port if self.port else None, @@ -102,8 +104,10 @@ class Mailbox(models.Model): return msg def get_new_mail(self): - connection = self.get_connection() new_mail = [] + connection = self.get_connection() + if not connection: + return new_mail for message in connection.get_message(): msg = self.process_incoming_message(message) new_mail.append(msg) From 00a4bde767b4392e19ef366c7d9f916415af2762 Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 15:33:07 +0000 Subject: [PATCH 07/11] Use TO header for creating mailbox name. --HG-- branch : exim4_pipe --- django_mailbox/management/commands/processincomingmessage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mailbox/management/commands/processincomingmessage.py b/django_mailbox/management/commands/processincomingmessage.py index b286ae7..e33b369 100644 --- a/django_mailbox/management/commands/processincomingmessage.py +++ b/django_mailbox/management/commands/processincomingmessage.py @@ -21,7 +21,7 @@ class Command(BaseCommand): logger.warning("Message not processable.") def get_mailbox_for_message(self, message): - email_address = rfc822.parseaddr(message['from'])[1][0:255] + email_address = rfc822.parseaddr(message['to'])[1][0:255] mailbox, created = Mailbox.objects.get_or_create( name=email_address, ) From 0162d235564d5c49224500b4b139895b806f0ab3 Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 19:59:11 +0000 Subject: [PATCH 08/11] Adding instructions for pipe-based delivery. --HG-- branch : exim4_pipe --- readme.rst | 79 ++++++++++++++++++++++++++++++++++------------- scripts/router | 6 ---- scripts/transport | 8 ----- 3 files changed, 58 insertions(+), 35 deletions(-) delete mode 100644 scripts/router delete mode 100644 scripts/transport diff --git a/readme.rst b/readme.rst index 3b7a43d..9855045 100755 --- a/readme.rst +++ b/readme.rst @@ -26,8 +26,8 @@ You can either install from pip:: cd django-mailbox python setup.py install -Setting up your mailbox -======================= +Polling for mail in POP3/IMAP or a local mailbox +================================================ Django Mailbox supports both common internet mailboxes like POP3 and IMAP as well as local file-based mailboxes. @@ -71,6 +71,62 @@ If you happen to want to consume a file-based mailbox like an Maildir, Mbox, Bab Note that there is an additional ``/`` in the above URI after the protocol; this is important. +Getting incoming mail +--------------------- + +In your code +~~~~~~~~~~~~ + +Mailbox instances have a method named ``get_new_mail``; this method will gather new messages from the server. + +Using the Django Admin +~~~~~~~~~~~~~~~~~~~~~~ + +Check the box next to each of the mailboxes you'd like to fetch e-mail from, and select the 'Get new mail' option. + +Using a cron job +~~~~~~~~~~~~~~~~ + +You can easily consume incoming mail by running the management command named ``getmail`` (optionally with an argument of the name of the mailbox you'd like to get the mail for).:: + + python manage.py getmail + +Receiving mail directly from EXIM4 or Postfix via a pipe +======================================================== + +Django Mailbox's ``processincomingmessage`` management command accespts, via ``stdin``, incoming messages. You can configure Postfix or Exim4 to pipe incoming mail to this management command to import messages directly without polling. + +Receiving Mail from Exim4 +------------------------- + +You should add a new router configuration to your Exim4 configuration like:: + + django_mailbox: + debug_print = 'R: django_mailbox for $localpart@$domain' + driver = accept + domains = +local_domains + transport = send_to_django_mailbox + local_parts = emailusernameone : emailusernametwo + +You will want to make sure that the e-mail addresses you would like handled by Django Mailbox are not handled by another router, so you may need to disable some existing routers. Also, be sure to change the contents of ``local_parts`` to match a colon-delimited list of usernames for which you would like to receive mail for. For example, if one of the e-mail addresses targeted at this machine is ``jane@example.com``, the contents of ``local_parts`` would be, simply ``jane``. + +You should also add a new transport configuration to your Exim4 configuration:: + + send_to_django_mailbox: + driver = pipe + command = /path/to/your/environments/python /path/to/your/projects/manage.py processincomingmessage + user = www-data + group = www-data + return_path_add + delivery_date_add + +Like your router configuration, you will need to alter this transport configuration. First, you will want to modify the ``command`` setting such that it points at the proper python binary (if you're using a virtual environment, you'll want to direct that at the python binary in your virtual environment) and project ``manage.py`` script. Additionally, you'll need to set ``user`` and ``group`` such that they match a reasonable user and group (on Ubuntu, ``www-data`` suffices for both). + +Receiving mail from Postfix +--------------------------- + +Although I have not personally tried using Postfix for this, Postfix is capable of delivering new mail to a script using ``pipe``. Please consult the `Postfix documentation for pipe here `_. You may want to consult the above Exim4 configuration for tips. + Subscribing to the incoming mail signal ======================================= @@ -83,25 +139,6 @@ To subscribe to the incoming mail signal, following this lead:: def dance_jig(sender, message, **args): print "I just recieved a message titled %s from a mailbox named %s" % (message.subject, message.mailbox.name, ) -Getting incoming mail -======================= - -In your code ------------- - -Mailbox instances have a method named ``get_new_mail``; this method will gather new messages from the server. - -Using the Django Admin ----------------------- - -Check the box next to each of the mailboxes you'd like to fetch e-mail from, and select the 'Get new mail' option. - -Using a cron job ----------------- - -You can easily consume incoming mail by running the management command named ``getmail`` (optionally with an argument of the name of the mailbox you'd like to get the mail for).:: - - python manage.py getmail Settings ======== diff --git a/scripts/router b/scripts/router deleted file mode 100644 index 3980d16..0000000 --- a/scripts/router +++ /dev/null @@ -1,6 +0,0 @@ -django_mailbox: - debug_print = 'R: django_mailbox for $localpart@$domain" - driver = accept - domains = +local_domains - transport = send_to_django_mailbox - local_parts = emailaddressusername : anotheremailaddressusername diff --git a/scripts/transport b/scripts/transport deleted file mode 100644 index 21a2d1a..0000000 --- a/scripts/transport +++ /dev/null @@ -1,8 +0,0 @@ - -send_to_django_mailbox: - driver = pipe - command = /path/to/python /path/to/manage.py processincomingmessage - user = www-data - group = www-data - return_path_add - delivery_date_add From 677290c3904cba1cc24b5df3dcffbc1fa138c15b Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 20:04:33 +0000 Subject: [PATCH 09/11] Adding more-thorough explanations of pipe-based mailboxes. --HG-- branch : exim4_pipe --- readme.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/readme.rst b/readme.rst index 9855045..3ddd495 100755 --- a/readme.rst +++ b/readme.rst @@ -1,7 +1,7 @@ Introduction ~~~~~~~~~~~~ -How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content? One too many times for me. +How many times have you had to consume some sort of POP3, IMAP, or local mailbox for incoming content, or had to otherwise construct an application driven by e-mail? One too many times, I'm sure. This small Django application will allow you to specify mailboxes that you would like consumed for incoming content; the e-mail will be stored, and you can process it at will (or, if you're in a hurry, by subscribing to a signal). @@ -29,13 +29,13 @@ You can either install from pip:: Polling for mail in POP3/IMAP or a local mailbox ================================================ -Django Mailbox supports both common internet mailboxes like POP3 and IMAP as well as local file-based mailboxes. +Django Mailbox supports both common internet mailboxes like POP3 and IMAP, local file-based mailboxes, as well as direct pipe-based delivery. .. table:: 'Protocol' Options - ============ ============== ========================================= + ============ ============== ==================================================== Mailbox Type 'Protocol':// Notes - ============ ============== ========================================= + ============ ============== ==================================================== POP3 ``pop3://`` Can also specify SSL with ``pop3+ssl://`` IMAP ``imap://`` Can also specify SSL with ``imap+ssl://`` Maildir ``maildir://`` @@ -43,7 +43,8 @@ Django Mailbox supports both common internet mailboxes like POP3 and IMAP as wel Babyl ``babyl://`` MH ``mh://`` MMDF ``mmdf://`` - ============ ============== ========================================= + Piped Mail *empty* For pipe-based mailboxes, no URI should be specified + ============ ============== ==================================================== POP3 and IMAP Mailboxes ----------------------- @@ -139,7 +140,6 @@ To subscribe to the incoming mail signal, following this lead:: def dance_jig(sender, message, **args): print "I just recieved a message titled %s from a mailbox named %s" % (message.subject, message.mailbox.name, ) - Settings ======== From dd1393a6536035dab3356c58c26742c4be118787 Mon Sep 17 00:00:00 2001 From: "me@adamcoddington.net" Date: Mon, 8 Oct 2012 20:06:40 +0000 Subject: [PATCH 10/11] Fixing documentation headlining. --HG-- branch : exim4_pipe --- readme.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readme.rst b/readme.rst index 3ddd495..faf24a7 100755 --- a/readme.rst +++ b/readme.rst @@ -76,17 +76,17 @@ Getting incoming mail --------------------- In your code -~~~~~~~~~~~~ +............ Mailbox instances have a method named ``get_new_mail``; this method will gather new messages from the server. Using the Django Admin -~~~~~~~~~~~~~~~~~~~~~~ +...................... Check the box next to each of the mailboxes you'd like to fetch e-mail from, and select the 'Get new mail' option. Using a cron job -~~~~~~~~~~~~~~~~ +................ You can easily consume incoming mail by running the management command named ``getmail`` (optionally with an argument of the name of the mailbox you'd like to get the mail for).:: From 1758edd606a99ac814feba27217248105ed314bf Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 8 Oct 2012 21:23:05 -0700 Subject: [PATCH 11/11] Closing exim4 branch. --HG-- branch : exim4_pipe extra : close : 1