diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 315048b..ddc7344 100755 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -24,11 +24,6 @@ from django_mailbox.transports import Pop3Transport, ImapTransport,\ from django_mailbox.signals import message_received import six -SKIPPED_EXTENSIONS = getattr( - settings, - 'DJANGO_MAILBOX_SKIPPED_EXTENSIONS', - ['.p7s'] -) STRIP_UNALLOWED_MIMETYPES = getattr( settings, 'DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES', diff --git a/docs/conf.py b/docs/conf.py index c5ef780..4b039e6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ copyright = u'2013, Adam Coddington' # built documents. # # The short X.Y version. -version = '1.9' +version = '2.1' # The full version, including alpha/beta/rc tags. -release = '1.9' +release = '2.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/topics/mailbox_types.rst b/docs/topics/mailbox_types.rst index 9f74468..0e60bae 100644 --- a/docs/topics/mailbox_types.rst +++ b/docs/topics/mailbox_types.rst @@ -2,7 +2,8 @@ Supported Mailbox Types ======================= -Django Mailbox supports polling both common internet mailboxes like POP3 and IMAP as well as local file-based mailboxes. +Django Mailbox supports polling both common internet mailboxes like +POP3 and IMAP as well as local file-based mailboxes. .. table:: 'Protocol' Options @@ -19,9 +20,11 @@ Django Mailbox supports polling both common internet mailboxes like POP3 and IMA Piped Mail *empty* See :ref:`receiving-mail-from-exim4-or-postfix` ============ ============== =============================================== -.. WARNING:: +.. warning:: + This will delete any messages it can find in the inbox you specify; - do not use an e-mail inbox that you would like to share between applications. + do not use an e-mail inbox that you would like to share between + applications. POP3 and IMAP Mailboxes @@ -37,12 +40,13 @@ Basic POP3 Example: ``pop3://username:password@server`` Most mailboxes these days are SSL-enabled; if yours is, add ``+ssl`` to your URI. -Also, if your username or password include any non-ascii characters, they should be URL-encoded -(for example, if your username includes an ``@``, it should be changed to ``%40`` in your URI). +Also, if your username or password include any non-ascii characters, +they should be URL-encoded (for example, if your username includes an +``@``, it should be changed to ``%40`` in your URI). -If you have an account named ``youremailaddress@gmail.com`` with a password of ``1234`` on GMail, -which uses a POP3 server of 'pop.gmail.com' and requires SSL, -you would enter the following as your URI:: +If you have an account named ``youremailaddress@gmail.com`` with a password +of ``1234`` on GMail, which uses a POP3 server of 'pop.gmail.com' and requires +SSL, you would enter the following as your URI:: pop3+ssl://youremailaddress%40gmail.com:1234@pop.gmail.com @@ -50,9 +54,10 @@ you would enter the following as your URI:: Local File-based Mailboxes -------------------------- -If you happen to want to consume a file-based mailbox like an Maildir, Mbox, Babyl, MH, or MMDF mailbox, -you can use this too by entering the appropriate 'protocol' in the URI. -If you had a maildir, for example, at ``/var/mail/``, you would enter a URI like:: +If you happen to want to consume a file-based mailbox like an Maildir, Mbox, +Babyl, MH, or MMDF mailbox, you can use this too by entering the appropriate +'protocol' in the URI. If you had a maildir, for example, at ``/var/mail/``, +you would enter a URI like:: maildir:///var/mail diff --git a/docs/topics/message-storage.rst b/docs/topics/message-storage.rst new file mode 100644 index 0000000..7f96b9c --- /dev/null +++ b/docs/topics/message-storage.rst @@ -0,0 +1,149 @@ + +Message Storage Details +======================= + +First, it may be helpful to know a little bit about how e-mail messages +are actually sent across the wire: + +.. code-block:: http + + MIME-Version: 1.0 + Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 12:07:07 -0800 (PST) + X-Originating-IP: [24.22.122.177] + Date: Sun, 20 Jan 2013 12:07:07 -0800 + Delivered-To: test@adamcoddington.net + Message-ID: + Subject: Message With Attachment + From: Adam Coddington + To: Adam Coddington + Content-Type: multipart/mixed; boundary=047d7b33dd729737fe04d3bde348 + + --047d7b33dd729737fe04d3bde348 + Content-Type: text/plain; charset=UTF-8 + + This message has an attachment. + + --047d7b33dd729737fe04d3bde348 + Content-Type: image/png; name="heart.png" + Content-Disposition: attachment; filename="heart.png" + Content-Transfer-Encoding: base64 + X-Attachment-Id: f_hc6mair60 + + iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAFoTx1HAAAAzUlEQVQoz32RWxXDIBBEr4NIQEIl + ICESkFAJkRAJSIgEpEQCEqYfu6QUkn7sCcyDGQiSACKSKCAkGwBJwhDwZQNMEiYAIBdQvk7rfaHf + AO8NBJwCxTGhtFgTHVNaNaJeWFu44AXEHzKCktc7zZ0vss+bMoHSiM2b9mQoX1eZCgGqnWskY3gi + XXAAxb8BqFiUgBNY7k49Tu/kV7UKPsefrjEOT9GmghYzrk9V03pjDGYKj3d0c06dKZkpTboRaD9o + B+1m2m81d2Az948xzgdjLaFe95e83AAAAABJRU5ErkJggg== + + --047d7b33dd729737fe04d3bde348-- + +Messages are grouped into multiple message payload parts, and should binary +attachments exist, they are encoded into text using, generally, ``base64`` or +``quoted-printable`` encodings. + +Earlier versions of this library would preserve the above text verbatim in the +database, but neither of the above encodings are very efficient methods of +storing binary data, and databases aren't really ideal for storing large +chunks of binary data anyway. + +Modern versions of this library (>=2.1) will walk through the original message, +write ``models.MessageAttachment`` records for each non-text attachment, +and alter the message body removing the original payload component, but writing +a custom header providing the library enough information to re-build the +message in the event that one needs a python ``email.message.Message`` object. + +.. code-block:: http + + MIME-Version: 1.0 + Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 12:07:07 -0800 (PST) + X-Originating-IP: [24.22.122.177] + Date: Sun, 20 Jan 2013 12:07:07 -0800 + Delivered-To: test@adamcoddington.net + Message-ID: + Subject: Message With Attachment + From: Adam Coddington + To: Adam Coddington + Content-Type: multipart/mixed; boundary=047d7b33dd729737fe04d3bde348 + + --047d7b33dd729737fe04d3bde348 + Content-Type: text/plain; charset=UTF-8 + + This message has an attachment. + + --047d7b33dd729737fe04d3bde348 + X-Django-Mailbox-Interpolate-Attachment: 1308 + + + --047d7b33dd729737fe04d3bde348-- + +The above payload is what would continue to be stored in the database. +Although in this constructed example, this reduces the message's size only +marginally, in most instances, attached files are much larger than the +attachment shown here. + +Although the attachment is no longer preserved in the message body above, +and only the ``X-Django-Mailbox-Interpolate-Attachment: 1308`` header remains +in the place of the original attachment, the attachment was stored in a +``django_mailbox.MesageAttachment`` record: + +.. list-table:: + :header-rows: 1 + + * - Field + - Value + - Description + * - Primary Key + - ``1308`` + - Uniquely generated for each attachment. + * - Headers + - ``Content-Type: image/png; name="heart.png" + Content-Disposition: attachment; filename="heart.png" + Content-Transfer-Encoding: base64 + X-Attachment-Id: f_hc6mair60`` + - Raw headers from the actual message's payload part. + * - File + - ``(binary file object)`` + - References a stored-on-disk binary file corresponding with this + attachment. + +And were one to run the ``django_mailbox.Message`` instance's +``get_email_object`` method, the following message will be returned: + +.. code-block:: http + + MIME-Version: 1.0 + Received: by 10.221.0.211 with HTTP; Sun, 20 Jan 2013 12:07:07 -0800 (PST) + X-Originating-IP: [24.22.122.177] + Date: Sun, 20 Jan 2013 12:07:07 -0800 + Delivered-To: test@adamcoddington.net + Message-ID: + Subject: Message With Attachment + From: Adam Coddington + To: Adam Coddington + Content-Type: multipart/mixed; boundary=047d7b33dd729737fe04d3bde348 + + --047d7b33dd729737fe04d3bde348 + Content-Type: text/plain; charset=UTF-8 + + This message has an attachment. + + --047d7b33dd729737fe04d3bde348 + Content-Type: image/png; name="heart.png" + Content-Disposition: attachment; filename="heart.png" + X-Attachment-Id: f_hc6mair60 + Content-Transfer-Encoding: base64 + + iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAFoTx1HAAAAzUlEQVQoz32RWxXDIBBEr4NIQEIl + ICESkFAJkRAJSIgEpEQCEqYfu6QUkn7sCcyDGQiSACKSKCAkGwBJwhDwZQNMEiYAIBdQvk7rfaHf + AO8NBJwCxTGhtFgTHVNaNaJeWFu44AXEHzKCktc7zZ0vss+bMoHSiM2b9mQoX1eZCgGqnWskY3gi + XXAAxb8BqFiUgBNY7k49Tu/kV7UKPsefrjEOT9GmghYzrk9V03pjDGYKj3d0c06dKZkpTboRaD9o + B+1m2m81d2Az948xzgdjLaFe95e83AAAAABJRU5ErkJggg== + + --047d7b33dd729737fe04d3bde348-- + +.. note:: + + Note that although the above is functionally identical to the originally + received message, there were changes in the order of headers in rehydrated + message components, and whitespace changes are also possible (but not + shown above). diff --git a/docs/topics/polling.rst b/docs/topics/polling.rst index eb463dd..7b52d47 100644 --- a/docs/topics/polling.rst +++ b/docs/topics/polling.rst @@ -3,7 +3,8 @@ Getting incoming mail ===================== If you are utilizing one of the polling methods above, -you will need to periodically poll the mailbox for messages using one of the below methods. +you will need to periodically poll the mailbox for messages using one of the +below methods. If you are receiving mail directly from a mailserver via a pipe -- using the ``processincomingmessage`` management command -- you need not concern yourself with this section. @@ -23,8 +24,9 @@ and select the 'Get new mail' option. Using a cron job ---------------- -You can easily consume incoming mail by running the management command named ``getmail`` -(optionally with an argument of the name of the mailbox you'd like to get the mail for).:: +You can easily consume incoming mail by running the management command named +``getmail`` (optionally with an argument of the name of the mailbox you'd like +to get the mail for).:: python manage.py getmail @@ -34,15 +36,18 @@ You can easily consume incoming mail by running the management command named ``g Receiving mail directly from Exim4 or Postfix via a pipe -------------------------------------------------------- -Django Mailbox's ``processincomingmessage`` management command accepts, via ``stdin``, incoming messages. -You can configure Postfix or Exim4 to pipe incoming mail to this management command -to import messages directly without polling. +Django Mailbox's ``processincomingmessage`` management command accepts, via +``stdin``, incoming messages. +You can configure Postfix or Exim4 to pipe incoming mail to this management +command to import messages directly without polling. You need not configure mailbox settings when piping-in messages, -mailbox entries will be automatically created matching the e-mail address to which incoming messages are sent, +mailbox entries will be automatically created matching the e-mail address to +which incoming messages are sent, but if you would like to specify the mailbox name, you may provide a single argument to the ``processincmingmessage`` command -specifying the name of the mailbox you would like it to use (and, if neccessary, create). +specifying the name of the mailbox you would like it to use +(and, if necessary, create). Receiving Mail from Exim4 ......................... @@ -57,11 +62,14 @@ start by adding a new router configuration to your Exim4 configuration like:: transport = send_to_django_mailbox local_parts = emailusernameone : emailusernametwo -Make sure that the e-mail addresses you would like handled by Django Mailbox are not handled by another router; +Make sure that the e-mail addresses you would like handled by Django Mailbox +are not handled by another router; you may need to disable some existing routers. -Change the contents of ``local_parts`` to match a colon-delimited list of usernames for which you would like to receive mail. -For example, if one of the e-mail addresses targeted at this machine is ``jane@example.com``, +Change the contents of ``local_parts`` to match a colon-delimited list of +usernames for which you would like to receive mail. +For example, if one of the e-mail addresses targeted at this machine is +``jane@example.com``, the contents of ``local_parts`` would be, simply ``jane``. Next, a new transport configuration to your Exim4 configuration:: @@ -74,18 +82,23 @@ Next, a new transport configuration to your Exim4 configuration:: return_path_add delivery_date_add -Like your router configuration, transport configuration should be altered to match your environment. -First, modify the ``command`` setting such that it points at the proper python executable -(if you're using a virtual environment, you'll want to direct that at the python executable in your virtual environment) +Like your router configuration, transport configuration should be altered to +match your environment. +First, modify the ``command`` setting such that it points at the proper +python executable +(if you're using a virtual environment, you'll want to direct that at the +python executable in your virtual environment) and project ``manage.py`` script. Additionally, you'll need to set ``user`` and ``group`` such that -they match a reasonable user and group (on Ubuntu, ``www-data`` suffices for both). +they match a reasonable user and group +(on Ubuntu, ``www-data`` suffices for both). Receiving mail from Postfix ........................... Although I have not personally tried using Postfix for this, Postfix is capable of delivering new mail to a script using ``pipe``. -Please consult the `Postfix documentation for pipe here `_. +Please consult the +`Postfix documentation for pipe here `_. You may want to consult the above Exim4 configuration for tips. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 18585b4..6023046 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -3,19 +3,58 @@ Settings ======== * ``DJANGO_MAILBOX_ADMIN_ENABLED`` - * Default: ``True`` - * Type: ``boolean`` - * Controls whether mailboxes appear in the Django Admin. -* ``DJANGO_MAILBOX_SKIPPED_EXTENSIONS`` - * Default: ``['.p7s']`` - * Type: ``list`` - * A list of extensions to skip when processing e-mail message attachments. -* ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES`` - * Default: ``False`` - * Type: ``boolean`` - * Controls whether or not we remove mimetypes not specified in ``DJANGO_MAILBOX_PRESERVED_MIMETYPES``. -* ``DJANGO_MAILBOX_ALLOWED_MIMETYPES`` - * Default ``['text/html', 'text/plain']`` - * Type: ``list`` - * A list of mimetypes that will remain and be stored in the message payload of the message object. Has no effect unless ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES`` is set to ``True``. + * Default: ``True`` + * Type: ``boolean`` + * Controls whether mailboxes appear in the Django Admin. + +* ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES`` + + * Default: ``False`` + * Type: ``boolean`` + * Controls whether or not we remove mimetypes not specified in + ``DJANGO_MAILBOX_PRESERVED_MIMETYPES`` from the message prior to storage. + +* ``DJANGO_MAILBOX_ALLOWED_MIMETYPES`` + + * Default ``['text/html', 'text/plain']`` + * Type: ``list`` + * Should ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES`` be ``True``, this is + a list of mimetypes that will not be stripped from the message prior + to processing attachments. + Has no effect unless ``DJANGO_MAILBOX_STRIP_UNALLOWED_MIMETYPES`` + is set to ``True``. + +* ``DJANGO_MAILBOX_TEXT_STORED_MIMETYPES`` + + * Default: ``['text/html', 'text/plain']`` + * Type: ``list`` + * A list of mimetypes that will remain stored in the text body of the + message in the database. See :doc:`message-storage`. + +* ``DJANGO_MAILBOX_ALTERED_MESSAGE_HEADER`` + + * Default: ``X-Django-Mailbox-Altered-Message`` + * Type: ``string`` + * Header to add to a message payload part in the event that the message + cannot be reproduced accurately. Possible values include: + + * ``Missing``: The message could not be reconstructed because the message + payload component (stored outside this database record) could not be + found. This will be followed by a semicolon (``;``) and a short, more + detailed description of which record was not found. + * ``Stripped`` The message could not be reconstructed because the message + payload component was intentionally stripped from the message body prior + to storage. This will be followed by a semicolon (``;``) and a short, + more detailed description of why this payload component was stripped. + +* ``DJANGO_MAILBOX_ATTACHMENT_INTERPOLATION_HEADER`` + + * Default: ``X-Django-Mailbox-Interpolate-Attachment`` + * Type: ``string`` + * Header to add to the temporary 'dehydrated' message body in lieu of + a non-text message payload component. The value of this header will be used + to 'rehydrate' the message into a proper e-mail object in the event of + a message instance's ``get_email_object`` method being called. Value of + this field is the primary key of the ``django_mailbox.MessageAttachment`` + instance currently storing this payload component's contents. diff --git a/docs/topics/signal.rst b/docs/topics/signal.rst index b778081..3c5e8e2 100644 --- a/docs/topics/signal.rst +++ b/docs/topics/signal.rst @@ -11,3 +11,11 @@ To subscribe to the incoming mail signal, following this lead:: def dance_jig(sender, message, **args): print "I just recieved a message titled %s from a mailbox named %s" % (message.subject, message.mailbox.name, ) +.. warning:: + + `As with all django signals `_, + this should be loaded either in an app's ``models.py`` + or somewhere else loaded early on. + If you do not load it early enough, the signal may be fired before your + signal handler's registration is processed! +