Interpret unknown encodings as ASCII. Fixes #34.

django_mailbox/models.py

@@ -299,6 +299,17 @@ class Mailbox(models.Model):
                 # defined charset, if it can't, let's mash some things
                 # inside the payload :-\
                 msg.get_payload(decode=True).decode(content_charset)
+            except LookupError:
+                logger.exception(
+                    "Unknown encoding %s; interpreting as ascii!",
+                    content_charset
+                )
+                msg.set_payload(
+                    msg.get_payload(decode=True).decode(
+                        'ascii',
+                        'ignore'
+                    )
+                )
             except UnicodeDecodeError:
                 msg.set_payload(
                     msg.get_payload(decode=True).decode(

django_mailbox/tests/messages/message_with_invalid_encoding.eml

@@ -0,0 +1,13 @@
+Reply-To: <private00@qq.com>
+From: "Refinance"<finance@khas.edu.tr>
+Subject: Apply For Loans @ 2% Per Annum
+Date: Sun, 19 Oct 2014 11:48:49 +0100
+MIME-Version: 1.0
+Content-Type: text/plain;
+	charset="_iso-2022-jp$ESC"
+Content-Transfer-Encoding: 7bit
+
+We offer loans to private individuals and corporate organizations at 2% interest rate. Interested serious applicants should apply via email with details of their requirements.
+
+Warm Regards,
+Loan Team

django_mailbox/tests/test_message_flattening.py

@@ -85,3 +85,20 @@ class TestMessageFlattening(EmailMessageTestCase):
             actual_email_object,
             expected_email_object,
         )
+
+    def test_message_processing_unknown_encoding(self):
+        incoming_email_object = self._get_email_object(
+            'message_with_invalid_encoding.eml',
+        )
+
+        msg = self.mailbox.process_incoming_message(incoming_email_object)
+
+        expected_text = (
+            "We offer loans to private individuals and corporate "
+            "organizations at 2% interest rate. Interested serious "
+            "applicants should apply via email with details of their "
+            "requirements.\n\nWarm Regards,\nLoan Team"
+        )
+        actual_text = msg.text
+
+        self.assertEqual(actual_text, expected_text)

django_mailbox/utils.py

@@ -52,21 +52,29 @@ def get_body_from_message(message, maintype, subtype):
             charset = part.get_content_charset()
             this_part = part.get_payload(decode=True)
             if charset:
-                this_part = this_part.decode(charset, 'replace')
+                try:
+                    this_part = this_part.decode(charset, 'replace')
+                except LookupError:
+                    this_part = this_part.decode('ascii', 'replace')
+                    logger.warning(
+                        'Unknown encoding %s encountered while decoding '
+                        'text payload.  Interpreting as ASCII with '
+                        'replacement, but some data may not be '
+                        'represented as the sender intended.',
+                        charset
+                    )
+                except ValueError:
+                    this_part = this_part.decode('ascii', 'replace')
+                    logger.warning(
+                        'Error encountered while decoding text '
+                        'payload from an incorrectly-constructed '
+                        'e-mail; payload was converted to ASCII with '
+                        'replacement, but some data may not be '
+                        'represented as the sender intended.'
+                    )
+            else:
+                this_part = this_part.decode('ascii', 'replace')
 
-            try:
+            body += this_part
-                body += this_part
-            except ValueError:
-                # Since it did not declare a charset, and we
-                # *should* be 7-bit clean right now, let's assume it
-                # is ASCII.
-                body += this_part.decode('ascii', 'replace')
-                logger.warning(
-                    'Error encountered while decoding text '
-                    'payload from an incorrectly-constructed '
-                    'e-mail; payload was converted to ASCII with '
-                    'replacement, but some data may not be '
-                    'represented as the sender intended.'
-                )
 
     return body