diff --git a/django_mailbox/models.py b/django_mailbox/models.py index 3c0a5c0..2f9dd13 100644 --- a/django_mailbox/models.py +++ b/django_mailbox/models.py @@ -299,6 +299,17 @@ class Mailbox(models.Model): # defined charset, if it can't, let's mash some things # inside the payload :-\ msg.get_payload(decode=True).decode(content_charset) + except LookupError: + logger.exception( + "Unknown encoding %s; interpreting as ascii!", + content_charset + ) + msg.set_payload( + msg.get_payload(decode=True).decode( + 'ascii', + 'ignore' + ) + ) except UnicodeDecodeError: msg.set_payload( msg.get_payload(decode=True).decode( diff --git a/django_mailbox/tests/messages/message_with_invalid_encoding.eml b/django_mailbox/tests/messages/message_with_invalid_encoding.eml new file mode 100644 index 0000000..fbe1f9f --- /dev/null +++ b/django_mailbox/tests/messages/message_with_invalid_encoding.eml @@ -0,0 +1,13 @@ +Reply-To: +From: "Refinance" +Subject: Apply For Loans @ 2% Per Annum +Date: Sun, 19 Oct 2014 11:48:49 +0100 +MIME-Version: 1.0 +Content-Type: text/plain; + charset="_iso-2022-jp$ESC" +Content-Transfer-Encoding: 7bit + +We offer loans to private individuals and corporate organizations at 2% interest rate. Interested serious applicants should apply via email with details of their requirements. + +Warm Regards, +Loan Team diff --git a/django_mailbox/tests/test_message_flattening.py b/django_mailbox/tests/test_message_flattening.py index 9574701..1399a34 100644 --- a/django_mailbox/tests/test_message_flattening.py +++ b/django_mailbox/tests/test_message_flattening.py @@ -85,3 +85,20 @@ class TestMessageFlattening(EmailMessageTestCase): actual_email_object, expected_email_object, ) + + def test_message_processing_unknown_encoding(self): + incoming_email_object = self._get_email_object( + 'message_with_invalid_encoding.eml', + ) + + msg = self.mailbox.process_incoming_message(incoming_email_object) + + expected_text = ( + "We offer loans to private individuals and corporate " + "organizations at 2% interest rate. Interested serious " + "applicants should apply via email with details of their " + "requirements.\n\nWarm Regards,\nLoan Team" + ) + actual_text = msg.text + + self.assertEqual(actual_text, expected_text) diff --git a/django_mailbox/utils.py b/django_mailbox/utils.py index ff8c6e0..9a802e1 100644 --- a/django_mailbox/utils.py +++ b/django_mailbox/utils.py @@ -52,21 +52,29 @@ def get_body_from_message(message, maintype, subtype): charset = part.get_content_charset() this_part = part.get_payload(decode=True) if charset: - this_part = this_part.decode(charset, 'replace') + try: + this_part = this_part.decode(charset, 'replace') + except LookupError: + this_part = this_part.decode('ascii', 'replace') + logger.warning( + 'Unknown encoding %s encountered while decoding ' + 'text payload. Interpreting as ASCII with ' + 'replacement, but some data may not be ' + 'represented as the sender intended.', + charset + ) + except ValueError: + this_part = this_part.decode('ascii', 'replace') + logger.warning( + 'Error encountered while decoding text ' + 'payload from an incorrectly-constructed ' + 'e-mail; payload was converted to ASCII with ' + 'replacement, but some data may not be ' + 'represented as the sender intended.' + ) + else: + this_part = this_part.decode('ascii', 'replace') - try: - body += this_part - except ValueError: - # Since it did not declare a charset, and we - # *should* be 7-bit clean right now, let's assume it - # is ASCII. - body += this_part.decode('ascii', 'replace') - logger.warning( - 'Error encountered while decoding text ' - 'payload from an incorrectly-constructed ' - 'e-mail; payload was converted to ASCII with ' - 'replacement, but some data may not be ' - 'represented as the sender intended.' - ) + body += this_part return body