also handle other text encodings than UTF8

core/mail/protocol.py

@@ -122,7 +122,7 @@ def parse_email_body(raw, log=None):
 
             # skip any text/plain (txt) attachments
             if ctype == 'text/plain' and 'attachment' not in cdispo:
-                segment = part.get_payload(decode=True).decode('utf-8')
+                segment = part.get_payload()
                 if not segment:
                     continue
                 segment = unescape_and_decode_quoted_printable(segment)
@@ -146,11 +146,11 @@ def parse_email_body(raw, log=None):
                 log.info("Attachment", ctype, cdispo)
     else:
         if parsed.get_content_type() == 'text/plain':
-            body = parsed.get_payload(decode=True).decode('utf-8')
+            body = parsed.get_payload()
         elif parsed.get_content_type() == 'text/html':
             from bs4 import BeautifulSoup
             import re
-            body = parsed.get_payload(decode=True).decode('utf-8')
+            body = parsed.get_payload()
             soup = BeautifulSoup(body, 'html.parser')
             body = re.sub(r'([\r\n]+.?)*[\r\n]', r'\n', soup.get_text()).strip('\n')
         else:
@@ -192,7 +192,7 @@ def receive_email(envelope, log=None):
 
     email = Email.objects.create(
         sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id,
-        in_reply_to=header_in_reply_to, raw=envelope.content.decode('utf-8'), event=target_event,
+        in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event,
         issue_thread=active_issue_thread)
     for attachment in attachments:
         email.attachments.add(attachment)
@@ -243,7 +243,9 @@ class LMTPHandler:
         log.info('Message for %s' % envelope.rcpt_tos)
         log.info('Message data:\n')
 
+        content = None
         try:
+            content = envelope.content
             email, new, reply = await sync_to_async(receive_email)(envelope, log)
             log.info(f"Created email {email.id}")
             systemevent = await sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id)
@@ -260,5 +262,9 @@ class LMTPHandler:
 
             return '250 Message accepted for delivery'
         except Exception as e:
-            log.error(type(e), e)
+            import uuid
+            random_filename = 'mail-' + str(uuid.uuid4())
+            with open(random_filename, 'wb') as f:
+                f.write(content)
+            log.error(type(e), e, f"Saved email to {random_filename}")
             return '451 Internal server error'

core/mail/tests/v2/test_mails.py

@@ -828,3 +828,55 @@ thank you =?utf-8?Q?=F0=9F=98=8A?='''  # thank you 😊
         states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0])
         self.assertEqual(1, len(states))
         self.assertEqual('pending_new', states[0].state)
+
+    def test_mail_non_utf8(self):
+        from aiosmtpd.smtp import Envelope
+        from asgiref.sync import async_to_sync
+        import aiosmtplib
+
+        aiosmtplib.send = make_mocked_coro()
+
+        handler = LMTPHandler()
+        server = mock.Mock()
+        session = mock.Mock()
+        envelope = Envelope()
+
+        envelope.mail_from = 'test1@test'
+        envelope.rcpt_tos = ['test2@test']
+
+        envelope.content = b'''Subject: test
+From: test1@test
+To: test2@test
+Message-ID: <1@test>
+Content-Type: text/html; charset=iso-8859-1
+
+hello \xe4\xf6\xfc'''
+
+        result = async_to_sync(handler.handle_DATA)(server, session, envelope)
+        self.assertEqual('250 Message accepted for delivery', result)
+        self.assertEqual(2, len(Email.objects.all()))
+        self.assertEqual(1, len(IssueThread.objects.all()))
+        aiosmtplib.send.assert_called_once()
+        self.assertEqual('test', Email.objects.all()[0].subject)
+        self.assertEqual('test1@test', Email.objects.all()[0].sender)
+        self.assertEqual('test2@test', Email.objects.all()[0].recipient)
+        self.assertEqual('hello äöü', Email.objects.all()[0].body)
+        self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread)
+        self.assertEqual('<1@test>', Email.objects.all()[0].reference)
+        self.assertEqual(None, Email.objects.all()[0].in_reply_to)
+        self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()),
+                         Email.objects.all()[1].subject)
+        self.assertEqual('test2@test', Email.objects.all()[1].sender)
+        self.assertEqual('test1@test', Email.objects.all()[1].recipient)
+        self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()),
+                         Email.objects.all()[1].body)
+        self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread)
+        self.assertTrue(Email.objects.all()[1].reference.startswith("<"))
+        self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>"))
+        self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to)
+        self.assertEqual('test', IssueThread.objects.all()[0].name)
+        self.assertEqual('pending_new', IssueThread.objects.all()[0].state)
+        self.assertEqual(None, IssueThread.objects.all()[0].assigned_to)
+        states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0])
+        self.assertEqual(1, len(states))
+        self.assertEqual('pending_new', states[0].state)