From 49d3b02b9c073e7268005228d67ccd97f5ebcd24 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 22 Apr 2024 20:05:53 +0200 Subject: [PATCH] also handle other text encodings than UTF8 --- core/mail/protocol.py | 16 +++++++--- core/mail/tests/v2/test_mails.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 405b790..5d732af 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -122,7 +122,7 @@ def parse_email_body(raw, log=None): # skip any text/plain (txt) attachments if ctype == 'text/plain' and 'attachment' not in cdispo: - segment = part.get_payload(decode=True).decode('utf-8') + segment = part.get_payload() if not segment: continue segment = unescape_and_decode_quoted_printable(segment) @@ -146,11 +146,11 @@ def parse_email_body(raw, log=None): log.info("Attachment", ctype, cdispo) else: if parsed.get_content_type() == 'text/plain': - body = parsed.get_payload(decode=True).decode('utf-8') + body = parsed.get_payload() elif parsed.get_content_type() == 'text/html': from bs4 import BeautifulSoup import re - body = parsed.get_payload(decode=True).decode('utf-8') + body = parsed.get_payload() soup = BeautifulSoup(body, 'html.parser') body = re.sub(r'([\r\n]+.?)*[\r\n]', r'\n', soup.get_text()).strip('\n') else: @@ -192,7 +192,7 @@ def receive_email(envelope, log=None): email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw=envelope.content.decode('utf-8'), event=target_event, + in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) @@ -243,7 +243,9 @@ class LMTPHandler: log.info('Message for %s' % envelope.rcpt_tos) log.info('Message data:\n') + content = None try: + content = envelope.content email, new, reply = await sync_to_async(receive_email)(envelope, log) log.info(f"Created email {email.id}") systemevent = await sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id) @@ -260,5 +262,9 @@ class LMTPHandler: return '250 Message accepted for delivery' except Exception as e: - log.error(type(e), e) + import uuid + random_filename = 'mail-' + str(uuid.uuid4()) + with open(random_filename, 'wb') as f: + f.write(content) + log.error(type(e), e, f"Saved email to {random_filename}") return '451 Internal server error' diff --git a/core/mail/tests/v2/test_mails.py b/core/mail/tests/v2/test_mails.py index b045b91..d5d1c0c 100644 --- a/core/mail/tests/v2/test_mails.py +++ b/core/mail/tests/v2/test_mails.py @@ -828,3 +828,55 @@ thank you =?utf-8?Q?=F0=9F=98=8A?=''' # thank you 😊 states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) self.assertEqual(1, len(states)) self.assertEqual('pending_new', states[0].state) + + def test_mail_non_utf8(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + + aiosmtplib.send = make_mocked_coro() + + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + + envelope.content = b'''Subject: test +From: test1@test +To: test2@test +Message-ID: <1@test> +Content-Type: text/html; charset=iso-8859-1 + +hello \xe4\xf6\xfc''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual('250 Message accepted for delivery', result) + self.assertEqual(2, len(Email.objects.all())) + self.assertEqual(1, len(IssueThread.objects.all())) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('hello äöü', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('test1@test', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state)