fix database problem with 4 byte utf-8

core/core/settings.py

@@ -133,6 +133,9 @@ else:
             'NAME': os.getenv('DB_NAME', 'system3'),
             'USER': os.getenv('DB_USER', 'system3'),
             'PASSWORD': os.getenv('DB_PASSWORD', 'system3'),
+            'OPTIONS': {
+                'charset': 'utf8mb4'
+            }
         }
     }
 

core/mail/protocol.py

@@ -145,7 +145,17 @@ def parse_email_body(raw, log=None):
             else:
                 log.info("Attachment", ctype, cdispo)
     else:
-        body = parsed.get_payload(decode=True).decode('utf-8')
+        if parsed.get_content_type() == 'text/plain':
+            body = parsed.get_payload(decode=True).decode('utf-8')
+        elif parsed.get_content_type() == 'text/html':
+            from bs4 import BeautifulSoup
+            import re
+            body = parsed.get_payload(decode=True).decode('utf-8')
+            soup = BeautifulSoup(body, 'html.parser')
+            body = re.sub(r'([\r\n]+.?)*[\r\n]', r'\n', soup.get_text()).strip('\n')
+        else:
+            log.warning("Unknown content type", parsed.get_content_type())
+            body = "Unknown content type"
         body = unescape_and_decode_quoted_printable(body)
         body = unescape_and_decode_base64(body)
         log.debug(body)
@@ -250,5 +260,5 @@ class LMTPHandler:
 
             return '250 Message accepted for delivery'
         except Exception as e:
-            log.error(e)
+            log.error(type(e), e)
             return '451 Internal server error'

core/mail/tests/v2/test_mails.py

@@ -385,6 +385,63 @@ class LMTPHandlerTestCase(TestCase):  # TODO replace with less hacky test
         self.assertEqual(1, len(states))
         self.assertEqual('pending_new', states[0].state)
 
+    def test_mail_html_body(self):
+        from aiosmtpd.smtp import Envelope
+        from asgiref.sync import async_to_sync
+        import aiosmtplib
+
+        aiosmtplib.send = make_mocked_coro()
+
+        handler = LMTPHandler()
+        server = mock.Mock()
+        session = mock.Mock()
+        envelope = Envelope()
+
+        envelope.mail_from = 'test1@test'
+        envelope.rcpt_tos = ['test2@test']
+
+        envelope.content = b'''Subject: test
+From: test1@test
+To: test2@test
+Message-ID: <1@test>
+Content-Type: text/html; charset=utf-8
+
+<div>
+<div>
+<p>test</p>
+</div>
+</div>'''
+
+        result = async_to_sync(handler.handle_DATA)(server, session, envelope)
+        self.assertEqual(result, '250 Message accepted for delivery')
+        self.assertEqual(len(Email.objects.all()), 2)
+        self.assertEqual(len(IssueThread.objects.all()), 1)
+        self.assertEqual(len(EmailAttachment.objects.all()), 0)
+        aiosmtplib.send.assert_called_once()
+        self.assertEqual('test', Email.objects.all()[0].subject)
+        self.assertEqual('test1@test', Email.objects.all()[0].sender)
+        self.assertEqual('test2@test', Email.objects.all()[0].recipient)
+        self.assertEqual('test', Email.objects.all()[0].body)
+        self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread)
+        self.assertEqual('<1@test>', Email.objects.all()[0].reference)
+        self.assertEqual(None, Email.objects.all()[0].in_reply_to)
+        self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()),
+                         Email.objects.all()[1].subject)
+        self.assertEqual('test2@test', Email.objects.all()[1].sender)
+        self.assertEqual('test1@test', Email.objects.all()[1].recipient)
+        self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()),
+                         Email.objects.all()[1].body)
+        self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread)
+        self.assertTrue(Email.objects.all()[1].reference.startswith("<"))
+        self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>"))
+        self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to)
+        self.assertEqual('test', IssueThread.objects.all()[0].name)
+        self.assertEqual('pending_new', IssueThread.objects.all()[0].state)
+        self.assertEqual(None, IssueThread.objects.all()[0].assigned_to)
+        states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0])
+        self.assertEqual(1, len(states))
+        self.assertEqual('pending_new', states[0].state)
+
     def test_split_text_inline_image(self):
         from aiosmtpd.smtp import Envelope
         from asgiref.sync import async_to_sync
@@ -719,3 +776,55 @@ dGVzdGltYWdl
         self.assertEqual('test subject', IssueThread.objects.all()[0].name)
         self.assertEqual('pending_new', IssueThread.objects.all()[0].state)
         self.assertEqual(None, IssueThread.objects.all()[0].assigned_to)
+
+    def test_mail_4byte_unicode_emoji(self):
+        from aiosmtpd.smtp import Envelope
+        from asgiref.sync import async_to_sync
+        import aiosmtplib
+
+        aiosmtplib.send = make_mocked_coro()
+
+        handler = LMTPHandler()
+        server = mock.Mock()
+        session = mock.Mock()
+        envelope = Envelope()
+
+        envelope.mail_from = 'test1@test'
+        envelope.rcpt_tos = ['test2@test']
+
+        envelope.content = b'''Subject: test
+From: test1@test
+To: test2@test
+Message-ID: <1@test>
+Content-Type: text/html; charset=utf-8
+
+thank you =?utf-8?Q?=F0=9F=98=8A?='''  # thank you 😊
+
+        result = async_to_sync(handler.handle_DATA)(server, session, envelope)
+        self.assertEqual('250 Message accepted for delivery', result)
+        self.assertEqual(2, len(Email.objects.all()))
+        self.assertEqual(1, len(IssueThread.objects.all()))
+        aiosmtplib.send.assert_called_once()
+        self.assertEqual('test', Email.objects.all()[0].subject)
+        self.assertEqual('test1@test', Email.objects.all()[0].sender)
+        self.assertEqual('test2@test', Email.objects.all()[0].recipient)
+        self.assertEqual('thank you 😊', Email.objects.all()[0].body)
+        self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread)
+        self.assertEqual('<1@test>', Email.objects.all()[0].reference)
+        self.assertEqual(None, Email.objects.all()[0].in_reply_to)
+        self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()),
+                         Email.objects.all()[1].subject)
+        self.assertEqual('test2@test', Email.objects.all()[1].sender)
+        self.assertEqual('test1@test', Email.objects.all()[1].recipient)
+        self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()),
+                         Email.objects.all()[1].body)
+        self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread)
+        self.assertTrue(Email.objects.all()[1].reference.startswith("<"))
+        self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>"))
+        self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to)
+        self.assertEqual('test', IssueThread.objects.all()[0].name)
+        self.assertEqual('pending_new', IssueThread.objects.all()[0].state)
+        self.assertEqual(None, IssueThread.objects.all()[0].assigned_to)
+        states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0])
+        self.assertEqual(1, len(states))
+        self.assertEqual('pending_new', states[0].state)

core/requirements.dev.txt

@@ -7,6 +7,8 @@ atpublic==4.0
 attrs==23.1.0
 autobahn==23.6.2
 Automat==22.10.0
+beautifulsoup4==4.12.2
+bs4==0.0.1
 certifi==2023.11.17
 cffi==1.16.0
 channels==4.0.0
@@ -53,6 +55,7 @@ service-identity==23.1.0
 setproctitle==1.3.3
 six==1.16.0
 sniffio==1.3.0
+soupsieve==2.5
 sqlparse==0.4.4
 Twisted==23.10.0
 txaio==23.1.1

core/requirements.prod.txt

@@ -2,6 +2,8 @@ aiosmtpd==1.4.4.post2
 aiosmtplib==3.0.1
 asgiref==3.7.2
 attrs==23.1.0
+beautifulsoup4==4.12.2
+bs4==0.0.1
 certifi==2023.11.17
 channels==4.0.0
 channels-redis==4.1.0
@@ -28,6 +30,7 @@ requests==2.31.0
 sdnotify==0.3.2
 setproctitle==1.3.3
 sniffio==1.3.0
+soupsieve==2.5
 sqlparse==0.4.4
 typing_extensions==4.8.0
 uritemplate==4.1.1