diff --git a/core/mail/protocol.py b/core/mail/protocol.py index b199f36..e6d6ba1 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -82,7 +82,13 @@ async def send_smtp(message, log): await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False) -def find_active_issue_thread(in_reply_to, subject=None): +def find_active_issue_thread(in_reply_to, address, subject): + from re import match + uuid_match = match(r'^ticket\+([a-f0-9-]{36})@', address) + if uuid_match: + issue = IssueThread.objects.filter(uuid=uuid_match.group(1)) + if issue.exists(): + return issue.first(), False reply_to = Email.objects.filter(reference=in_reply_to) if reply_to.exists(): return reply_to.first().issue_thread, False @@ -117,6 +123,8 @@ def parse_email_body(raw, log=None): # skip any text/plain (txt) attachments if ctype == 'text/plain' and 'attachment' not in cdispo: segment = part.get_payload(decode=True).decode('utf-8') + if not segment: + continue segment = unescape_and_decode_quoted_printable(segment) segment = unescape_and_decode_base64(segment) log.debug(segment) @@ -138,6 +146,9 @@ def parse_email_body(raw, log=None): log.info("Attachment", ctype, cdispo) else: body = parsed.get_payload(decode=True).decode('utf-8') + body = unescape_and_decode_quoted_printable(body) + body = unescape_and_decode_base64(body) + log.debug(body) return parsed, body, attachments @@ -161,17 +172,16 @@ def receive_email(envelope, log=None): recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower() sender = envelope.mail_from if envelope.mail_from else header_from subject = parsed.get('Subject') + if not subject: + subject = "No subject" subject = unescape_and_decode_quoted_printable(subject) subject = unescape_and_decode_base64(subject) target_event = find_target_event(recipient) - active_issue_thread, new = find_active_issue_thread(header_in_reply_to, subject) - body_decoded = body - body_decoded = unescape_and_decode_quoted_printable(body_decoded) - body_decoded = unescape_and_decode_base64(body_decoded) + active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject) email = Email.objects.create( - sender=sender, recipient=recipient, body=body_decoded, subject=subject, reference=header_message_id, + sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, in_reply_to=header_in_reply_to, raw=envelope.content.decode('utf-8'), event=target_event, issue_thread=active_issue_thread) for attachment in attachments: @@ -182,11 +192,22 @@ def receive_email(envelope, log=None): if new: # auto reply if new issue references = collect_references(active_issue_thread) + if not sender.startswith('noreply'): + subject = f"Re: {subject} [#{active_issue_thread.short_uuid()}]" + body = '''Your request (#{}) has been received and will be reviewed by our lost&found angels. - reply_email = Email.objects.create( - sender=recipient, recipient=sender, body="Thank you for your message.", subject="Message received", - in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) - reply = make_reply(reply_email, references) +We are reviewing incoming requests during the event and teardown. Immediately after the event, expect a delay as the \ +workload is high. We will not forget about your request and get back in touch once we have updated information on your \ +request. Requests for devices, wallets, credit cards or similar items will be handled with priority. + +If you happen to find your lost item or just want to add additional information, please reply to this email. Please \ +do not create a new request. + +Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid()) + reply_email = Email.objects.create( + sender=recipient, recipient=sender, body=body, subject=subject, + in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) + reply = make_reply(reply_email, references) else: # change state if not new if active_issue_thread.state != 'pending_new': @@ -223,7 +244,7 @@ class LMTPHandler: "message": "email received"} ) log.info(f"Sent message to frontend") - if new: + if new and reply: await send_smtp(reply, log) log.info("Sent auto reply") diff --git a/core/mail/tests/v2/test_mails.py b/core/mail/tests/v2/test_mails.py index a66eb66..75ca740 100644 --- a/core/mail/tests/v2/test_mails.py +++ b/core/mail/tests/v2/test_mails.py @@ -12,6 +12,19 @@ from mail.models import Email, EventAddress, EmailAttachment from mail.protocol import LMTPHandler from tickets.models import IssueThread, StateChange +expected_auto_reply_subject = 'Re: {} [#{}]' + +expected_auto_reply = '''Your request (#{}) has been received and will be reviewed by our lost&found angels. + +We are reviewing incoming requests during the event and teardown. Immediately after the event, expect a delay as the \ +workload is high. We will not forget about your request and get back in touch once we have updated information on your \ +request. Requests for devices, wallets, credit cards or similar items will be handled with priority. + +If you happen to find your lost item or just want to add additional information, please reply to this email. Please \ +do not create a new request. + +Your c3lf (Cloakroom + Lost&Found) Team''' + def make_mocked_coro(return_value=mock.sentinel, raise_exception=mock.sentinel): async def mock_coro(*args, **kwargs): @@ -93,10 +106,12 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) self.assertEqual('<1@test>', Email.objects.all()[0].reference) self.assertEqual(None, Email.objects.all()[0].in_reply_to) - self.assertEqual('Message received', Email.objects.all()[1].subject) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) self.assertEqual('test2@test', Email.objects.all()[1].sender) self.assertEqual('test1@test', Email.objects.all()[1].recipient) - self.assertEqual('Thank you for your message.', Email.objects.all()[1].body) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) self.assertTrue(Email.objects.all()[1].reference.startswith("<")) self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) @@ -332,13 +347,15 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual(event, Email.objects.all()[0].event) self.assertEqual(event, Email.objects.all()[1].event) self.assertEqual('test', Email.objects.all()[0].subject) - self.assertEqual('Message received', Email.objects.all()[1].subject) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) self.assertEqual('test1@test', Email.objects.all()[0].sender) self.assertEqual('test_event@localhost', Email.objects.all()[0].recipient) self.assertEqual('test_event@localhost', Email.objects.all()[1].sender) self.assertEqual('test1@test', Email.objects.all()[1].recipient) self.assertEqual('test', Email.objects.all()[0].body) - self.assertEqual('Thank you for your message.', Email.objects.all()[1].body) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) self.assertEqual('<1@test>', Email.objects.all()[0].reference) @@ -404,10 +421,12 @@ test2 self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) self.assertEqual('<1@test>', Email.objects.all()[0].reference) self.assertEqual(None, Email.objects.all()[0].in_reply_to) - self.assertEqual('Message received', Email.objects.all()[1].subject) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) self.assertEqual('test2@test', Email.objects.all()[1].sender) self.assertEqual('test1@test', Email.objects.all()[1].recipient) - self.assertEqual('Thank you for your message.', Email.objects.all()[1].body) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) self.assertTrue(Email.objects.all()[1].reference.startswith("<")) self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) @@ -470,10 +489,12 @@ dGVzdGltYWdl self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) self.assertEqual('<1@test>', Email.objects.all()[0].reference) self.assertEqual(None, Email.objects.all()[0].in_reply_to) - self.assertEqual('Message received', Email.objects.all()[1].subject) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) self.assertEqual('test2@test', Email.objects.all()[1].sender) self.assertEqual('test1@test', Email.objects.all()[1].recipient) - self.assertEqual('Thank you for your message.', Email.objects.all()[1].body) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) self.assertTrue(Email.objects.all()[1].reference.startswith("<")) self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) @@ -491,3 +512,169 @@ dGVzdGltYWdl file_content = EmailAttachment.objects.all()[0].file.read() self.assertEqual(b'testimage', file_content) + def test_mail_noreply(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'noreply@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'Subject: test\nFrom: noreply@test\nTo: test2@test\nMessage-ID: <1@test>\n\ntest' + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 1) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_not_called() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('noreply@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('test', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) + + def test_mail_empty_subject(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + import logging + logging.disable(logging.CRITICAL) + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'From: noreply@test\nTo: test2@test\nMessage-ID: <1@test>\n\ntest' + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + logging.disable(logging.NOTSET) + self.assertEqual('250 Message accepted for delivery', result) + self.assertEqual(2, len(Email.objects.all())) + self.assertEqual(1, len(IssueThread.objects.all())) + aiosmtplib.send.assert_called_once() + self.assertEqual('No subject', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('test', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('No subject', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('test1@test', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('No subject', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) + + def test_mail_empty_body(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + import logging + logging.disable(logging.CRITICAL) + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = '' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'Subject: test\nFrom: \nTo: test2@test\nMessage-ID: <1@test>\n\n' + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + logging.disable(logging.NOTSET) + self.assertEqual('250 Message accepted for delivery', result) + self.assertEqual(2, len(Email.objects.all())) + self.assertEqual(1, len(IssueThread.objects.all())) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) + + def test_mail_plus_issue_thread(self): + issue_thread = IssueThread.objects.create( + name="test", + ) + mail1 = Email.objects.create( + subject='test subject', + body='test', + sender='test1@test', + recipient='test2@test', + issue_thread=issue_thread, + ) + mail1_reply = Email.objects.create( + subject='Message received', + body='Thank you for your message.', + sender='test2@test', + recipient='test1@test', + in_reply_to=mail1.reference, + issue_thread=issue_thread, + ) + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + import logging + logging.disable(logging.CRITICAL) + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = '' + envelope.rcpt_tos = ['ticket+{}@test'.format(issue_thread.uuid)] + envelope.content = (f'Subject: foo\nFrom: \nTo: ticket+{issue_thread.uuid}@test\n' + f'Message-ID: <3@test>\n\nbar'.encode('utf-8')) + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + logging.disable(logging.NOTSET) + self.assertEqual('250 Message accepted for delivery', result) + self.assertEqual(3, len(Email.objects.all())) + self.assertEqual(3, len(Email.objects.filter(issue_thread=issue_thread))) + self.assertEqual(1, len(IssueThread.objects.all())) + aiosmtplib.send.assert_not_called() + self.assertEqual(Email.objects.all()[2].subject, 'foo') + self.assertEqual(Email.objects.all()[2].sender, '') + self.assertEqual(Email.objects.all()[2].recipient, 'ticket+{}@test'.format(issue_thread.uuid)) + self.assertEqual(Email.objects.all()[2].body, 'bar') + self.assertEqual(Email.objects.all()[2].issue_thread, issue_thread) + self.assertEqual(Email.objects.all()[2].reference, '<3@test>') + self.assertEqual('test', IssueThread.objects.all()[0].name) diff --git a/core/tickets/api_v2.py b/core/tickets/api_v2.py index a4beaaa..8cadcb1 100644 --- a/core/tickets/api_v2.py +++ b/core/tickets/api_v2.py @@ -22,8 +22,8 @@ class IssueSerializer(serializers.ModelSerializer): class Meta: model = IssueThread - fields = ('id', 'timeline', 'name', 'state', 'assigned_to', 'last_activity') - read_only_fields = ('id', 'timeline', 'last_activity') + fields = ('id', 'timeline', 'name', 'state', 'assigned_to', 'last_activity', 'uuid') + read_only_fields = ('id', 'timeline', 'last_activity', 'uuid') def to_internal_value(self, data): ret = super().to_internal_value(data) diff --git a/core/tickets/migrations/0006_issuethread_uuid.py b/core/tickets/migrations/0006_issuethread_uuid.py new file mode 100644 index 0000000..146809a --- /dev/null +++ b/core/tickets/migrations/0006_issuethread_uuid.py @@ -0,0 +1,32 @@ +# Generated by Django 4.2.7 on 2024-01-12 21:28 + +from django.db import migrations, models + +from tickets.models import IssueThread + + +class Migration(migrations.Migration): + + dependencies = [ + ('tickets', '0005_remove_issuethread_last_activity'), + ] + + def set_uuid(apps, schema_editor): + import uuid + for issue_thread in IssueThread.objects.all(): + issue_thread.uuid = str(uuid.uuid4()) + issue_thread.save() + + operations = [ + migrations.AddField( + model_name='issuethread', + name='uuid', + field=models.CharField(max_length=255, null=True), + ), + migrations.RunPython(set_uuid), + migrations.AlterField( + model_name='issuethread', + name='uuid', + field=models.CharField(max_length=255, unique=True, null=False, blank=False), + ), + ] diff --git a/core/tickets/models.py b/core/tickets/models.py index a721385..8616c92 100644 --- a/core/tickets/models.py +++ b/core/tickets/models.py @@ -2,7 +2,7 @@ from django.db import models from django_softdelete.models import SoftDeleteModel from inventory.models import Event -from django.db.models.signals import post_save +from django.db.models.signals import post_save, pre_save from django.dispatch import receiver STATE_CHOICES = ( @@ -25,10 +25,14 @@ STATE_CHOICES = ( class IssueThread(SoftDeleteModel): id = models.AutoField(primary_key=True) + uuid = models.CharField(max_length=255, unique=True, null=False, blank=False) name = models.CharField(max_length=255) assigned_to = models.CharField(max_length=255, null=True) manually_created = models.BooleanField(default=False) + def short_uuid(self): + return self.uuid[:8] + @property def state(self): try: @@ -49,6 +53,13 @@ class IssueThread(SoftDeleteModel): ] +@receiver(pre_save, sender=IssueThread) +def set_uuid(sender, instance, **kwargs): + import uuid + if instance.uuid is None or instance.uuid == '': + instance.uuid = str(uuid.uuid4()) + + @receiver(post_save, sender=IssueThread) def create_issue_thread(sender, instance, created, **kwargs): if created: diff --git a/core/tickets/tests/v2/test_tickets.py b/core/tickets/tests/v2/test_tickets.py index 01b598f8..09cc298 100644 --- a/core/tickets/tests/v2/test_tickets.py +++ b/core/tickets/tests/v2/test_tickets.py @@ -51,7 +51,10 @@ class IssueApiTest(TestCase): comment="test", timestamp=now + timedelta(seconds=3), ) - + self.assertEqual('pending_new', issue.state) + self.assertEqual('test issue', issue.name) + self.assertEqual(None, issue.assigned_to) + self.assertEqual(36, len(issue.uuid)) response = self.client.get('/api/2/tickets/') self.assertEqual(response.status_code, 200) self.assertEqual(len(response.json()), 1) @@ -59,6 +62,7 @@ class IssueApiTest(TestCase): self.assertEqual(response.json()[0]['name'], "test issue") self.assertEqual(response.json()[0]['state'], "pending_new") self.assertEqual(response.json()[0]['assigned_to'], None) + self.assertEqual(response.json()[0]['uuid'], issue.uuid) self.assertEqual(response.json()[0]['last_activity'], comment.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) self.assertEqual(len(response.json()[0]['timeline']), 4) self.assertEqual(response.json()[0]['timeline'][0]['type'], 'state') @@ -137,12 +141,11 @@ class IssueApiTest(TestCase): self.assertEqual('test', response.json()[1]['timeline'][1]['subject']) self.assertEqual('test', response.json()[1]['timeline'][1]['body']) self.assertEqual(mail1.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), - response.json()[1]['timeline'][1]['timestamp']) + response.json()[1]['timeline'][1]['timestamp']) self.assertEqual('pending_new', response.json()[2]['timeline'][0]['state']) self.assertEqual('test', response.json()[2]['timeline'][1]['comment']) self.assertEqual(comment.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), - response.json()[2]['timeline'][1]['timestamp']) - + response.json()[2]['timeline'][1]['timestamp']) def test_manual_creation(self): response = self.client.post('/api/2/tickets/manual/',