wip

core/mail/admin.py

@@ -0,0 +1,17 @@
+from django.contrib import admin
+
+from mail.models import Email, EventAddress
+
+
+class EmailAdmin(admin.ModelAdmin):
+    pass
+
+
+admin.site.register(Email, EmailAdmin)
+
+
+class EventAddressAdmin(admin.ModelAdmin):
+    pass
+
+
+admin.site.register(EventAddress, EventAddressAdmin)

core/mail/api_v2.py

@@ -0,0 +1,22 @@
+from rest_framework import routers, viewsets, serializers
+
+from mail.models import Email
+
+
+class EmailSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Email
+        fields = '__all__'
+
+
+class EmailViewSet(viewsets.ModelViewSet):
+    serializer_class = EmailSerializer
+    queryset = Email.objects.all()
+    permission_classes = []
+    authentication_classes = []
+
+
+router = routers.SimpleRouter()
+router.register(r'mails', EmailViewSet, basename='mails')
+
+urlpatterns = router.urls

core/mail/migrations/0001_initial.py

@@ -0,0 +1,46 @@
+# Generated by Django 4.2.7 on 2023-12-06 02:52
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('tickets', '0001_initial'),
+        ('inventory', '0002_container_deleted_at_container_is_deleted_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EventAddress',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('address', models.CharField(max_length=255)),
+                ('event', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='inventory.event')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Email',
+            fields=[
+                ('is_deleted', models.BooleanField(default=False)),
+                ('deleted_at', models.DateTimeField(blank=True, null=True)),
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('timestamp', models.DateTimeField(auto_now_add=True)),
+                ('body', models.TextField()),
+                ('subject', models.CharField(max_length=255)),
+                ('sender', models.CharField(max_length=255)),
+                ('recipient', models.CharField(max_length=255)),
+                ('reference', models.CharField(max_length=255, null=True, unique=True)),
+                ('in_reply_to', models.CharField(max_length=255, null=True)),
+                ('raw', models.TextField()),
+                ('event', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='inventory.event')),
+                ('issue_thread', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='emails', to='tickets.issuethread')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

core/mail/models.py

@@ -0,0 +1,34 @@
+import random
+
+from django.db import models
+from django_softdelete.models import SoftDeleteModel
+
+from core.settings import MAIL_DOMAIN
+from inventory.models import Event
+from tickets.models import IssueThread
+
+
+class Email(SoftDeleteModel):
+    id = models.AutoField(primary_key=True)
+    timestamp = models.DateTimeField(auto_now_add=True)
+    body = models.TextField()
+    subject = models.CharField(max_length=255)
+    sender = models.CharField(max_length=255)
+    recipient = models.CharField(max_length=255)
+    reference = models.CharField(max_length=255, null=True, unique=True)
+    in_reply_to = models.CharField(max_length=255, null=True)
+    raw = models.TextField()
+    issue_thread = models.ForeignKey(IssueThread, models.SET_NULL, null=True, related_name='emails')
+    event = models.ForeignKey(Event, models.SET_NULL, null=True)
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        if not self.reference:
+            self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>'
+            self.save()
+
+
+class EventAddress(models.Model):
+    id = models.AutoField(primary_key=True)
+    event = models.ForeignKey(Event, models.SET_NULL, null=True)
+    address = models.CharField(max_length=255)

core/mail/protocol.py

@@ -1,16 +1,42 @@
 import logging
 import aiosmtplib
+from asgiref.sync import sync_to_async
+from channels.layers import get_channel_layer
+
+from mail.models import Email, EventAddress
+from notify_sessions.models import SystemEvent
+from tickets.models import IssueThread
 
 
-def make_reply(message, to, subject):
+def collect_references(issue_thread):
+    mails = issue_thread.emails.order_by('timestamp')
+    references = []
+    for mail in mails:
+        if mail.reference:
+            references.append(mail.reference)
+    return references
+
+
+def make_reply(reply_email, references=None):
     from email.message import EmailMessage
     from core.settings import MAIL_DOMAIN
 
     reply = EmailMessage()
-    reply["From"] = "noreply@" + MAIL_DOMAIN
-    reply["To"] = to
-    reply["Subject"] = subject
-    reply.set_content(message)
+    reply["From"] = reply_email.sender
+    reply["To"] = reply_email.recipient
+    reply["Subject"] = reply_email.subject
+    if reply_email.in_reply_to:
+        reply["In-Reply-To"] = reply_email.in_reply_to
+    if reply_email.reference:
+        reply["Message-ID"] = reply_email.reference
+    else:
+        reply["Message-ID"] = reply_email.id + "@" + MAIL_DOMAIN
+        reply_email.reference = reply["Message-ID"]
+        reply_email.save()
+    if references:
+        reply["References"] = " ".join(references)
+
+    reply.set_content(reply_email.body)
 
     return reply
 
@@ -20,6 +46,14 @@ async def send_smtp(message, log):
     await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False)
 
 
+def find_active_issue_thread(in_reply_to):
+    reply_to = Email.objects.filter(reference=in_reply_to)
+    if reply_to.exists():
+        return reply_to.first().issue_thread
+    else:
+        return IssueThread.objects.create()
+
+
 class LMTPHandler:
     async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
         from core.settings import MAIL_DOMAIN
@@ -31,6 +65,7 @@ class LMTPHandler:
     async def handle_DATA(self, server, session, envelope):
         import email
         log = logging.getLogger('mail.log')
+        log.setLevel(logging.DEBUG)
         log.info('Message from %s' % envelope.mail_from)
         log.info('Message for %s' % envelope.rcpt_tos)
         log.info('Message data:\n')
@@ -54,15 +89,60 @@ class LMTPHandler:
 
             header_from = parsed.get('From')
             header_to = parsed.get('To')
+            header_in_reply_to = parsed.get('In-Reply-To')
+            header_message_id = parsed.get('Message-ID')
 
             if header_from != envelope.mail_from:
                 log.warning("Header from does not match envelope from")
+                log.info(f"Header from: {header_from}, envelope from: {envelope.mail_from}")
 
             if header_to != envelope.rcpt_tos[0]:
                 log.warning("Header to does not match envelope to")
+                log.info(f"Header to: {header_to}, envelope to: {envelope.rcpt_tos[0]}")
 
-            await send_smtp(make_reply("Thank you for your message.", envelope.mail_from, 'Message received'), log)
+            recipient = envelope.rcpt_tos[0]
+            sender = envelope.mail_from
+            target_event = None
+            try:
+                address_map = await sync_to_async(EventAddress.objects.get)(address=recipient)
+                if address_map.event:
+                    target_event = address_map.event
+            except EventAddress.DoesNotExist:
+                pass
+
+            active_issue_thread = await sync_to_async(find_active_issue_thread)(header_in_reply_to)
+
+            email = await sync_to_async(Email.objects.create)(sender=sender,
+                                                              recipient=recipient,
+                                                              body=body.decode('utf-8'),
+                                                              subject=parsed.get('Subject'),
+                                                              reference=header_message_id,
+                                                              in_reply_to=header_in_reply_to,
+                                                              raw=envelope.content.decode('utf-8'),
+                                                              event=target_event,
+                                                              issue_thread=active_issue_thread)
+            log.info(f"Created email {email.id}")
+            systemevent = await sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id)
+            log.info(f"Created system event {systemevent.id}")
+            channel_layer = get_channel_layer()
+            await channel_layer.group_send(
+                'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id,
+                            "message": "email received"}
+            )
+            log.info(f"Sent message to frontend")
+
+            reply_email = await sync_to_async(Email.objects.create)(sender=recipient,  # "noreply@" + MAIL_DOMAIN,
+                                                                    recipient=sender,
+                                                                    body="Thank you for your message.",
+                                                                    subject="Message received",
+                                                                    in_reply_to=header_message_id,
+                                                                    event=target_event,
+                                                                    issue_thread=active_issue_thread)
+
+            references = await sync_to_async(collect_references)(active_issue_thread)
+            await send_smtp(make_reply(reply_email, references), log)
             log.info("Sent reply")
+
             return '250 Message accepted for delivery'
         except Exception as e:
             log.error(e)

core/mail/tests/v2/test_mails.py

@@ -0,0 +1,198 @@
+import inspect
+from unittest import mock
+
+from django.test import TestCase, Client
+
+from inventory.models import Event
+from mail.models import Email
+from mail.protocol import LMTPHandler
+from tickets.models import IssueThread
+
+client = Client()
+
+
+def make_mocked_coro(return_value=mock.sentinel, raise_exception=mock.sentinel):
+    async def mock_coro(*args, **kwargs):
+        if raise_exception is not mock.sentinel:
+            raise raise_exception
+        if not inspect.isawaitable(return_value):
+            return return_value
+        await return_value
+
+    return mock.Mock(wraps=mock_coro)
+
+
+class EmailsApiTest(TestCase):
+
+    def test_mails(self):
+        Event.objects.get_or_create(
+            name="Test event",
+            slug="test-event",
+        )
+        Email.objects.create(
+            subject='test',
+            body='test',
+            sender='test',
+            recipient='test',
+        )
+        response = client.get('/api/2/mails/')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.json()), 1)
+        self.assertEqual(response.json()[0]['subject'], 'test')
+        self.assertEqual(response.json()[0]['body'], 'test')
+        self.assertEqual(response.json()[0]['sender'], 'test')
+        self.assertEqual(response.json()[0]['recipient'], 'test')
+
+    def test_mails_empty(self):
+        response = client.get('/api/2/mails/')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json(), [])
+
+
+class LMTPHandlerTestCase(TestCase):  # TODO replace with less hacky test
+
+    def test_handle_client(self):
+        from aiosmtpd.smtp import Envelope
+        from asgiref.sync import async_to_sync
+        import aiosmtplib
+        aiosmtplib.send = make_mocked_coro()
+        handler = LMTPHandler()
+        server = mock.Mock()
+        session = mock.Mock()
+        envelope = Envelope()
+        envelope.mail_from = 'test1@test'
+        envelope.rcpt_tos = ['test2@test']
+        envelope.content = b'Subject: test\nFrom: test3@test\nTo: test4@test\nMessage-ID: <1@test>\n\ntest'
+        result = async_to_sync(handler.handle_DATA)(server, session, envelope)
+        self.assertEqual(result, '250 Message accepted for delivery')
+        self.assertEqual(len(Email.objects.all()), 2)
+        self.assertEqual(len(IssueThread.objects.all()), 1)
+        aiosmtplib.send.assert_called_once()
+        self.assertEqual(Email.objects.all()[0].subject, 'test')
+        self.assertEqual(Email.objects.all()[0].sender, 'test1@test')
+        self.assertEqual(Email.objects.all()[0].recipient, 'test2@test')
+        self.assertEqual(Email.objects.all()[0].body, 'test')
+        self.assertEqual(Email.objects.all()[0].issue_thread, IssueThread.objects.all()[0])
+        self.assertEqual(Email.objects.all()[0].reference, '<1@test>')
+        self.assertEqual(Email.objects.all()[0].in_reply_to, None)
+        self.assertEqual(Email.objects.all()[1].subject, 'Message received')
+        self.assertEqual(Email.objects.all()[1].sender, 'test2@test')
+        self.assertEqual(Email.objects.all()[1].recipient, 'test1@test')
+        self.assertEqual(Email.objects.all()[1].body, 'Thank you for your message.')
+        self.assertEqual(Email.objects.all()[1].issue_thread, IssueThread.objects.all()[0])
+        self.assertTrue(Email.objects.all()[1].reference.startswith("<"))
+        self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>"))
+        self.assertEqual(Email.objects.all()[1].in_reply_to, "<1@test>")
+
+    def test_handle_client_reply(self):
+        issue_thread = IssueThread.objects.create()
+        mail1 = Email.objects.create(
+            subject='test',
+            body='test',
+            sender='test1@test',
+            recipient='test2@test',
+            issue_thread=issue_thread,
+        )
+        mail1_reply = Email.objects.create(
+            subject='Message received',
+            body='Thank you for your message.',
+            sender='test2@test',
+            recipient='test1@test',
+            in_reply_to=mail1.reference,
+            issue_thread=issue_thread,
+        )
+        from aiosmtpd.smtp import Envelope
+        from asgiref.sync import async_to_sync
+        import aiosmtplib
+        aiosmtplib.send = make_mocked_coro()
+        handler = LMTPHandler()
+        server = mock.Mock()
+        session = mock.Mock()
+        envelope = Envelope()
+        envelope.mail_from = 'test1@test'
+        envelope.rcpt_tos = ['test2@test']
+        # envelope.content = b'Subject: Re: test\nFrom: test3@test\nTo: test4@test\nMessage-ID: 3@test\nIn-Reply-To: 2@localhost\n\ntest'
+        envelope.content = (f'Subject: Re: test\nFrom: test3@test\nTo: test4@test\nMessage-ID: <3@test>\n'
+                            f'In-Reply-To: {mail1_reply.reference}'.encode('utf-8') + b'\n\ntest')
+        result = async_to_sync(handler.handle_DATA)(server, session, envelope)
+        self.assertEqual(result, '250 Message accepted for delivery')
+        self.assertEqual(len(Email.objects.all()), 4)
+        self.assertEqual(len(IssueThread.objects.all()), 1)
+        aiosmtplib.send.assert_called_once()
+        self.assertEqual(Email.objects.all()[2].subject, 'Re: test')
+        self.assertEqual(Email.objects.all()[2].sender, 'test1@test')
+        self.assertEqual(Email.objects.all()[2].recipient, 'test2@test')
+        self.assertEqual(Email.objects.all()[2].body, 'test')
+        self.assertEqual(Email.objects.all()[2].issue_thread, issue_thread)
+        self.assertEqual(Email.objects.all()[2].reference, '<3@test>')
+        self.assertEqual(Email.objects.all()[2].in_reply_to, mail1_reply.reference)
+        self.assertEqual(Email.objects.all()[3].subject, 'Message received')
+        self.assertEqual(Email.objects.all()[3].sender, 'test2@test')
+        self.assertEqual(Email.objects.all()[3].recipient, 'test1@test')
+        self.assertEqual(Email.objects.all()[3].body, 'Thank you for your message.')
+        self.assertEqual(Email.objects.all()[3].issue_thread, issue_thread)
+        self.assertTrue(Email.objects.all()[3].reference.startswith("<"))
+        self.assertTrue(Email.objects.all()[3].reference.endswith("@localhost>"))
+        self.assertEqual(Email.objects.all()[3].in_reply_to, "<3@test>")
+
+# class AsyncLMTPTestCase(TestCase):
+#
+#    def setUp(self):
+#        server = mock.Mock()
+#        self.create_unix_server = make_mocked_coro(server)
+#        self.wait_closed = make_mocked_coro()
+#        self.loop = asyncio.new_event_loop()
+#        self.loop.create_unix_server = self.create_unix_server
+#        asyncio.set_event_loop(self.loop)
+#
+#    async def test_connect(self):
+#        server = await UnixSocketLMTPController(LMTPHandler(), unix_socket='lmtp.sock', loop=self.loop).serve()
+#        self.assertEqual(self.create_unix_server.call_count, 1)
+#        self.assertEqual(self.wait_closed.call_count, 0)
+#        server.close()
+#        # self.assertEqual(self.wait_closed.call_count, 1)
+#
+#    def test_receive_mail(self):
+#        from logging import getLogger
+#        from aiosmtpd.lmtp import LMTP
+#        from asgiref.sync import async_to_sync
+#        log = getLogger('mail.log')
+#        log.addHandler(logging.StreamHandler())
+#        log.setLevel(logging.DEBUG)
+#        handler = LMTP(LMTPHandler(), loop=self.loop)
+#        transport = mock.Mock()
+#        handler.connection_made(transport)
+#
+#        def _handle_client():
+#            print("Handling client")
+#            async_to_sync(handler._handle_client)()
+#            print("Client handled")
+#
+#        thread = threading.Thread(target=_handle_client)
+#        thread.start()
+#
+#        # handler.data_received(
+#        #    b'HELO test\nMAIL FROM:<test1@test>\nRCPT TO:<test2@test>\nDATA\nSubject: test\nFrom: test1@test\nTo: '
+#        #    b'test2@test\n\ntest\n.\nQUIT')
+#        handler.data_received(b'HELO test\n')
+#        handler.data_received(b'MAIL FROM:<test1@test>\n')
+#        handler.data_received(b'RCPT TO:<test2@test>\n')
+#        handler.data_received(b'DATA\n')
+#        handler.data_received(b'Subject: test\n')
+#        handler.data_received(b'From: test1@test\n')
+#        handler.data_received(b'To: test2@test\n')
+#        handler.data_received(b'\n')
+#        handler.data_received(b'test\n')
+#        handler.data_received(b'.\n')
+#        handler.data_received(b'QUIT\n')
+#
+#        thread.join()
+#
+#        handler.connection_lost(None)
+#        thread.join()
+#
+#        # self.assertEqual(len(Email.objects.all()), 1)
+#        # self.assertEqual(Email.objects.all()[0].subject, 'test')
+#        # self.assertEqual(Email.objects.all()[0].body, 'test')
+#        # self.assertEqual(Email.objects.all()[0].sender, 'test')
+#        # self.assertEqual(Email.objects.all()[0].recipient, 'test@test')