tickets module: api_v2, admin views and tests

core/core/settings.py

@@ -197,14 +197,21 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 1024 * 1024 * 128  # 128 MB
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
-CHANNEL_LAYERS = {
+if 'test' in sys.argv:
-    'default': {
+    CHANNEL_LAYERS = {
-        'BACKEND': 'channels_redis.core.RedisChannelLayer',
+        'default': {
-        'CONFIG': {
+            'BACKEND': 'channels.layers.InMemoryChannelLayer'
-            'hosts': [('localhost', 6379)],
+        }
-        },
+    }
+else:
+    CHANNEL_LAYERS = {
+        'default': {
+            'BACKEND': 'channels_redis.core.RedisChannelLayer',
+            'CONFIG': {
+                'hosts': [('localhost', 6379)],
+            },
+        }
+
     }
 
-}
-
 TEST_RUNNER = 'core.test_runner.FastTestRunner'

core/core/urls.py

@@ -27,6 +27,8 @@ urlpatterns = [
     path('api/2/', include('inventory.api_v2')),
     path('api/2/', include('files.api_v2')),
     path('media/2/', include('files.media_v2')),
+    path('api/2/', include('tickets.api_v2')),
+    path('api/2/', include('notify_sessions.api_v2')),
     path('api/2/', include('authentication.api_v2')),
     path('api/', get_info),
 ]

core/mail/protocol.py

@@ -1,16 +1,43 @@
 import logging
 import aiosmtplib
+from asgiref.sync import sync_to_async
+from channels.layers import get_channel_layer
+
+from mail.models import Email, EventAddress
+from notify_sessions.models import SystemEvent
+from tickets.models import IssueThread, StateChange
 
 
-def make_reply(message, to, subject):
+def collect_references(issue_thread):
+    mails = issue_thread.emails.order_by('timestamp')
+    references = []
+    for mail in mails:
+        if mail.reference:
+            references.append(mail.reference)
+    return references
+
+
+def make_reply(reply_email, references=None, event=None, issue_thread=None):
     from email.message import EmailMessage
     from core.settings import MAIL_DOMAIN
-
+    event = event or "noreply"
     reply = EmailMessage()
-    reply["From"] = "noreply@" + MAIL_DOMAIN
+    reply["From"] = reply_email.sender
-    reply["To"] = to
+    reply["To"] = reply_email.recipient
-    reply["Subject"] = subject
+    reply["Subject"] = reply_email.subject
-    reply.set_content(message)
+    reply["Reply-To"] = f"{event}+{issue_thread}@{MAIL_DOMAIN}"
+    if reply_email.in_reply_to:
+        reply["In-Reply-To"] = reply_email.in_reply_to
+    if reply_email.reference:
+        reply["Message-ID"] = reply_email.reference
+    else:
+        reply["Message-ID"] = reply_email.id + "@" + MAIL_DOMAIN
+        reply_email.reference = reply["Message-ID"]
+        reply_email.save()
+    if references:
+        reply["References"] = " ".join(references)
+
+    reply.set_content(reply_email.body)
 
     return reply
 
@@ -20,6 +47,15 @@ async def send_smtp(message, log):
     await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False)
 
 
+def find_active_issue_thread(in_reply_to, subject=None):
+    reply_to = Email.objects.filter(reference=in_reply_to)
+    if reply_to.exists():
+        return reply_to.first().issue_thread, False
+    else:
+        issue = IssueThread.objects.create(name=subject)
+        return issue, True
+
+
 class LMTPHandler:
     async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
         from core.settings import MAIL_DOMAIN
@@ -31,6 +67,7 @@ class LMTPHandler:
     async def handle_DATA(self, server, session, envelope):
         import email
         log = logging.getLogger('mail.log')
+        log.setLevel(logging.DEBUG)
         log.info('Message from %s' % envelope.mail_from)
         log.info('Message for %s' % envelope.rcpt_tos)
         log.info('Message data:\n')
@@ -54,15 +91,63 @@ class LMTPHandler:
 
             header_from = parsed.get('From')
             header_to = parsed.get('To')
+            header_in_reply_to = parsed.get('In-Reply-To')
+            header_message_id = parsed.get('Message-ID')
 
             if header_from != envelope.mail_from:
                 log.warning("Header from does not match envelope from")
+                log.info(f"Header from: {header_from}, envelope from: {envelope.mail_from}")
 
             if header_to != envelope.rcpt_tos[0]:
                 log.warning("Header to does not match envelope to")
+                log.info(f"Header to: {header_to}, envelope to: {envelope.rcpt_tos[0]}")
+
+            recipient = envelope.rcpt_tos[0]
+            sender = envelope.mail_from
+            subject = parsed.get('Subject')
+            target_event = None
+            try:
+                address_map = await sync_to_async(EventAddress.objects.get)(address=recipient)
+                if address_map.event:
+                    target_event = address_map.event
+            except EventAddress.DoesNotExist:
+                pass
+
+            active_issue_thread, new = await sync_to_async(find_active_issue_thread)(header_in_reply_to, subject)
+
+            email = await sync_to_async(Email.objects.create)(sender=sender,
+                                                              recipient=recipient,
+                                                              body=body.decode('utf-8'),
+                                                              subject=subject,
+                                                              reference=header_message_id,
+                                                              in_reply_to=header_in_reply_to,
+                                                              raw=envelope.content.decode('utf-8'),
+                                                              event=target_event,
+                                                              issue_thread=active_issue_thread)
+            log.info(f"Created email {email.id}")
+            systemevent = await sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id)
+            log.info(f"Created system event {systemevent.id}")
+            channel_layer = get_channel_layer()
+            await channel_layer.group_send(
+                'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id,
+                            "message": "email received"}
+            )
+            log.info(f"Sent message to frontend")
+            if new:
+                await sync_to_async(StateChange.objects.create)(issue_thread=active_issue_thread, state='new')
+
+                references = await sync_to_async(collect_references)(active_issue_thread)
+
+                reply_email = await sync_to_async(Email.objects.create)(sender=recipient,  # "noreply@" + MAIL_DOMAIN,
+                                                                        recipient=sender,
+                                                                        body="Thank you for your message.",
+                                                                        subject="Message received",
+                                                                        in_reply_to=header_message_id,
+                                                                        event=target_event,
+                                                                        issue_thread=active_issue_thread)
+                await send_smtp(make_reply(reply_email, references), log)
+                log.info("Sent auto reply")
 
-            await send_smtp(make_reply("Thank you for your message.", envelope.mail_from, 'Message received'), log)
-            log.info("Sent reply")
             return '250 Message accepted for delivery'
         except Exception as e:
             log.error(e)

core/tickets/admin.py

@@ -0,0 +1,20 @@
+from django.contrib import admin
+
+from tickets.models import IssueThread, Comment, StateChange
+
+
+class IssueThreadAdmin(admin.ModelAdmin):
+    pass
+
+
+class CommentAdmin(admin.ModelAdmin):
+    pass
+
+
+class StateChangeAdmin(admin.ModelAdmin):
+    pass
+
+
+admin.site.register(IssueThread, IssueThreadAdmin)
+admin.site.register(Comment, CommentAdmin)
+admin.site.register(StateChange, StateChangeAdmin)

core/tickets/api_v2.py

@@ -0,0 +1,125 @@
+import logging
+
+from django.urls import re_path
+from django.contrib.auth.decorators import permission_required
+from rest_framework import routers, viewsets, serializers, status
+from rest_framework.decorators import api_view, permission_classes
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
+from asgiref.sync import async_to_sync
+from channels.layers import get_channel_layer
+
+from core.settings import MAIL_DOMAIN
+from mail.models import Email
+from mail.protocol import send_smtp, make_reply, collect_references
+from notify_sessions.models import SystemEvent
+from tickets.models import IssueThread
+
+
+class IssueSerializer(serializers.ModelSerializer):
+    timeline = serializers.SerializerMethodField()
+
+    class Meta:
+        model = IssueThread
+        fields = ('id', 'timeline', 'name', 'state', 'assigned_to', 'last_activity')
+        read_only_fields = ('id', 'timeline', 'last_activity')
+
+    @staticmethod
+    def get_timeline(obj):
+        timeline = []
+        for comment in obj.comments.all():
+            timeline.append({
+                'type': 'comment',
+                'id': comment.id,
+                'timestamp': comment.timestamp,
+                'comment': comment.comment,
+            })
+        for state_change in obj.state_changes.all():
+            timeline.append({
+                'type': 'state',
+                'id': state_change.id,
+                'timestamp': state_change.timestamp,
+                'state': state_change.state,
+            })
+        for email in obj.emails.all():
+            timeline.append({
+                'type': 'mail',
+                'id': email.id,
+                'timestamp': email.timestamp,
+                'sender': email.sender,
+                'recipient': email.recipient,
+                'subject': email.subject,
+                'body': email.body,
+            })
+        return sorted(timeline, key=lambda x: x['timestamp'])
+
+
+class IssueViewSet(viewsets.ModelViewSet):
+    serializer_class = IssueSerializer
+    queryset = IssueThread.objects.all()
+
+
+@api_view(['POST'])
+@permission_classes([IsAuthenticated])
+@permission_required('tickets.add_issuethread', raise_exception=True)
+def reply(request, pk):
+    issue = IssueThread.objects.get(pk=pk)
+    # email = issue.reply(request.data['body']) # TODO evaluate if this is a useful abstraction
+    references = collect_references(issue)
+    most_recent = Email.objects.filter(issue_thread=issue, recipient__endswith='@' + MAIL_DOMAIN).order_by(
+        '-timestamp').first()
+    mail = Email.objects.create(
+        issue_thread=issue,
+        sender=most_recent.recipient,
+        recipient=most_recent.sender,
+        subject=f'Re: {most_recent.subject}',
+        body=request.data['message'],
+        in_reply_to=most_recent.reference,
+    )
+    log = logging.getLogger('mail.log')
+    async_to_sync(send_smtp)(make_reply(mail, references), log)
+
+    return Response({'status': 'ok'}, status=status.HTTP_201_CREATED)
+
+
+@api_view(['POST'])
+@permission_classes([IsAuthenticated])
+@permission_required('tickets.add_issuethread_manual', raise_exception=True)
+def manual_ticket(request):
+    if 'name' not in request.data:
+        return Response({'status': 'error', 'message': 'missing name'}, status=status.HTTP_400_BAD_REQUEST)
+    if 'sender' not in request.data:
+        return Response({'status': 'error', 'message': 'missing sender'}, status=status.HTTP_400_BAD_REQUEST)
+    if 'recipient' not in request.data:
+        return Response({'status': 'error', 'message': 'missing recipient'}, status=status.HTTP_400_BAD_REQUEST)
+    if 'body' not in request.data:
+        return Response({'status': 'error', 'message': 'missing body'}, status=status.HTTP_400_BAD_REQUEST)
+
+    issue = IssueThread.objects.create(
+        name=request.data['name'],
+        manually_created=True,
+    )
+    email = Email.objects.create(
+        issue_thread=issue,
+        sender=request.data['sender'],
+        recipient=request.data['recipient'],
+        subject=request.data['name'],
+        body=request.data['body'],
+    )
+    systemevent = SystemEvent.objects.create(type='email received', reference=email.id)
+    channel_layer = get_channel_layer()
+    async_to_sync(channel_layer.group_send)(
+        'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id,
+                    "message": "email received"}
+    )
+
+    return Response(IssueSerializer(issue).data, status=status.HTTP_201_CREATED)
+
+
+router = routers.SimpleRouter()
+router.register(r'tickets', IssueViewSet, basename='issues')
+
+urlpatterns = ([
+                   re_path(r'^tickets/(?P<pk>\d+)/reply/$', reply, name='reply'),
+                   re_path(r'^tickets/manual/$', manual_ticket, name='manual_ticket'),
+               ] + router.urls)

core/tickets/tests/v2/test_tickets.py

@@ -0,0 +1,110 @@
+from datetime import datetime, timedelta
+
+from django.test import TestCase, Client
+
+from authentication.models import ExtendedUser
+from mail.models import Email
+from tickets.models import IssueThread, StateChange, Comment
+from django.contrib.auth.models import Permission
+from knox.models import AuthToken
+
+
+class IssueApiTest(TestCase):
+
+    def setUp(self):
+        super().setUp()
+        self.user = ExtendedUser.objects.create_user('testuser', 'test', 'test')
+        self.user.user_permissions.add(*Permission.objects.all())
+        self.user.save()
+        self.token = AuthToken.objects.create(user=self.user)
+        self.client = Client(headers={'Authorization': 'Token ' + self.token[1]})
+
+    def test_issues_empty(self):
+        response = self.client.get('/api/2/tickets/')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json(), [])
+
+    def test_issues(self):
+        now = datetime.now()
+        issue = IssueThread.objects.create(
+            name="test issue",
+        )
+        mail1 = Email.objects.create(
+            subject='test',
+            body='test',
+            sender='test',
+            recipient='test',
+            issue_thread=issue,
+            timestamp=now,
+        )
+        state = StateChange.objects.create(
+            issue_thread=issue,
+            state="new",
+            timestamp=now + timedelta(seconds=1),
+        )
+        mail2 = Email.objects.create(
+            subject='test',
+            body='test',
+            sender='test',
+            recipient='test',
+            issue_thread=issue,
+            in_reply_to=mail1.reference,
+            timestamp=now + timedelta(seconds=2),
+        )
+        comment = Comment.objects.create(
+            issue_thread=issue,
+            comment="test",
+            timestamp=now + timedelta(seconds=3),
+        )
+
+        response = self.client.get('/api/2/tickets/')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.json()), 1)
+        self.assertEqual(response.json()[0]['id'], issue.id)
+        self.assertEqual(response.json()[0]['name'], "test issue")
+        self.assertEqual(response.json()[0]['state'], "new")
+        self.assertEqual(response.json()[0]['assigned_to'], None)
+        self.assertEqual(response.json()[0]['last_activity'], issue.last_activity.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
+        self.assertEqual(len(response.json()[0]['timeline']), 4)
+        self.assertEqual(response.json()[0]['timeline'][0]['type'], 'mail')
+        self.assertEqual(response.json()[0]['timeline'][1]['type'], 'state')
+        self.assertEqual(response.json()[0]['timeline'][2]['type'], 'mail')
+        self.assertEqual(response.json()[0]['timeline'][3]['type'], 'comment')
+        self.assertEqual(response.json()[0]['timeline'][0]['id'], mail1.id)
+        self.assertEqual(response.json()[0]['timeline'][1]['id'], state.id)
+        self.assertEqual(response.json()[0]['timeline'][2]['id'], mail2.id)
+        self.assertEqual(response.json()[0]['timeline'][3]['id'], comment.id)
+        self.assertEqual(response.json()[0]['timeline'][0]['sender'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][0]['recipient'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][0]['subject'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][0]['body'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][0]['timestamp'],
+                         mail1.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
+        self.assertEqual(response.json()[0]['timeline'][1]['state'], 'new')
+        self.assertEqual(response.json()[0]['timeline'][1]['timestamp'],
+                         state.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
+        self.assertEqual(response.json()[0]['timeline'][2]['sender'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][2]['recipient'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][2]['subject'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][2]['body'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][2]['timestamp'],
+                         mail2.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
+        self.assertEqual(response.json()[0]['timeline'][3]['comment'], 'test')
+        self.assertEqual(response.json()[0]['timeline'][3]['timestamp'],
+                         comment.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
+
+    def test_manual_creation(self):
+        response = self.client.post('/api/2/tickets/manual/', {'name': 'test issue', 'sender': 'test',
+                                                               'recipient': 'test', 'body': 'test'})
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(response.json()['state'], 'new')
+        self.assertEqual(response.json()['name'], 'test issue')
+        self.assertEqual(response.json()['assigned_to'], None)
+        timeline = response.json()['timeline']
+        self.assertEqual(len(timeline), 1)
+        self.assertEqual(timeline[0]['type'], 'mail')
+        self.assertEqual(timeline[0]['sender'], 'test')
+        self.assertEqual(timeline[0]['recipient'], 'test')
+        self.assertEqual(timeline[0]['subject'], 'test issue')
+        self.assertEqual(timeline[0]['body'], 'test')
+