From cbc27b143fae997fbb954c5081fd33d4380639d6 Mon Sep 17 00:00:00 2001 From: jedi Date: Sun, 7 Jan 2024 21:18:42 +0100 Subject: [PATCH] tickets module: api_v2, admin views and tests --- core/core/settings.py | 23 +++-- core/core/urls.py | 2 + core/mail/protocol.py | 101 +++++++++++++++++++-- core/tickets/admin.py | 20 +++++ core/tickets/api_v2.py | 125 ++++++++++++++++++++++++++ core/tickets/tests/__init__.py | 0 core/tickets/tests/v2/__init__.py | 0 core/tickets/tests/v2/test_tickets.py | 110 +++++++++++++++++++++++ 8 files changed, 365 insertions(+), 16 deletions(-) create mode 100644 core/tickets/admin.py create mode 100644 core/tickets/api_v2.py create mode 100644 core/tickets/tests/__init__.py create mode 100644 core/tickets/tests/v2/__init__.py create mode 100644 core/tickets/tests/v2/test_tickets.py diff --git a/core/core/settings.py b/core/core/settings.py index a6f7ce6..295c5af 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -197,14 +197,21 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 1024 * 1024 * 128 # 128 MB DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' -CHANNEL_LAYERS = { - 'default': { - 'BACKEND': 'channels_redis.core.RedisChannelLayer', - 'CONFIG': { - 'hosts': [('localhost', 6379)], - }, +if 'test' in sys.argv: + CHANNEL_LAYERS = { + 'default': { + 'BACKEND': 'channels.layers.InMemoryChannelLayer' + } + } +else: + CHANNEL_LAYERS = { + 'default': { + 'BACKEND': 'channels_redis.core.RedisChannelLayer', + 'CONFIG': { + 'hosts': [('localhost', 6379)], + }, + } + } -} - TEST_RUNNER = 'core.test_runner.FastTestRunner' diff --git a/core/core/urls.py b/core/core/urls.py index bb5a7d9..fff25ae 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -27,6 +27,8 @@ urlpatterns = [ path('api/2/', include('inventory.api_v2')), path('api/2/', include('files.api_v2')), path('media/2/', include('files.media_v2')), + path('api/2/', include('tickets.api_v2')), + path('api/2/', include('notify_sessions.api_v2')), path('api/2/', include('authentication.api_v2')), path('api/', get_info), ] diff --git a/core/mail/protocol.py b/core/mail/protocol.py index d466f51..dae4d96 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -1,16 +1,43 @@ import logging import aiosmtplib +from asgiref.sync import sync_to_async +from channels.layers import get_channel_layer + +from mail.models import Email, EventAddress +from notify_sessions.models import SystemEvent +from tickets.models import IssueThread, StateChange -def make_reply(message, to, subject): +def collect_references(issue_thread): + mails = issue_thread.emails.order_by('timestamp') + references = [] + for mail in mails: + if mail.reference: + references.append(mail.reference) + return references + + +def make_reply(reply_email, references=None, event=None, issue_thread=None): from email.message import EmailMessage from core.settings import MAIL_DOMAIN - + event = event or "noreply" reply = EmailMessage() - reply["From"] = "noreply@" + MAIL_DOMAIN - reply["To"] = to - reply["Subject"] = subject - reply.set_content(message) + reply["From"] = reply_email.sender + reply["To"] = reply_email.recipient + reply["Subject"] = reply_email.subject + reply["Reply-To"] = f"{event}+{issue_thread}@{MAIL_DOMAIN}" + if reply_email.in_reply_to: + reply["In-Reply-To"] = reply_email.in_reply_to + if reply_email.reference: + reply["Message-ID"] = reply_email.reference + else: + reply["Message-ID"] = reply_email.id + "@" + MAIL_DOMAIN + reply_email.reference = reply["Message-ID"] + reply_email.save() + if references: + reply["References"] = " ".join(references) + + reply.set_content(reply_email.body) return reply @@ -20,6 +47,15 @@ async def send_smtp(message, log): await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False) +def find_active_issue_thread(in_reply_to, subject=None): + reply_to = Email.objects.filter(reference=in_reply_to) + if reply_to.exists(): + return reply_to.first().issue_thread, False + else: + issue = IssueThread.objects.create(name=subject) + return issue, True + + class LMTPHandler: async def handle_RCPT(self, server, session, envelope, address, rcpt_options): from core.settings import MAIL_DOMAIN @@ -31,6 +67,7 @@ class LMTPHandler: async def handle_DATA(self, server, session, envelope): import email log = logging.getLogger('mail.log') + log.setLevel(logging.DEBUG) log.info('Message from %s' % envelope.mail_from) log.info('Message for %s' % envelope.rcpt_tos) log.info('Message data:\n') @@ -54,15 +91,63 @@ class LMTPHandler: header_from = parsed.get('From') header_to = parsed.get('To') + header_in_reply_to = parsed.get('In-Reply-To') + header_message_id = parsed.get('Message-ID') if header_from != envelope.mail_from: log.warning("Header from does not match envelope from") + log.info(f"Header from: {header_from}, envelope from: {envelope.mail_from}") if header_to != envelope.rcpt_tos[0]: log.warning("Header to does not match envelope to") + log.info(f"Header to: {header_to}, envelope to: {envelope.rcpt_tos[0]}") + + recipient = envelope.rcpt_tos[0] + sender = envelope.mail_from + subject = parsed.get('Subject') + target_event = None + try: + address_map = await sync_to_async(EventAddress.objects.get)(address=recipient) + if address_map.event: + target_event = address_map.event + except EventAddress.DoesNotExist: + pass + + active_issue_thread, new = await sync_to_async(find_active_issue_thread)(header_in_reply_to, subject) + + email = await sync_to_async(Email.objects.create)(sender=sender, + recipient=recipient, + body=body.decode('utf-8'), + subject=subject, + reference=header_message_id, + in_reply_to=header_in_reply_to, + raw=envelope.content.decode('utf-8'), + event=target_event, + issue_thread=active_issue_thread) + log.info(f"Created email {email.id}") + systemevent = await sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id) + log.info(f"Created system event {systemevent.id}") + channel_layer = get_channel_layer() + await channel_layer.group_send( + 'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id, + "message": "email received"} + ) + log.info(f"Sent message to frontend") + if new: + await sync_to_async(StateChange.objects.create)(issue_thread=active_issue_thread, state='new') + + references = await sync_to_async(collect_references)(active_issue_thread) + + reply_email = await sync_to_async(Email.objects.create)(sender=recipient, # "noreply@" + MAIL_DOMAIN, + recipient=sender, + body="Thank you for your message.", + subject="Message received", + in_reply_to=header_message_id, + event=target_event, + issue_thread=active_issue_thread) + await send_smtp(make_reply(reply_email, references), log) + log.info("Sent auto reply") - await send_smtp(make_reply("Thank you for your message.", envelope.mail_from, 'Message received'), log) - log.info("Sent reply") return '250 Message accepted for delivery' except Exception as e: log.error(e) diff --git a/core/tickets/admin.py b/core/tickets/admin.py new file mode 100644 index 0000000..d862811 --- /dev/null +++ b/core/tickets/admin.py @@ -0,0 +1,20 @@ +from django.contrib import admin + +from tickets.models import IssueThread, Comment, StateChange + + +class IssueThreadAdmin(admin.ModelAdmin): + pass + + +class CommentAdmin(admin.ModelAdmin): + pass + + +class StateChangeAdmin(admin.ModelAdmin): + pass + + +admin.site.register(IssueThread, IssueThreadAdmin) +admin.site.register(Comment, CommentAdmin) +admin.site.register(StateChange, StateChangeAdmin) diff --git a/core/tickets/api_v2.py b/core/tickets/api_v2.py new file mode 100644 index 0000000..695aa7f --- /dev/null +++ b/core/tickets/api_v2.py @@ -0,0 +1,125 @@ +import logging + +from django.urls import re_path +from django.contrib.auth.decorators import permission_required +from rest_framework import routers, viewsets, serializers, status +from rest_framework.decorators import api_view, permission_classes +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer + +from core.settings import MAIL_DOMAIN +from mail.models import Email +from mail.protocol import send_smtp, make_reply, collect_references +from notify_sessions.models import SystemEvent +from tickets.models import IssueThread + + +class IssueSerializer(serializers.ModelSerializer): + timeline = serializers.SerializerMethodField() + + class Meta: + model = IssueThread + fields = ('id', 'timeline', 'name', 'state', 'assigned_to', 'last_activity') + read_only_fields = ('id', 'timeline', 'last_activity') + + @staticmethod + def get_timeline(obj): + timeline = [] + for comment in obj.comments.all(): + timeline.append({ + 'type': 'comment', + 'id': comment.id, + 'timestamp': comment.timestamp, + 'comment': comment.comment, + }) + for state_change in obj.state_changes.all(): + timeline.append({ + 'type': 'state', + 'id': state_change.id, + 'timestamp': state_change.timestamp, + 'state': state_change.state, + }) + for email in obj.emails.all(): + timeline.append({ + 'type': 'mail', + 'id': email.id, + 'timestamp': email.timestamp, + 'sender': email.sender, + 'recipient': email.recipient, + 'subject': email.subject, + 'body': email.body, + }) + return sorted(timeline, key=lambda x: x['timestamp']) + + +class IssueViewSet(viewsets.ModelViewSet): + serializer_class = IssueSerializer + queryset = IssueThread.objects.all() + + +@api_view(['POST']) +@permission_classes([IsAuthenticated]) +@permission_required('tickets.add_issuethread', raise_exception=True) +def reply(request, pk): + issue = IssueThread.objects.get(pk=pk) + # email = issue.reply(request.data['body']) # TODO evaluate if this is a useful abstraction + references = collect_references(issue) + most_recent = Email.objects.filter(issue_thread=issue, recipient__endswith='@' + MAIL_DOMAIN).order_by( + '-timestamp').first() + mail = Email.objects.create( + issue_thread=issue, + sender=most_recent.recipient, + recipient=most_recent.sender, + subject=f'Re: {most_recent.subject}', + body=request.data['message'], + in_reply_to=most_recent.reference, + ) + log = logging.getLogger('mail.log') + async_to_sync(send_smtp)(make_reply(mail, references), log) + + return Response({'status': 'ok'}, status=status.HTTP_201_CREATED) + + +@api_view(['POST']) +@permission_classes([IsAuthenticated]) +@permission_required('tickets.add_issuethread_manual', raise_exception=True) +def manual_ticket(request): + if 'name' not in request.data: + return Response({'status': 'error', 'message': 'missing name'}, status=status.HTTP_400_BAD_REQUEST) + if 'sender' not in request.data: + return Response({'status': 'error', 'message': 'missing sender'}, status=status.HTTP_400_BAD_REQUEST) + if 'recipient' not in request.data: + return Response({'status': 'error', 'message': 'missing recipient'}, status=status.HTTP_400_BAD_REQUEST) + if 'body' not in request.data: + return Response({'status': 'error', 'message': 'missing body'}, status=status.HTTP_400_BAD_REQUEST) + + issue = IssueThread.objects.create( + name=request.data['name'], + manually_created=True, + ) + email = Email.objects.create( + issue_thread=issue, + sender=request.data['sender'], + recipient=request.data['recipient'], + subject=request.data['name'], + body=request.data['body'], + ) + systemevent = SystemEvent.objects.create(type='email received', reference=email.id) + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + 'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id, + "message": "email received"} + ) + + return Response(IssueSerializer(issue).data, status=status.HTTP_201_CREATED) + + +router = routers.SimpleRouter() +router.register(r'tickets', IssueViewSet, basename='issues') + +urlpatterns = ([ + re_path(r'^tickets/(?P\d+)/reply/$', reply, name='reply'), + re_path(r'^tickets/manual/$', manual_ticket, name='manual_ticket'), + ] + router.urls) diff --git a/core/tickets/tests/__init__.py b/core/tickets/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/tickets/tests/v2/__init__.py b/core/tickets/tests/v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/tickets/tests/v2/test_tickets.py b/core/tickets/tests/v2/test_tickets.py new file mode 100644 index 0000000..1f29963 --- /dev/null +++ b/core/tickets/tests/v2/test_tickets.py @@ -0,0 +1,110 @@ +from datetime import datetime, timedelta + +from django.test import TestCase, Client + +from authentication.models import ExtendedUser +from mail.models import Email +from tickets.models import IssueThread, StateChange, Comment +from django.contrib.auth.models import Permission +from knox.models import AuthToken + + +class IssueApiTest(TestCase): + + def setUp(self): + super().setUp() + self.user = ExtendedUser.objects.create_user('testuser', 'test', 'test') + self.user.user_permissions.add(*Permission.objects.all()) + self.user.save() + self.token = AuthToken.objects.create(user=self.user) + self.client = Client(headers={'Authorization': 'Token ' + self.token[1]}) + + def test_issues_empty(self): + response = self.client.get('/api/2/tickets/') + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), []) + + def test_issues(self): + now = datetime.now() + issue = IssueThread.objects.create( + name="test issue", + ) + mail1 = Email.objects.create( + subject='test', + body='test', + sender='test', + recipient='test', + issue_thread=issue, + timestamp=now, + ) + state = StateChange.objects.create( + issue_thread=issue, + state="new", + timestamp=now + timedelta(seconds=1), + ) + mail2 = Email.objects.create( + subject='test', + body='test', + sender='test', + recipient='test', + issue_thread=issue, + in_reply_to=mail1.reference, + timestamp=now + timedelta(seconds=2), + ) + comment = Comment.objects.create( + issue_thread=issue, + comment="test", + timestamp=now + timedelta(seconds=3), + ) + + response = self.client.get('/api/2/tickets/') + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.json()), 1) + self.assertEqual(response.json()[0]['id'], issue.id) + self.assertEqual(response.json()[0]['name'], "test issue") + self.assertEqual(response.json()[0]['state'], "new") + self.assertEqual(response.json()[0]['assigned_to'], None) + self.assertEqual(response.json()[0]['last_activity'], issue.last_activity.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) + self.assertEqual(len(response.json()[0]['timeline']), 4) + self.assertEqual(response.json()[0]['timeline'][0]['type'], 'mail') + self.assertEqual(response.json()[0]['timeline'][1]['type'], 'state') + self.assertEqual(response.json()[0]['timeline'][2]['type'], 'mail') + self.assertEqual(response.json()[0]['timeline'][3]['type'], 'comment') + self.assertEqual(response.json()[0]['timeline'][0]['id'], mail1.id) + self.assertEqual(response.json()[0]['timeline'][1]['id'], state.id) + self.assertEqual(response.json()[0]['timeline'][2]['id'], mail2.id) + self.assertEqual(response.json()[0]['timeline'][3]['id'], comment.id) + self.assertEqual(response.json()[0]['timeline'][0]['sender'], 'test') + self.assertEqual(response.json()[0]['timeline'][0]['recipient'], 'test') + self.assertEqual(response.json()[0]['timeline'][0]['subject'], 'test') + self.assertEqual(response.json()[0]['timeline'][0]['body'], 'test') + self.assertEqual(response.json()[0]['timeline'][0]['timestamp'], + mail1.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) + self.assertEqual(response.json()[0]['timeline'][1]['state'], 'new') + self.assertEqual(response.json()[0]['timeline'][1]['timestamp'], + state.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) + self.assertEqual(response.json()[0]['timeline'][2]['sender'], 'test') + self.assertEqual(response.json()[0]['timeline'][2]['recipient'], 'test') + self.assertEqual(response.json()[0]['timeline'][2]['subject'], 'test') + self.assertEqual(response.json()[0]['timeline'][2]['body'], 'test') + self.assertEqual(response.json()[0]['timeline'][2]['timestamp'], + mail2.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) + self.assertEqual(response.json()[0]['timeline'][3]['comment'], 'test') + self.assertEqual(response.json()[0]['timeline'][3]['timestamp'], + comment.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) + + def test_manual_creation(self): + response = self.client.post('/api/2/tickets/manual/', {'name': 'test issue', 'sender': 'test', + 'recipient': 'test', 'body': 'test'}) + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json()['state'], 'new') + self.assertEqual(response.json()['name'], 'test issue') + self.assertEqual(response.json()['assigned_to'], None) + timeline = response.json()['timeline'] + self.assertEqual(len(timeline), 1) + self.assertEqual(timeline[0]['type'], 'mail') + self.assertEqual(timeline[0]['sender'], 'test') + self.assertEqual(timeline[0]['recipient'], 'test') + self.assertEqual(timeline[0]['subject'], 'test issue') + self.assertEqual(timeline[0]['body'], 'test') +