Compare commits

..

No commits in common. "cbc27b143fae997fbb954c5081fd33d4380639d6" and "fc05c7c1d84ad3e65e664c580565a82d1ade66f7" have entirely different histories.

15 changed files with 25 additions and 553 deletions

View file

@ -197,13 +197,6 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 1024 * 1024 * 128 # 128 MB
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
if 'test' in sys.argv:
CHANNEL_LAYERS = {
'default': {
'BACKEND': 'channels.layers.InMemoryChannelLayer'
}
}
else:
CHANNEL_LAYERS = {
'default': {
'BACKEND': 'channels_redis.core.RedisChannelLayer',

View file

@ -27,8 +27,6 @@ urlpatterns = [
path('api/2/', include('inventory.api_v2')),
path('api/2/', include('files.api_v2')),
path('media/2/', include('files.media_v2')),
path('api/2/', include('tickets.api_v2')),
path('api/2/', include('notify_sessions.api_v2')),
path('api/2/', include('authentication.api_v2')),
path('api/', get_info),
]

View file

@ -1,43 +1,16 @@
import logging
import aiosmtplib
from asgiref.sync import sync_to_async
from channels.layers import get_channel_layer
from mail.models import Email, EventAddress
from notify_sessions.models import SystemEvent
from tickets.models import IssueThread, StateChange
def collect_references(issue_thread):
mails = issue_thread.emails.order_by('timestamp')
references = []
for mail in mails:
if mail.reference:
references.append(mail.reference)
return references
def make_reply(reply_email, references=None, event=None, issue_thread=None):
def make_reply(message, to, subject):
from email.message import EmailMessage
from core.settings import MAIL_DOMAIN
event = event or "noreply"
reply = EmailMessage()
reply["From"] = reply_email.sender
reply["To"] = reply_email.recipient
reply["Subject"] = reply_email.subject
reply["Reply-To"] = f"{event}+{issue_thread}@{MAIL_DOMAIN}"
if reply_email.in_reply_to:
reply["In-Reply-To"] = reply_email.in_reply_to
if reply_email.reference:
reply["Message-ID"] = reply_email.reference
else:
reply["Message-ID"] = reply_email.id + "@" + MAIL_DOMAIN
reply_email.reference = reply["Message-ID"]
reply_email.save()
if references:
reply["References"] = " ".join(references)
reply.set_content(reply_email.body)
reply = EmailMessage()
reply["From"] = "noreply@" + MAIL_DOMAIN
reply["To"] = to
reply["Subject"] = subject
reply.set_content(message)
return reply
@ -47,15 +20,6 @@ async def send_smtp(message, log):
await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False)
def find_active_issue_thread(in_reply_to, subject=None):
reply_to = Email.objects.filter(reference=in_reply_to)
if reply_to.exists():
return reply_to.first().issue_thread, False
else:
issue = IssueThread.objects.create(name=subject)
return issue, True
class LMTPHandler:
async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
from core.settings import MAIL_DOMAIN
@ -67,7 +31,6 @@ class LMTPHandler:
async def handle_DATA(self, server, session, envelope):
import email
log = logging.getLogger('mail.log')
log.setLevel(logging.DEBUG)
log.info('Message from %s' % envelope.mail_from)
log.info('Message for %s' % envelope.rcpt_tos)
log.info('Message data:\n')
@ -91,63 +54,15 @@ class LMTPHandler:
header_from = parsed.get('From')
header_to = parsed.get('To')
header_in_reply_to = parsed.get('In-Reply-To')
header_message_id = parsed.get('Message-ID')
if header_from != envelope.mail_from:
log.warning("Header from does not match envelope from")
log.info(f"Header from: {header_from}, envelope from: {envelope.mail_from}")
if header_to != envelope.rcpt_tos[0]:
log.warning("Header to does not match envelope to")
log.info(f"Header to: {header_to}, envelope to: {envelope.rcpt_tos[0]}")
recipient = envelope.rcpt_tos[0]
sender = envelope.mail_from
subject = parsed.get('Subject')
target_event = None
try:
address_map = await sync_to_async(EventAddress.objects.get)(address=recipient)
if address_map.event:
target_event = address_map.event
except EventAddress.DoesNotExist:
pass
active_issue_thread, new = await sync_to_async(find_active_issue_thread)(header_in_reply_to, subject)
email = await sync_to_async(Email.objects.create)(sender=sender,
recipient=recipient,
body=body.decode('utf-8'),
subject=subject,
reference=header_message_id,
in_reply_to=header_in_reply_to,
raw=envelope.content.decode('utf-8'),
event=target_event,
issue_thread=active_issue_thread)
log.info(f"Created email {email.id}")
systemevent = await sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id)
log.info(f"Created system event {systemevent.id}")
channel_layer = get_channel_layer()
await channel_layer.group_send(
'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id,
"message": "email received"}
)
log.info(f"Sent message to frontend")
if new:
await sync_to_async(StateChange.objects.create)(issue_thread=active_issue_thread, state='new')
references = await sync_to_async(collect_references)(active_issue_thread)
reply_email = await sync_to_async(Email.objects.create)(sender=recipient, # "noreply@" + MAIL_DOMAIN,
recipient=sender,
body="Thank you for your message.",
subject="Message received",
in_reply_to=header_message_id,
event=target_event,
issue_thread=active_issue_thread)
await send_smtp(make_reply(reply_email, references), log)
log.info("Sent auto reply")
await send_smtp(make_reply("Thank you for your message.", envelope.mail_from, 'Message received'), log)
log.info("Sent reply")
return '250 Message accepted for delivery'
except Exception as e:
log.error(e)

View file

@ -1,10 +0,0 @@
from django.contrib import admin
from notify_sessions.models import SystemEvent
class SystemEventAdmin(admin.ModelAdmin):
pass
admin.site.register(SystemEvent, SystemEventAdmin)

View file

@ -1,21 +0,0 @@
from rest_framework import routers, viewsets, serializers
from tickets.models import IssueThread
from notify_sessions.models import SystemEvent
class SystemEventSerializer(serializers.ModelSerializer):
class Meta:
model = SystemEvent
fields = '__all__'
class SystemEventViewSet(viewsets.ModelViewSet):
serializer_class = SystemEventSerializer
queryset = SystemEvent.objects.all()
router = routers.SimpleRouter()
router.register(r'systemevents', SystemEventViewSet, basename='systemevents')
urlpatterns = router.urls

View file

@ -1,4 +1,3 @@
import logging
from json.decoder import JSONDecodeError
from json import loads as json_loads
from json import dumps as json_dumps
@ -7,16 +6,18 @@ from channels.generic.websocket import AsyncWebsocketConsumer
class NotifyConsumer(AsyncWebsocketConsumer):
def __init__(self, *args, **kwargs):
super().__init__(args, kwargs)
self.log = logging.getLogger("server.log")
self.room_group_name = "general"
# self.event_slug = None
async def connect(self):
# self.event_slug = self.scope["url_route"]["kwargs"]["event_slug"]
# self.room_group_name = f"chat_{self.event_slug}"
# Join room group
await self.channel_layer.group_add(self.room_group_name, self.channel_name)
self.log.info(f"Added {self.channel_name} channel to {self.room_group_name} group")
await self.accept()
async def disconnect(self, close_code):
@ -24,34 +25,26 @@ class NotifyConsumer(AsyncWebsocketConsumer):
await self.channel_layer.group_discard(self.room_group_name, self.channel_name)
# Receive message from WebSocket
async def receive(self, text_data=None, bytes_data=None):
self.log.info(f"Received message: {text_data}")
async def receive(self, text_data):
try:
text_data_json = json_loads(text_data)
message = text_data_json["message"]
# Send message to room group
await self.channel_layer.group_send(
self.room_group_name,
{"type": "generic.event", "message": message, "name": "send_message_to_frontend", "event_id": 1}
self.room_group_name, {"type": "chat.message", "message": message}
)
except JSONDecodeError as e:
except JSONDecodeError:
await self.send(text_data=json_dumps({"message": "error", "error": "malformed json"}))
self.log.error(e)
except KeyError as e:
await self.send(text_data=json_dumps({"message": "error", "error": f"missing key: {str(e)}"}))
self.log.error(e)
except Exception as e:
await self.send(text_data=json_dumps({"message": "error", "error": "unknown error"}))
self.log.error(e)
raise e
# Receive message from room group
async def generic_event(self, event):
self.log.info(f"Received event: {event}")
async def chat_message(self, event):
message = event["message"]
name = event["name"]
event_id = event["event_id"]
# Send message to WebSocket
await self.send(text_data=json_dumps({"message": message, "name": name, "event_id": event_id}))
await self.send(text_data=json_dumps({"message": message}))

View file

@ -1,27 +0,0 @@
# Generated by Django 4.2.7 on 2023-12-09 02:13
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='SystemEvent',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('timestamp', models.DateTimeField(auto_now_add=True)),
('type', models.CharField(choices=[('ticket_created', 'ticket_created'), ('ticket_updated', 'ticket_updated'), ('ticket_deleted', 'ticket_deleted'), ('item_created', 'item_created'), ('item_updated', 'item_updated'), ('item_deleted', 'item_deleted'), ('user_created', 'user_created'), ('event_created', 'event_created'), ('event_updated', 'event_updated'), ('event_deleted', 'event_deleted')], max_length=255)),
('reference', models.IntegerField(blank=True, null=True)),
('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
],
),
]

View file

@ -1,48 +0,0 @@
import logging
from django.db import models
from asgiref.sync import sync_to_async
from channels.layers import get_channel_layer
from authentication.models import ExtendedUser
class SystemEvent(models.Model):
TYPE_CHOICES = [('ticket_created', 'ticket_created'),
('ticket_updated', 'ticket_updated'),
('ticket_deleted', 'ticket_deleted'),
('item_created', 'item_created'),
('item_updated', 'item_updated'),
('item_deleted', 'item_deleted'),
('user_created', 'user_created'),
('event_created', 'event_created'),
('event_updated', 'event_updated'),
('event_deleted', 'event_deleted'), ]
id = models.AutoField(primary_key=True)
timestamp = models.DateTimeField(auto_now_add=True)
user = models.ForeignKey(ExtendedUser, models.SET_NULL, null=True)
type = models.CharField(max_length=255, choices=TYPE_CHOICES)
reference = models.IntegerField(blank=True, null=True)
async def trigger_event(user, type, reference=None):
log = logging.getLogger('server.log')
log.info(f"Triggering event {type} for user {user} with reference {reference}")
try:
event = await sync_to_async(SystemEvent.objects.create, thread_sensitive=True)(user=user, type=type,
reference=reference)
channel_layer = get_channel_layer()
await channel_layer.group_send(
'general',
{
'type': 'generic.event',
'name': 'send_message_to_frontend',
'message': "event_trigered_from_views",
'event_id': event.id,
}
)
log.info(f"SystemEvent {event.id} triggered")
return event
except Exception as e:
log.error(e)
raise e

View file

@ -1,66 +0,0 @@
from django.test import TestCase
from channels.testing import WebsocketCommunicator
from notify_sessions.consumers import NotifyConsumer
from asgiref.sync import async_to_sync
class NotifyWebsocketTestCase(TestCase):
async def test_connect(self):
communicator = WebsocketCommunicator(NotifyConsumer.as_asgi(), "/ws/2/notify/")
connected, subprotocol = await communicator.connect()
self.assertTrue(connected)
await communicator.disconnect()
async def fut_send_message(self):
communicator = WebsocketCommunicator(NotifyConsumer.as_asgi(), "/ws/2/notify/")
connected, subprotocol = await communicator.connect()
self.assertTrue(connected)
await communicator.send_json_to({
"name": "foo",
"message": "bar",
})
response = await communicator.receive_json_from()
await communicator.disconnect()
return response
def test_send_message(self):
response = async_to_sync(self.fut_send_message)()
self.assertEqual(response["message"], "bar")
self.assertEqual(response["event_id"], 1)
self.assertEqual(response["name"], "send_message_to_frontend")
# events = SystemEvent.objects.all()
# self.assertEqual(len(events), 1)
# event = events[0]
# self.assertEqual(event.event_id, 1)
# self.assertEqual(event.name, "send_message_to_frontend")
# self.assertEqual(event.message, "bar")
async def fut_send_and_receive_message(self):
communicator1 = WebsocketCommunicator(NotifyConsumer.as_asgi(), "/ws/2/notify/")
communicator2 = WebsocketCommunicator(NotifyConsumer.as_asgi(), "/ws/2/notify/")
connected1, subprotocol1 = await communicator1.connect()
connected2, subprotocol2 = await communicator2.connect()
self.assertTrue(connected1)
self.assertTrue(connected2)
await communicator1.send_json_to({
"name": "foo",
"message": "bar",
})
response = await communicator2.receive_json_from()
await communicator1.disconnect()
await communicator2.disconnect()
return response
def test_send_and_receive_message(self):
response = async_to_sync(self.fut_send_and_receive_message)()
self.assertEqual(response["message"], "bar")
self.assertEqual(response["event_id"], 1)
self.assertEqual(response["name"], "send_message_to_frontend")
# events = SystemEvent.objects.all()
# self.assertEqual(len(events), 1)
# event = events[0]
# self.assertEqual(event.event_id, 1)
# self.assertEqual(event.name, "send_message_to_frontend")
# self.assertEqual(event.message, "bar")

View file

@ -1,20 +0,0 @@
from django.contrib import admin
from tickets.models import IssueThread, Comment, StateChange
class IssueThreadAdmin(admin.ModelAdmin):
pass
class CommentAdmin(admin.ModelAdmin):
pass
class StateChangeAdmin(admin.ModelAdmin):
pass
admin.site.register(IssueThread, IssueThreadAdmin)
admin.site.register(Comment, CommentAdmin)
admin.site.register(StateChange, StateChangeAdmin)

View file

@ -1,125 +0,0 @@
import logging
from django.urls import re_path
from django.contrib.auth.decorators import permission_required
from rest_framework import routers, viewsets, serializers, status
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer
from core.settings import MAIL_DOMAIN
from mail.models import Email
from mail.protocol import send_smtp, make_reply, collect_references
from notify_sessions.models import SystemEvent
from tickets.models import IssueThread
class IssueSerializer(serializers.ModelSerializer):
timeline = serializers.SerializerMethodField()
class Meta:
model = IssueThread
fields = ('id', 'timeline', 'name', 'state', 'assigned_to', 'last_activity')
read_only_fields = ('id', 'timeline', 'last_activity')
@staticmethod
def get_timeline(obj):
timeline = []
for comment in obj.comments.all():
timeline.append({
'type': 'comment',
'id': comment.id,
'timestamp': comment.timestamp,
'comment': comment.comment,
})
for state_change in obj.state_changes.all():
timeline.append({
'type': 'state',
'id': state_change.id,
'timestamp': state_change.timestamp,
'state': state_change.state,
})
for email in obj.emails.all():
timeline.append({
'type': 'mail',
'id': email.id,
'timestamp': email.timestamp,
'sender': email.sender,
'recipient': email.recipient,
'subject': email.subject,
'body': email.body,
})
return sorted(timeline, key=lambda x: x['timestamp'])
class IssueViewSet(viewsets.ModelViewSet):
serializer_class = IssueSerializer
queryset = IssueThread.objects.all()
@api_view(['POST'])
@permission_classes([IsAuthenticated])
@permission_required('tickets.add_issuethread', raise_exception=True)
def reply(request, pk):
issue = IssueThread.objects.get(pk=pk)
# email = issue.reply(request.data['body']) # TODO evaluate if this is a useful abstraction
references = collect_references(issue)
most_recent = Email.objects.filter(issue_thread=issue, recipient__endswith='@' + MAIL_DOMAIN).order_by(
'-timestamp').first()
mail = Email.objects.create(
issue_thread=issue,
sender=most_recent.recipient,
recipient=most_recent.sender,
subject=f'Re: {most_recent.subject}',
body=request.data['message'],
in_reply_to=most_recent.reference,
)
log = logging.getLogger('mail.log')
async_to_sync(send_smtp)(make_reply(mail, references), log)
return Response({'status': 'ok'}, status=status.HTTP_201_CREATED)
@api_view(['POST'])
@permission_classes([IsAuthenticated])
@permission_required('tickets.add_issuethread_manual', raise_exception=True)
def manual_ticket(request):
if 'name' not in request.data:
return Response({'status': 'error', 'message': 'missing name'}, status=status.HTTP_400_BAD_REQUEST)
if 'sender' not in request.data:
return Response({'status': 'error', 'message': 'missing sender'}, status=status.HTTP_400_BAD_REQUEST)
if 'recipient' not in request.data:
return Response({'status': 'error', 'message': 'missing recipient'}, status=status.HTTP_400_BAD_REQUEST)
if 'body' not in request.data:
return Response({'status': 'error', 'message': 'missing body'}, status=status.HTTP_400_BAD_REQUEST)
issue = IssueThread.objects.create(
name=request.data['name'],
manually_created=True,
)
email = Email.objects.create(
issue_thread=issue,
sender=request.data['sender'],
recipient=request.data['recipient'],
subject=request.data['name'],
body=request.data['body'],
)
systemevent = SystemEvent.objects.create(type='email received', reference=email.id)
channel_layer = get_channel_layer()
async_to_sync(channel_layer.group_send)(
'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id,
"message": "email received"}
)
return Response(IssueSerializer(issue).data, status=status.HTTP_201_CREATED)
router = routers.SimpleRouter()
router.register(r'tickets', IssueViewSet, basename='issues')
urlpatterns = ([
re_path(r'^tickets/(?P<pk>\d+)/reply/$', reply, name='reply'),
re_path(r'^tickets/manual/$', manual_ticket, name='manual_ticket'),
] + router.urls)

View file

@ -1,110 +0,0 @@
from datetime import datetime, timedelta
from django.test import TestCase, Client
from authentication.models import ExtendedUser
from mail.models import Email
from tickets.models import IssueThread, StateChange, Comment
from django.contrib.auth.models import Permission
from knox.models import AuthToken
class IssueApiTest(TestCase):
def setUp(self):
super().setUp()
self.user = ExtendedUser.objects.create_user('testuser', 'test', 'test')
self.user.user_permissions.add(*Permission.objects.all())
self.user.save()
self.token = AuthToken.objects.create(user=self.user)
self.client = Client(headers={'Authorization': 'Token ' + self.token[1]})
def test_issues_empty(self):
response = self.client.get('/api/2/tickets/')
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), [])
def test_issues(self):
now = datetime.now()
issue = IssueThread.objects.create(
name="test issue",
)
mail1 = Email.objects.create(
subject='test',
body='test',
sender='test',
recipient='test',
issue_thread=issue,
timestamp=now,
)
state = StateChange.objects.create(
issue_thread=issue,
state="new",
timestamp=now + timedelta(seconds=1),
)
mail2 = Email.objects.create(
subject='test',
body='test',
sender='test',
recipient='test',
issue_thread=issue,
in_reply_to=mail1.reference,
timestamp=now + timedelta(seconds=2),
)
comment = Comment.objects.create(
issue_thread=issue,
comment="test",
timestamp=now + timedelta(seconds=3),
)
response = self.client.get('/api/2/tickets/')
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.json()), 1)
self.assertEqual(response.json()[0]['id'], issue.id)
self.assertEqual(response.json()[0]['name'], "test issue")
self.assertEqual(response.json()[0]['state'], "new")
self.assertEqual(response.json()[0]['assigned_to'], None)
self.assertEqual(response.json()[0]['last_activity'], issue.last_activity.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
self.assertEqual(len(response.json()[0]['timeline']), 4)
self.assertEqual(response.json()[0]['timeline'][0]['type'], 'mail')
self.assertEqual(response.json()[0]['timeline'][1]['type'], 'state')
self.assertEqual(response.json()[0]['timeline'][2]['type'], 'mail')
self.assertEqual(response.json()[0]['timeline'][3]['type'], 'comment')
self.assertEqual(response.json()[0]['timeline'][0]['id'], mail1.id)
self.assertEqual(response.json()[0]['timeline'][1]['id'], state.id)
self.assertEqual(response.json()[0]['timeline'][2]['id'], mail2.id)
self.assertEqual(response.json()[0]['timeline'][3]['id'], comment.id)
self.assertEqual(response.json()[0]['timeline'][0]['sender'], 'test')
self.assertEqual(response.json()[0]['timeline'][0]['recipient'], 'test')
self.assertEqual(response.json()[0]['timeline'][0]['subject'], 'test')
self.assertEqual(response.json()[0]['timeline'][0]['body'], 'test')
self.assertEqual(response.json()[0]['timeline'][0]['timestamp'],
mail1.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
self.assertEqual(response.json()[0]['timeline'][1]['state'], 'new')
self.assertEqual(response.json()[0]['timeline'][1]['timestamp'],
state.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
self.assertEqual(response.json()[0]['timeline'][2]['sender'], 'test')
self.assertEqual(response.json()[0]['timeline'][2]['recipient'], 'test')
self.assertEqual(response.json()[0]['timeline'][2]['subject'], 'test')
self.assertEqual(response.json()[0]['timeline'][2]['body'], 'test')
self.assertEqual(response.json()[0]['timeline'][2]['timestamp'],
mail2.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
self.assertEqual(response.json()[0]['timeline'][3]['comment'], 'test')
self.assertEqual(response.json()[0]['timeline'][3]['timestamp'],
comment.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
def test_manual_creation(self):
response = self.client.post('/api/2/tickets/manual/', {'name': 'test issue', 'sender': 'test',
'recipient': 'test', 'body': 'test'})
self.assertEqual(response.status_code, 201)
self.assertEqual(response.json()['state'], 'new')
self.assertEqual(response.json()['name'], 'test issue')
self.assertEqual(response.json()['assigned_to'], None)
timeline = response.json()['timeline']
self.assertEqual(len(timeline), 1)
self.assertEqual(timeline[0]['type'], 'mail')
self.assertEqual(timeline[0]['sender'], 'test')
self.assertEqual(timeline[0]['recipient'], 'test')
self.assertEqual(timeline[0]['subject'], 'test issue')
self.assertEqual(timeline[0]['body'], 'test')