c3lf/c3lf-system-3
9
0
Fork 0
Code Issues 55 Pull requests Projects 2 Releases Wiki Activity Actions

parse and save email attachments

Browse source
This commit is contained in:
j3d1 2024-01-09 22:42:47 +01:00
parent f9a95317a2
commit 734af10525
8 changed files with 357 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

59
core/mail/migrations/0003_emailattachment.py Normal file
View file

@ -0,0 +1,59 @@
# Generated by Django 4.2.7 on 2024-01-09 20:56
from django.db import migrations, models
import django.db.models.deletion
import files.models
from mail.models import Email
from mail.protocol import parse_email_body
class NullLogger:
def info(self, *args, **kwargs):
pass
def warning(self, *args, **kwargs):
pass
def debug(self, *args, **kwargs):
pass
class Migration(migrations.Migration):
dependencies = [
('mail', '0002_printed_quotable'),
]
def generate_email_attachments(apps, schema_editor):
for email in Email.objects.all():
raw = email.raw
if raw is None:
continue
parsed, body, attachments = parse_email_body(raw.encode('utf-8'), NullLogger())
email.attachments.clear()
for attachment in attachments:
email.attachments.add(attachment)
email.body = body
email.save()
operations = [
migrations.CreateModel(
name='EmailAttachment',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(blank=True, null=True)),
('updated_at', models.DateTimeField(blank=True, null=True)),
('deleted_at', models.DateTimeField(blank=True, null=True)),
('file', models.ImageField(upload_to=files.models.hash_upload)),
('mime_type', models.CharField(max_length=255)),
('hash', models.CharField(max_length=64, unique=True)),
('name', models.CharField(max_length=255)),
('email',
models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='attachments',
to='mail.email')),
],
options={
'abstract': False,
},
),
migrations.RunPython(generate_email_attachments),
]

6
core/mail/models.py
View file

@ -4,6 +4,7 @@ from django.db import models
from django_softdelete.models import SoftDeleteModel
from core.settings import MAIL_DOMAIN
from files.models import AbstractFile
from inventory.models import Event
from tickets.models import IssueThread
@ -32,3 +33,8 @@ class EventAddress(models.Model):
id = models.AutoField(primary_key=True)
event = models.ForeignKey(Event, models.SET_NULL, null=True)
address = models.CharField(max_length=255)
class EmailAttachment(AbstractFile):
email = models.ForeignKey(Email, models.CASCADE, related_name='attachments', null=True)
name = models.CharField(max_length=255)

151
core/mail/protocol.py
View file

@ -3,10 +3,11 @@ import logging
import aiosmtplib
from asgiref.sync import sync_to_async
from channels.layers import get_channel_layer
from django.core.files.base import ContentFile
from mail.models import Email, EventAddress
from mail.models import Email, EventAddress, EmailAttachment
from notify_sessions.models import SystemEvent
from tickets.models import IssueThread, StateChange
from tickets.models import IssueThread
def find_quoted_printable(s, marker):
@ -99,6 +100,96 @@ def find_target_event(address):
pass
return None
def parse_email_body(raw, log=None):
import email
from hashlib import sha256
attachments = []
parsed = email.message_from_bytes(raw)
body = ""
if parsed.is_multipart():
for part in parsed.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
# skip any text/plain (txt) attachments
if ctype == 'text/plain' and 'attachment' not in cdispo:
segment = part.get_payload(decode=True).decode('utf-8')
segment = unescape_and_decode_quoted_printable(segment)
segment = unescape_and_decode_base64(segment)
log.debug(segment)
body = body + segment
elif 'attachment' in cdispo or 'inline' in cdispo:
file = ContentFile(part.get_payload(decode=True))
chash = sha256(file.read()).hexdigest()
name = part.get_filename()
if name is None:
name = "unnamed"
attachment, _ = EmailAttachment.objects.get_or_create(
name=name, mime_type=ctype, file=file, hash=chash)
attachment.save()
attachments.append(attachment)
if 'inline' in cdispo:
body = body + f'<img src="cid:{attachment.id}">'
log.info("Image", ctype, attachment.id)
else:
log.info("Attachment", ctype, cdispo)
else:
body = parsed.get_payload(decode=True).decode('utf-8')
return parsed, body, attachments
def receive_email(envelope, log=None):
parsed, body, attachments = parse_email_body(envelope.content, log)
header_from = parsed.get('From')
header_to = parsed.get('To')
header_in_reply_to = parsed.get('In-Reply-To')
header_message_id = parsed.get('Message-ID')
if header_from != envelope.mail_from:
log.warning("Header from does not match envelope from")
log.info(f"Header from: {header_from}, envelope from: {envelope.mail_from}")
if header_to != envelope.rcpt_tos[0]:
log.warning("Header to does not match envelope to")
log.info(f"Header to: {header_to}, envelope to: {envelope.rcpt_tos[0]}")
recipient = envelope.rcpt_tos[0].lower()
sender = envelope.mail_from
subject = parsed.get('Subject')
subject = unescape_and_decode_quoted_printable(subject)
subject = unescape_and_decode_base64(subject)
target_event = find_target_event(recipient)
active_issue_thread, new = find_active_issue_thread(header_in_reply_to, subject)
body_decoded = body
body_decoded = unescape_and_decode_quoted_printable(body_decoded)
body_decoded = unescape_and_decode_base64(body_decoded)
email = Email.objects.create(
sender=sender, recipient=recipient, body=body_decoded, subject=subject, reference=header_message_id,
in_reply_to=header_in_reply_to, raw=envelope.content.decode('utf-8'), event=target_event,
issue_thread=active_issue_thread)
for attachment in attachments:
email.attachments.add(attachment)
email.save()
reply = None
if new:
references = collect_references(active_issue_thread)
reply_email = Email.objects.create(
sender=recipient, recipient=sender, body="Thank you for your message.", subject="Message received",
in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread)
reply = make_reply(reply_email, references)
return email, new, reply
class LMTPHandler:
async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
from core.settings import MAIL_DOMAIN
@ -109,7 +200,6 @@ class LMTPHandler:
return '250 OK'
async def handle_DATA(self, server, session, envelope):
import email
log = logging.getLogger('mail.log')
log.setLevel(logging.DEBUG)
log.info('Message from %s' % envelope.mail_from)
@ -117,51 +207,7 @@ class LMTPHandler:
log.info('Message data:\n')
try:
parsed = email.message_from_bytes(envelope.content)
body = ""
if parsed.is_multipart():
for part in parsed.walk():
ctype = part.get_content_type()
cdispo = str(part.get('Content-Disposition'))
# skip any text/plain (txt) attachments
if ctype == 'text/plain' and 'attachment' not in cdispo:
body = part.get_payload(decode=True)
else:
log.info("Attachment", ctype, cdispo)
else:
body = parsed.get_payload(decode=True)
log.info(body)
header_from = parsed.get('From')
header_to = parsed.get('To')
header_in_reply_to = parsed.get('In-Reply-To')
header_message_id = parsed.get('Message-ID')
if header_from != envelope.mail_from:
log.warning("Header from does not match envelope from")
log.info(f"Header from: {header_from}, envelope from: {envelope.mail_from}")
if header_to != envelope.rcpt_tos[0]:
log.warning("Header to does not match envelope to")
log.info(f"Header to: {header_to}, envelope to: {envelope.rcpt_tos[0]}")
recipient = envelope.rcpt_tos[0].lower()
sender = envelope.mail_from
subject = parsed.get('Subject')
subject = unescape_and_decode_quoted_printable(subject)
subject = unescape_and_decode_base64(subject)
target_event = await sync_to_async(find_target_event)(recipient)
active_issue_thread, new = await sync_to_async(find_active_issue_thread)(header_in_reply_to, subject)
body_decoded = body.decode('utf-8')
body_decoded = unescape_and_decode_quoted_printable(body_decoded)
body_decoded = unescape_and_decode_base64(body_decoded)
email = await sync_to_async(Email.objects.create)(
sender=sender, recipient=recipient, body=body_decoded, subject=subject, reference=header_message_id,
in_reply_to=header_in_reply_to, raw=envelope.content.decode('utf-8'), event=target_event,
issue_thread=active_issue_thread)
email, new, reply = await sync_to_async(receive_email)(envelope, log)
log.info(f"Created email {email.id}")
systemevent = await sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id)
log.info(f"Created system event {systemevent.id}")
@ -172,15 +218,10 @@ class LMTPHandler:
)
log.info(f"Sent message to frontend")
if new:
references = await sync_to_async(collect_references)(active_issue_thread)
reply_email = await sync_to_async(Email.objects.create)(
sender=recipient, recipient=sender, body="Thank you for your message.", subject="Message received",
in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread)
await send_smtp(make_reply(reply_email, references), log)
await send_smtp(reply, log)
log.info("Sent auto reply")
return '250 Message accepted for delivery'
except Exception as e:
log.error(e)
return '550 Message rejected'
return '451 Internal server error'

140
core/mail/tests/v2/test_mails.py
View file

@ -8,7 +8,7 @@ from knox.models import AuthToken
from authentication.models import ExtendedUser
from core.settings import MAIL_DOMAIN
from inventory.models import Event
from mail.models import Email, EventAddress
from mail.models import Email, EventAddress, EmailAttachment
from mail.protocol import LMTPHandler
from tickets.models import IssueThread, StateChange
@ -301,3 +301,141 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test
states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0])
self.assertEqual(1, len(states))
self.assertEqual('pending_new', states[0].state)
def test_split_text_inline_image(self):
from aiosmtpd.smtp import Envelope
from asgiref.sync import async_to_sync
import aiosmtplib
aiosmtplib.send = make_mocked_coro()
handler = LMTPHandler()
server = mock.Mock()
session = mock.Mock()
envelope = Envelope()
envelope.mail_from = 'test1@test'
envelope.rcpt_tos = ['test2@test']
envelope.content = b'''Subject: test
From: test1@test
To: test2@test
Message-ID: <1@test>
Content-Type: multipart/alternative; boundary="abc"
--abc
Content-Type: text/plain; charset=utf-8
test1
--abc
Content-Type: image/jpeg; name="test.jpg"
Content-Disposition: inline; filename="test.jpg"
Content-Transfer-Encoding: base64
Content-ID: <1>
X-Attachment-Id: 1
dGVzdGltYWdl
--abc
Content-Type: text/plain; charset=utf-8
test2
--abc--'''
result = async_to_sync(handler.handle_DATA)(server, session, envelope)
self.assertEqual(result, '250 Message accepted for delivery')
self.assertEqual(len(Email.objects.all()), 2)
self.assertEqual(len(IssueThread.objects.all()), 1)
aiosmtplib.send.assert_called_once()
self.assertEqual('test', Email.objects.all()[0].subject)
self.assertEqual('test1@test', Email.objects.all()[0].sender)
self.assertEqual('test2@test', Email.objects.all()[0].recipient)
self.assertEqual('test1\n<img src="cid:1">test2\n', Email.objects.all()[0].body)
self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread)
self.assertEqual('<1@test>', Email.objects.all()[0].reference)
self.assertEqual(None, Email.objects.all()[0].in_reply_to)
self.assertEqual('Message received', Email.objects.all()[1].subject)
self.assertEqual('test2@test', Email.objects.all()[1].sender)
self.assertEqual('test1@test', Email.objects.all()[1].recipient)
self.assertEqual('Thank you for your message.', Email.objects.all()[1].body)
self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread)
self.assertTrue(Email.objects.all()[1].reference.startswith("<"))
self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>"))
self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to)
self.assertEqual('test', IssueThread.objects.all()[0].name)
self.assertEqual('pending_new', IssueThread.objects.all()[0].state)
self.assertEqual(None, IssueThread.objects.all()[0].assigned_to)
states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0])
self.assertEqual(1, len(states))
self.assertEqual('pending_new', states[0].state)
self.assertEqual(1, len(EmailAttachment.objects.all()))
self.assertEqual(1, EmailAttachment.objects.all()[0].id)
self.assertEqual('image/jpeg', EmailAttachment.objects.all()[0].mime_type)
self.assertEqual('test.jpg', EmailAttachment.objects.all()[0].name)
file_content = EmailAttachment.objects.all()[0].file.read()
self.assertEqual(b'testimage', file_content)
def test_text_with_attachment(self):
from aiosmtpd.smtp import Envelope
from asgiref.sync import async_to_sync
import aiosmtplib
aiosmtplib.send = make_mocked_coro()
handler = LMTPHandler()
server = mock.Mock()
session = mock.Mock()
envelope = Envelope()
envelope.mail_from = 'test1@test'
envelope.rcpt_tos = ['test2@test']
envelope.content = b'''Subject: test
From: test1@test
To: test2@test
Message-ID: <1@test>
Content-Type: multipart/mixed; boundary="abc"
--abc
Content-Type: text/plain; charset=utf-8
test1
--abc
Content-Type: image/jpeg; name="test.jpg"
Content-Disposition: attachment; filename="test.jpg"
Content-Transfer-Encoding: base64
Content-ID: <1>
X-Attachment-Id: 1
dGVzdGltYWdl
--abc--'''
result = async_to_sync(handler.handle_DATA)(server, session, envelope)
self.assertEqual(result, '250 Message accepted for delivery')
self.assertEqual(len(Email.objects.all()), 2)
self.assertEqual(len(IssueThread.objects.all()), 1)
aiosmtplib.send.assert_called_once()
self.assertEqual('test', Email.objects.all()[0].subject)
self.assertEqual('test1@test', Email.objects.all()[0].sender)
self.assertEqual('test2@test', Email.objects.all()[0].recipient)
self.assertEqual('test1\n', Email.objects.all()[0].body)
self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread)
self.assertEqual('<1@test>', Email.objects.all()[0].reference)
self.assertEqual(None, Email.objects.all()[0].in_reply_to)
self.assertEqual('Message received', Email.objects.all()[1].subject)
self.assertEqual('test2@test', Email.objects.all()[1].sender)
self.assertEqual('test1@test', Email.objects.all()[1].recipient)
self.assertEqual('Thank you for your message.', Email.objects.all()[1].body)
self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread)
self.assertTrue(Email.objects.all()[1].reference.startswith("<"))
self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>"))
self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to)
self.assertEqual('test', IssueThread.objects.all()[0].name)
self.assertEqual('pending_new', IssueThread.objects.all()[0].state)
self.assertEqual(None, IssueThread.objects.all()[0].assigned_to)
states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0])
self.assertEqual(1, len(states))
self.assertEqual('pending_new', states[0].state)
self.assertEqual(1, len(EmailAttachment.objects.all()))
self.assertEqual(1, EmailAttachment.objects.all()[0].id)
self.assertEqual('image/jpeg', EmailAttachment.objects.all()[0].mime_type)
self.assertEqual('test.jpg', EmailAttachment.objects.all()[0].name)
file_content = EmailAttachment.objects.all()[0].file.read()
self.assertEqual(b'testimage', file_content)