c3lf-system-3/core/mail/protocol.py

import logging
from re import match

import aiosmtplib
from channels.layers import get_channel_layer
from channels.db import database_sync_to_async
from django.core.files.base import ContentFile

from mail.models import Email, EventAddress, EmailAttachment
from notify_sessions.models import SystemEvent
from tickets.models import IssueThread


class SpecialMailException(Exception):
    pass


def find_quoted_printable(s, marker):
    positions = [i for i in range(len(s)) if s.lower().startswith('=?utf-8?' + marker + '?', i)]
    for pos in positions:
        end = s.find('?=', pos + 10)
        if end == -1:
            continue
        yield pos, end + 3


def unescape_and_decode_quoted_printable(s):
    import quopri
    decoded = ''
    last_end = 0
    for start, end in find_quoted_printable(s, 'q'):
        decoded += s[last_end:start]
        decoded += quopri.decodestring(s[start + 10:end - 3]).decode('utf-8')
        last_end = end
    decoded += s[last_end:]
    return decoded


def unescape_and_decode_base64(s):
    import base64
    decoded = ''
    last_end = 0
    for start, end in find_quoted_printable(s, 'b'):
        decoded += s[last_end:start]
        decoded += base64.b64decode(s[start + 10:end - 3]).decode('utf-8')
        last_end = end
    decoded += s[last_end:]
    return decoded


def unescape_simplified_quoted_printable(s):
    import quopri
    return quopri.decodestring(s).decode('utf-8')


def collect_references(issue_thread):
    mails = issue_thread.emails.order_by('timestamp')
    references = []
    for mail in mails:
        if mail.reference:
            references.append(mail.reference)
    return references


def make_reply(reply_email, references=None, event=None):
    from email.message import EmailMessage
    from core.settings import MAIL_DOMAIN
    event = event or "mail"
    reply = EmailMessage()
    reply["From"] = reply_email.sender
    reply["To"] = reply_email.recipient
    reply["Subject"] = reply_email.subject
    reply["Reply-To"] = f"{event}@{MAIL_DOMAIN}"
    if reply_email.in_reply_to:
        reply["In-Reply-To"] = reply_email.in_reply_to
    if reply_email.reference:
        reply["Message-ID"] = reply_email.reference
    else:
        reply["Message-ID"] = reply_email.id + "@" + MAIL_DOMAIN
        reply_email.reference = reply["Message-ID"]
        reply_email.save()
    if references:
        reply["References"] = " ".join(references)

    reply.set_content(reply_email.body)

    return reply


async def send_smtp(message):
    await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False)


def find_active_issue_thread(in_reply_to, address, subject):
    from re import match
    uuid_match = match(r'^ticket\+([a-f0-9-]{36})@', address)
    if uuid_match:
        issue = IssueThread.objects.filter(uuid=uuid_match.group(1))
        if issue.exists():
            return issue.first(), False
    reply_to = Email.objects.filter(reference=in_reply_to)
    if reply_to.exists():
        return reply_to.first().issue_thread, False
    else:
        issue = IssueThread.objects.create(name=subject)
        return issue, True


def find_target_event(address):
    try:
        address_map = EventAddress.objects.get(address=address)
        if address_map.event:
            return address_map.event
    except EventAddress.DoesNotExist:
        pass
    return None


def parse_email_body(raw, log=None):
    import email
    from hashlib import sha256

    attachments = []

    parsed = email.message_from_bytes(raw)
    body = ""
    if parsed.is_multipart():
        for part in parsed.walk():
            ctype = part.get_content_type()
            cdispo = str(part.get('Content-Disposition'))

            # skip any text/plain (txt) attachments
            if ctype == 'text/plain' and 'attachment' not in cdispo:
                segment = part.get_payload()
                if not segment:
                    continue
                segment = unescape_and_decode_quoted_printable(segment)
                segment = unescape_and_decode_base64(segment)
                if part.get('Content-Transfer-Encoding') == 'quoted-printable':
                    segment = unescape_simplified_quoted_printable(segment)
                log.debug(segment)
                body = body + segment
            elif 'attachment' in cdispo or 'inline' in cdispo:
                file = ContentFile(part.get_payload(decode=True))
                chash = sha256(file.read()).hexdigest()
                name = part.get_filename()
                if name is None:
                    name = "unnamed"
                attachment, _ = EmailAttachment.objects.get_or_create(
                    name=name, mime_type=ctype, file=file, hash=chash)
                attachment.save()
                attachments.append(attachment)
                if 'inline' in cdispo:
                    body = body + f'<img src="cid:{attachment.id}">'
                log.info("Image %s %s", ctype, attachment.id)
            else:
                log.info("Attachment %s %s", ctype, cdispo)
    else:
        if parsed.get_content_type() == 'text/plain':
            body = parsed.get_payload()
        elif parsed.get_content_type() == 'text/html':
            from bs4 import BeautifulSoup
            import re
            body = parsed.get_payload()
            soup = BeautifulSoup(body, 'html.parser')
            body = re.sub(r'([\r\n]+.?)*[\r\n]', r'\n', soup.get_text()).strip('\n')
        else:
            log.warning("Unknown content type %s", parsed.get_content_type())
            body = "Unknown content type"
        body = unescape_and_decode_quoted_printable(body)
        body = unescape_and_decode_base64(body)
        if parsed.get('Content-Transfer-Encoding') == 'quoted-printable':
            body = unescape_simplified_quoted_printable(body)
        log.debug(body)

    return parsed, body, attachments


@database_sync_to_async
def receive_email(envelope, log=None):
    parsed, body, attachments = parse_email_body(envelope.content, log)

    header_from = parsed.get('From')
    header_to = parsed.get('To')
    header_in_reply_to = parsed.get('In-Reply-To')
    header_message_id = parsed.get('Message-ID')

    if match(r'^([a-zA-Z ]*<)?MAILER-DAEMON@', header_from) and envelope.mail_from.strip("<>") == "":
        log.warning("Ignoring mailer daemon")
        raise SpecialMailException("Ignoring mailer daemon")

    if Email.objects.filter(reference=header_message_id).exists():  # break before issue thread is created
        log.warning("Email already exists")
        raise Exception("Email already exists")

    recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower()
    sender = envelope.mail_from if envelope.mail_from else header_from
    subject = parsed.get('Subject')
    if not subject:
        subject = "No subject"
    subject = unescape_and_decode_quoted_printable(subject)
    subject = unescape_and_decode_base64(subject)
    log.info("search for {}".format(recipient))
    log.info("known {}".format([x.address for x in EventAddress.objects.all()]))
    target_event = find_target_event(recipient)

    active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject)

    email = Email.objects.create(
        sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id,
        in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content), event=target_event,
        issue_thread=active_issue_thread)
    for attachment in attachments:
        email.attachments.add(attachment)
    email.save()

    reply = None
    if new:
        # auto reply if new issue
        references = collect_references(active_issue_thread)
        if not sender.startswith('noreply'):
            subject = f"Re: {subject} [#{active_issue_thread.short_uuid()}]"
            body = '''Your request (#{}) has been received and will be reviewed by our lost&found angels.

We are reviewing incoming requests during the event and teardown. Immediately after the event, expect a delay as the \
workload is high. We will not forget about your request and get back in touch once we have updated information on your \
request. Requests for devices, wallets, credit cards or similar items will be handled with priority.

If you happen to find your lost item or just want to add additional information, please reply to this email. Please \
do not create a new request.

Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid())
            reply_email = Email.objects.create(
                sender=recipient, recipient=sender, body=body, subject=subject,
                in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread)
            reply = make_reply(reply_email, references, event=target_event.slug if target_event else None)
    else:
        # change state if not new
        if active_issue_thread.state != 'pending_new':
            active_issue_thread.state = 'pending_open'
            active_issue_thread.save()

    return email, new, reply, active_issue_thread


class LMTPHandler:
    async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
        from core.settings import MAIL_DOMAIN
        address = address.lower()
        if not address.endswith('@' + MAIL_DOMAIN):
            return '550 not relaying to that domain'
        envelope.rcpt_tos.append(address)
        return '250 OK'

    async def handle_DATA(self, server, session, envelope):
        log = logging.getLogger('mail.log')
        log.setLevel(logging.DEBUG)
        log.info('Message from %s' % envelope.mail_from)
        log.info('Message for %s' % envelope.rcpt_tos)
        log.info('Message data:\n')

        content = None
        try:
            content = envelope.content
            email, new, reply, thread = await receive_email(envelope, log)
            log.info(f"Created email {email.id}")
            systemevent = await database_sync_to_async(SystemEvent.objects.create)(type='email received',
                                                                                   reference=email.id)
            log.info(f"Created system event {systemevent.id}")
            channel_layer = get_channel_layer()
            await channel_layer.group_send(
                'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id,
                            "message": "email received"})
            log.info(f"Sent message to frontend")
            if new and reply:
                log.info('Sending message to %s' % reply['To'])
                await send_smtp(reply)
                log.info("Sent auto reply")

            return '250 Message accepted for delivery'
        except SpecialMailException as e:
            import uuid
            random_filename = 'special-' + str(uuid.uuid4())
            with open(random_filename, 'wb') as f:
                f.write(content)
            log.warning(f"Special mail exception: {e} saved to {random_filename}")
            return '250 Message accepted for delivery'
        except Exception as e:
            from hashlib import sha256
            random_filename = 'mail-' + sha256(content).hexdigest()
            with open(random_filename, 'wb') as f:
                f.write(content)
            log.error(f"Saved email to {random_filename} because of error %s (%s)", e, type(e))
            return '451 Internal server error'