c3lf-system-3/core/mail/protocol.py

import logging
from re import match

import aiosmtplib
from channels.layers import get_channel_layer
from channels.db import database_sync_to_async
from django.core.files.base import ContentFile

from mail.models import Email, EventAddress, EmailAttachment
from notifications.templates import render_auto_reply
from notify_sessions.models import SystemEvent
from tickets.models import IssueThread


class SpecialMailException(Exception):
    pass


def find_quoted_printable(s, marker):
    positions = [i for i in range(len(s)) if s.lower().startswith('=?utf-8?' + marker + '?', i)]
    for pos in positions:
        end = s.find('?=', pos + 10)
        if end == -1:
            continue
        yield pos, end + 3


def unescape_and_decode_quoted_printable(s):
    import quopri
    decoded = ''
    last_end = 0
    for start, end in find_quoted_printable(s, 'q'):
        decoded += s[last_end:start]
        decoded += quopri.decodestring(s[start + 10:end - 3]).decode('utf-8')
        last_end = end
    decoded += s[last_end:]
    return decoded


def unescape_and_decode_base64(s):
    import base64
    decoded = ''
    last_end = 0
    for start, end in find_quoted_printable(s, 'b'):
        decoded += s[last_end:start]
        decoded += base64.b64decode(s[start + 10:end - 3]).decode('utf-8')
        last_end = end
    decoded += s[last_end:]
    return decoded


def unescape_simplified_quoted_printable(s):
    import quopri
    return quopri.decodestring(s).decode('utf-8')


def collect_references(issue_thread):
    mails = issue_thread.emails.order_by('timestamp')
    references = []
    for mail in mails:
        if mail.reference:
            references.append(mail.reference)
    return references


def make_reply(reply_email, references=None, event=None):
    from email.message import EmailMessage
    from core.settings import MAIL_DOMAIN
    event = event or "mail"
    reply = EmailMessage()
    reply["From"] = reply_email.sender
    reply["To"] = reply_email.recipient
    reply["Subject"] = reply_email.subject
    reply["Reply-To"] = f"{event.lower()}@{MAIL_DOMAIN}"
    if reply_email.in_reply_to:
        reply["In-Reply-To"] = reply_email.in_reply_to
    if reply_email.reference:
        reply["Message-ID"] = reply_email.reference
    else:
        reply["Message-ID"] = reply_email.id + "@" + MAIL_DOMAIN
        reply_email.reference = reply["Message-ID"]
        reply_email.save()
    if references:
        reply["References"] = " ".join(references)

    reply.set_content(reply_email.body)

    return reply


def make_notification(message, to, title):  # TODO where should replies to this go
    from email.message import EmailMessage
    from core.settings import MAIL_DOMAIN
    notification = EmailMessage()
    notification["From"] = "notifications@%s" % MAIL_DOMAIN
    notification["To"] = to
    notification["Subject"] = f"[C3LF Notification]%s" % title
    # notification["Reply-To"] = f"{event}@{MAIL_DOMAIN}"
    # notification["In-Reply-To"] = email.reference
    # notification["Message-ID"] = email.id + "@" + MAIL_DOMAIN

    notification.set_content(message)

    return notification


async def send_smtp(message):
    await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False)


def find_active_issue_thread(in_reply_to, address, subject, event):
    from re import match
    uuid_match = match(r'^ticket\+([a-f0-9-]{36})@', address)
    if uuid_match:
        issue = IssueThread.objects.filter(uuid=uuid_match.group(1))
        if issue.exists():
            return issue.first(), False
    reply_to = Email.objects.filter(reference=in_reply_to)
    if reply_to.exists():
        return reply_to.first().issue_thread, False
    else:
        issue = IssueThread.objects.create(name=subject, event=event)
        return issue, True


def find_target_event(address):
    try:
        address_map = EventAddress.objects.get(address=address)
        if address_map.event:
            return address_map.event
    except EventAddress.DoesNotExist:
        pass
    return None


def parse_email_body(raw, log=None):
    import email
    from hashlib import sha256

    attachments = []

    parsed = email.message_from_bytes(raw)
    body = ""
    if parsed.is_multipart():
        for part in parsed.walk():
            ctype = part.get_content_type()
            cdispo = str(part.get('Content-Disposition'))

            # skip any text/plain (txt) attachments
            if ctype == 'text/plain' and 'attachment' not in cdispo:
                segment = part.get_payload()
                if not segment:
                    continue
                segment = unescape_and_decode_quoted_printable(segment)
                segment = unescape_and_decode_base64(segment)
                if part.get('Content-Transfer-Encoding') == 'quoted-printable':
                    segment = unescape_simplified_quoted_printable(segment)
                log.debug(segment)
                body = body + segment
            elif 'attachment' in cdispo or 'inline' in cdispo:
                file = ContentFile(part.get_payload(decode=True))
                chash = sha256(file.read()).hexdigest()
                name = part.get_filename()
                if name is None:
                    name = "unnamed"
                attachment, _ = EmailAttachment.objects.get_or_create(
                    name=name, mime_type=ctype, file=file, hash=chash)
                attachment.save()
                attachments.append(attachment)
                if 'inline' in cdispo:
                    body = body + f'<img src="cid:{attachment.id}">'
                log.info("Image %s %s", ctype, attachment.id)
            else:
                log.info("Attachment %s %s", ctype, cdispo)
    else:
        if parsed.get_content_type() == 'text/plain':
            body = parsed.get_payload()
        elif parsed.get_content_type() == 'text/html':
            from bs4 import BeautifulSoup
            import re
            body = parsed.get_payload()
            soup = BeautifulSoup(body, 'html.parser')
            body = re.sub(r'([\r\n]+.?)*[\r\n]', r'\n', soup.get_text()).strip('\n')
        else:
            log.warning("Unknown content type %s", parsed.get_content_type())
            body = "Unknown content type"
        body = unescape_and_decode_quoted_printable(body)
        body = unescape_and_decode_base64(body)
        if parsed.get('Content-Transfer-Encoding') == 'quoted-printable':
            body = unescape_simplified_quoted_printable(body)
        log.debug(body)

    return parsed, body, attachments


@database_sync_to_async
def receive_email(envelope, log=None):
    parsed, body, attachments = parse_email_body(envelope.content, log)

    header_from = parsed.get('From')
    header_to = parsed.get('To')
    header_in_reply_to = parsed.get('In-Reply-To')
    header_message_id = parsed.get('Message-ID')

    #    if header_from != envelope.mail_from:
    #        log.warning("Header from does not match envelope from")
    #        log.info(f"Header from: {header_from}, envelope from: {envelope.mail_from}")
    #
    #    if header_to != envelope.rcpt_tos[0]:
    #        log.warning("Header to does not match envelope to")
    #        log.info(f"Header to: {header_to}, envelope to: {envelope.rcpt_tos[0]}")

    # handle undelivered mail header_from : 'Mail Delivery System <MAILER-DAEMON@...'

    if match(r'^([a-zA-Z ]*<)?MAILER-DAEMON@', header_from) and envelope.mail_from.strip("<>") == "":
        log.warning("Ignoring mailer daemon")
        raise SpecialMailException("Ignoring mailer daemon")

    if Email.objects.filter(reference=header_message_id).exists():  # break before issue thread is created
        log.warning("Email already exists")
        raise Exception("Email already exists")

    recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower()
    sender = envelope.mail_from if envelope.mail_from else header_from
    subject = parsed.get('Subject')
    if not subject:
        subject = "No subject"
    subject = unescape_and_decode_quoted_printable(subject)
    subject = unescape_and_decode_base64(subject)
    target_event = find_target_event(recipient)

    active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event)

    from hashlib import sha256
    random_filename = 'mail-' + sha256(envelope.content).hexdigest()

    email = Email.objects.create(
        sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id,
        in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event,
        issue_thread=active_issue_thread)
    for attachment in attachments:
        email.attachments.add(attachment)
    email.save()

    reply = None
    if new:
        # auto reply if new issue
        references = collect_references(active_issue_thread)
        if not sender.startswith('noreply'):
            subject = f"Re: {subject} [#{active_issue_thread.short_uuid()}]"
            body = render_auto_reply(active_issue_thread)
            reply_email = Email.objects.create(
                sender=recipient, recipient=sender, body=body, subject=subject,
                in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread)
            reply = make_reply(reply_email, references, event=target_event.slug if target_event else None)
    else:
        # change state if not new
        if active_issue_thread.state != 'pending_new':
            active_issue_thread.state = 'pending_open'
            active_issue_thread.save()

    return email, new, reply, active_issue_thread


class LMTPHandler:
    async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
        from core.settings import MAIL_DOMAIN
        address = address.lower()
        if not address.endswith('@' + MAIL_DOMAIN):
            return '550 not relaying to that domain'
        envelope.rcpt_tos.append(address)
        return '250 OK'

    async def handle_DATA(self, server, session, envelope):
        log = logging.getLogger('mail.log')
        log.setLevel(logging.DEBUG)
        log.info('Message from %s' % envelope.mail_from)
        log.info('Message for %s' % envelope.rcpt_tos)
        log.info('Message data:\n')

        content = None
        try:
            content = envelope.content
            email, new, reply, thread = await receive_email(envelope, log)
            log.info(f"Created email {email.id}")
            systemevent = await database_sync_to_async(SystemEvent.objects.create)(type='email received',
                                                                                   reference=email.id)
            log.info(f"Created system event {systemevent.id}")
            channel_layer = get_channel_layer()
            await channel_layer.group_send(
                'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id,
                            "message": "email received"})
            log.info(f"Sent message to frontend")

            if new and reply:
                log.info('Sending message to %s' % reply['To'])
                await send_smtp(reply)
                log.info("Sent auto reply")

            if thread:
                await channel_layer.group_send(
                    'general', {"type": "generic.event", "name": "user_notification", "event_id": systemevent.id,
                                "ticket_id": thread.id, "new": new})
            else:
                print("No thread found")

            return '250 Message accepted for delivery'
        except SpecialMailException as e:
            import uuid
            random_filename = 'special-' + str(uuid.uuid4())
            with open(random_filename, 'wb') as f:
                f.write(content)
            log.warning(f"Special mail exception: {e} saved to {random_filename}")
            return '250 Message accepted for delivery'
        except Exception as e:
            from hashlib import sha256
            random_filename = 'mail-' + sha256(content).hexdigest()
            with open(random_filename, 'wb') as f:
                f.write(content)
            log.error(f"Saved email to {random_filename} because of error %s (%s)", e, type(e))
            return '451 Internal server error'