import logging from re import match import aiosmtplib from channels.layers import get_channel_layer from channels.db import database_sync_to_async from django.core.files.base import ContentFile from mail.models import Email, EventAddress, EmailAttachment from notifications.templates import render_auto_reply from notify_sessions.models import SystemEvent from tickets.models import IssueThread class SpecialMailException(Exception): pass def find_quoted_printable(s, marker): positions = [i for i in range(len(s)) if s.lower().startswith('=?utf-8?' + marker + '?', i)] for pos in positions: end = s.find('?=', pos + 10) if end == -1: continue yield pos, end + 3 def unescape_and_decode_quoted_printable(s): import quopri decoded = '' last_end = 0 for start, end in find_quoted_printable(s, 'q'): decoded += s[last_end:start] decoded += quopri.decodestring(s[start + 10:end - 3]).decode('utf-8') last_end = end decoded += s[last_end:] return decoded def unescape_and_decode_base64(s): import base64 decoded = '' last_end = 0 for start, end in find_quoted_printable(s, 'b'): decoded += s[last_end:start] decoded += base64.b64decode(s[start + 10:end - 3]).decode('utf-8') last_end = end decoded += s[last_end:] return decoded def unescape_simplified_quoted_printable(s): import quopri return quopri.decodestring(s).decode('utf-8') def collect_references(issue_thread): mails = issue_thread.emails.order_by('timestamp') references = [] for mail in mails: if mail.reference: references.append(mail.reference) return references def make_reply(reply_email, references=None, event=None): from email.message import EmailMessage from core.settings import MAIL_DOMAIN event = event or "mail" reply = EmailMessage() reply["From"] = reply_email.sender reply["To"] = reply_email.recipient reply["Subject"] = reply_email.subject reply["Reply-To"] = f"{event}@{MAIL_DOMAIN}" if reply_email.in_reply_to: reply["In-Reply-To"] = reply_email.in_reply_to if reply_email.reference: reply["Message-ID"] = reply_email.reference else: reply["Message-ID"] = reply_email.id + "@" + MAIL_DOMAIN reply_email.reference = reply["Message-ID"] reply_email.save() if references: reply["References"] = " ".join(references) reply.set_content(reply_email.body) return reply def make_notification(message, to, title): # TODO where should replies to this go from email.message import EmailMessage from core.settings import MAIL_DOMAIN notification = EmailMessage() notification["From"] = "notifications@%s" % MAIL_DOMAIN notification["To"] = to notification["Subject"] = f"[C3LF Notification]%s" % title # notification["Reply-To"] = f"{event}@{MAIL_DOMAIN}" # notification["In-Reply-To"] = email.reference # notification["Message-ID"] = email.id + "@" + MAIL_DOMAIN notification.set_content(message) return notification async def send_smtp(message): await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False) def find_active_issue_thread(in_reply_to, address, subject): from re import match uuid_match = match(r'^ticket\+([a-f0-9-]{36})@', address) if uuid_match: issue = IssueThread.objects.filter(uuid=uuid_match.group(1)) if issue.exists(): return issue.first(), False reply_to = Email.objects.filter(reference=in_reply_to) if reply_to.exists(): return reply_to.first().issue_thread, False else: issue = IssueThread.objects.create(name=subject) return issue, True def find_target_event(address): try: address_map = EventAddress.objects.get(address=address) if address_map.event: return address_map.event except EventAddress.DoesNotExist: pass return None def parse_email_body(raw, log=None): import email from hashlib import sha256 attachments = [] parsed = email.message_from_bytes(raw) body = "" if parsed.is_multipart(): for part in parsed.walk(): ctype = part.get_content_type() cdispo = str(part.get('Content-Disposition')) # skip any text/plain (txt) attachments if ctype == 'text/plain' and 'attachment' not in cdispo: segment = part.get_payload() if not segment: continue segment = unescape_and_decode_quoted_printable(segment) segment = unescape_and_decode_base64(segment) if part.get('Content-Transfer-Encoding') == 'quoted-printable': segment = unescape_simplified_quoted_printable(segment) log.debug(segment) body = body + segment elif 'attachment' in cdispo or 'inline' in cdispo: file = ContentFile(part.get_payload(decode=True)) chash = sha256(file.read()).hexdigest() name = part.get_filename() if name is None: name = "unnamed" attachment, _ = EmailAttachment.objects.get_or_create( name=name, mime_type=ctype, file=file, hash=chash) attachment.save() attachments.append(attachment) if 'inline' in cdispo: body = body + f'' log.info("Image %s %s", ctype, attachment.id) else: log.info("Attachment %s %s", ctype, cdispo) else: if parsed.get_content_type() == 'text/plain': body = parsed.get_payload() elif parsed.get_content_type() == 'text/html': from bs4 import BeautifulSoup import re body = parsed.get_payload() soup = BeautifulSoup(body, 'html.parser') body = re.sub(r'([\r\n]+.?)*[\r\n]', r'\n', soup.get_text()).strip('\n') else: log.warning("Unknown content type %s", parsed.get_content_type()) body = "Unknown content type" body = unescape_and_decode_quoted_printable(body) body = unescape_and_decode_base64(body) if parsed.get('Content-Transfer-Encoding') == 'quoted-printable': body = unescape_simplified_quoted_printable(body) log.debug(body) return parsed, body, attachments @database_sync_to_async def receive_email(envelope, log=None): parsed, body, attachments = parse_email_body(envelope.content, log) header_from = parsed.get('From') header_to = parsed.get('To') header_in_reply_to = parsed.get('In-Reply-To') header_message_id = parsed.get('Message-ID') # if header_from != envelope.mail_from: # log.warning("Header from does not match envelope from") # log.info(f"Header from: {header_from}, envelope from: {envelope.mail_from}") # # if header_to != envelope.rcpt_tos[0]: # log.warning("Header to does not match envelope to") # log.info(f"Header to: {header_to}, envelope to: {envelope.rcpt_tos[0]}") # handle undelivered mail header_from : 'Mail Delivery System ") == "": log.warning("Ignoring mailer daemon") raise SpecialMailException("Ignoring mailer daemon") if Email.objects.filter(reference=header_message_id).exists(): # break before issue thread is created log.warning("Email already exists") raise Exception("Email already exists") recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower() sender = envelope.mail_from if envelope.mail_from else header_from subject = parsed.get('Subject') if not subject: subject = "No subject" subject = unescape_and_decode_quoted_printable(subject) subject = unescape_and_decode_base64(subject) target_event = find_target_event(recipient) active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject) email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) email.save() reply = None if new: # auto reply if new issue references = collect_references(active_issue_thread) if not sender.startswith('noreply'): subject = f"Re: {subject} [#{active_issue_thread.short_uuid()}]" body = render_auto_reply(active_issue_thread) reply_email = Email.objects.create( sender=recipient, recipient=sender, body=body, subject=subject, in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) reply = make_reply(reply_email, references, event=target_event.slug if target_event else None) else: # change state if not new if active_issue_thread.state != 'pending_new': active_issue_thread.state = 'pending_open' active_issue_thread.save() return email, new, reply, active_issue_thread class LMTPHandler: async def handle_RCPT(self, server, session, envelope, address, rcpt_options): from core.settings import MAIL_DOMAIN address = address.lower() if not address.endswith('@' + MAIL_DOMAIN): return '550 not relaying to that domain' envelope.rcpt_tos.append(address) return '250 OK' async def handle_DATA(self, server, session, envelope): log = logging.getLogger('mail.log') log.setLevel(logging.DEBUG) log.info('Message from %s' % envelope.mail_from) log.info('Message for %s' % envelope.rcpt_tos) log.info('Message data:\n') content = None try: content = envelope.content email, new, reply, thread = await receive_email(envelope, log) log.info(f"Created email {email.id}") systemevent = await database_sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id) log.info(f"Created system event {systemevent.id}") channel_layer = get_channel_layer() await channel_layer.group_send( 'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id, "message": "email received"}) log.info(f"Sent message to frontend") if new and reply: log.info('Sending message to %s' % reply['To']) await send_smtp(reply) log.info("Sent auto reply") if thread: await channel_layer.group_send( 'general', {"type": "generic.event", "name": "user_notification", "event_id": systemevent.id, "ticket_id": thread.id, "new": new}) else: print("No thread found") return '250 Message accepted for delivery' except SpecialMailException as e: import uuid random_filename = 'special-' + str(uuid.uuid4()) with open(random_filename, 'wb') as f: f.write(content) log.warning(f"Special mail exception: {e} saved to {random_filename}") return '250 Message accepted for delivery' except Exception as e: from hashlib import sha256 random_filename = 'mail-' + sha256(content).hexdigest() with open(random_filename, 'wb') as f: f.write(content) log.error(f"Saved email to {random_filename} because of error %s (%s)", e, type(e)) return '451 Internal server error'