import logging from re import match import aiosmtplib from channels.layers import get_channel_layer from channels.db import database_sync_to_async from django.core.files.base import ContentFile from mail.models import Email, EventAddress, EmailAttachment from notify_sessions.models import SystemEvent from tickets.models import IssueThread class SpecialMailException(Exception): pass def find_quoted_printable(s, marker): positions = [i for i in range(len(s)) if s.lower().startswith('=?utf-8?' + marker + '?', i)] for pos in positions: end = s.find('?=', pos + 10) if end == -1: continue yield pos, end + 3 def unescape_and_decode_quoted_printable(s): import quopri decoded = '' last_end = 0 for start, end in find_quoted_printable(s, 'q'): decoded += s[last_end:start] decoded += quopri.decodestring(s[start + 10:end - 3]).decode('utf-8') last_end = end decoded += s[last_end:] return decoded def unescape_and_decode_base64(s): import base64 decoded = '' last_end = 0 for start, end in find_quoted_printable(s, 'b'): decoded += s[last_end:start] decoded += base64.b64decode(s[start + 10:end - 3]).decode('utf-8') last_end = end decoded += s[last_end:] return decoded def unescape_simplified_quoted_printable(s): import quopri return quopri.decodestring(s).decode('utf-8') def collect_references(issue_thread): mails = issue_thread.emails.order_by('timestamp') references = [] for mail in mails: if mail.reference: references.append(mail.reference) return references def make_reply(reply_email, references=None, event=None): from email.message import EmailMessage from core.settings import MAIL_DOMAIN event = event or "mail" reply = EmailMessage() reply["From"] = reply_email.sender reply["To"] = reply_email.recipient reply["Subject"] = reply_email.subject reply["Reply-To"] = f"{event}@{MAIL_DOMAIN}" if reply_email.in_reply_to: reply["In-Reply-To"] = reply_email.in_reply_to if reply_email.reference: reply["Message-ID"] = reply_email.reference else: reply["Message-ID"] = reply_email.id + "@" + MAIL_DOMAIN reply_email.reference = reply["Message-ID"] reply_email.save() if references: reply["References"] = " ".join(references) reply.set_content(reply_email.body) return reply async def send_smtp(message): await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False) def find_active_issue_thread(in_reply_to, address, subject, event): from re import match uuid_match = match(r'^ticket\+([a-f0-9-]{36})@', address) if uuid_match: issue = IssueThread.objects.filter(uuid=uuid_match.group(1)) if issue.exists(): return issue.first(), False reply_to = Email.objects.filter(reference=in_reply_to) if reply_to.exists(): return reply_to.first().issue_thread, False else: issue = IssueThread.objects.create(name=subject, event=event) return issue, True def find_target_event(address): try: address_map = EventAddress.objects.get(address=address) if address_map.event: return address_map.event except EventAddress.DoesNotExist: pass return None def parse_email_body(raw, log=None): import email from hashlib import sha256 attachments = [] parsed = email.message_from_bytes(raw) body = "" if parsed.is_multipart(): for part in parsed.walk(): ctype = part.get_content_type() cdispo = str(part.get('Content-Disposition')) # skip any text/plain (txt) attachments if ctype == 'text/plain' and 'attachment' not in cdispo: segment = part.get_payload() if not segment: continue segment = unescape_and_decode_quoted_printable(segment) segment = unescape_and_decode_base64(segment) if part.get('Content-Transfer-Encoding') == 'quoted-printable': segment = unescape_simplified_quoted_printable(segment) log.debug(segment) body = body + segment elif 'attachment' in cdispo or 'inline' in cdispo: file = ContentFile(part.get_payload(decode=True)) chash = sha256(file.read()).hexdigest() name = part.get_filename() if name is None: name = "unnamed" attachment, _ = EmailAttachment.objects.get_or_create( name=name, mime_type=ctype, file=file, hash=chash) attachment.save() attachments.append(attachment) if 'inline' in cdispo: body = body + f'' log.info("Image %s %s", ctype, attachment.id) else: log.info("Attachment %s %s", ctype, cdispo) else: if parsed.get_content_type() == 'text/plain': body = parsed.get_payload() elif parsed.get_content_type() == 'text/html': from bs4 import BeautifulSoup import re body = parsed.get_payload() soup = BeautifulSoup(body, 'html.parser') body = re.sub(r'([\r\n]+.?)*[\r\n]', r'\n', soup.get_text()).strip('\n') else: log.warning("Unknown content type %s", parsed.get_content_type()) body = "Unknown content type" body = unescape_and_decode_quoted_printable(body) body = unescape_and_decode_base64(body) if parsed.get('Content-Transfer-Encoding') == 'quoted-printable': body = unescape_simplified_quoted_printable(body) log.debug(body) return parsed, body, attachments @database_sync_to_async def receive_email(envelope, log=None): parsed, body, attachments = parse_email_body(envelope.content, log) header_from = parsed.get('From') header_to = parsed.get('To') header_in_reply_to = parsed.get('In-Reply-To') header_message_id = parsed.get('Message-ID') if match(r'^([a-zA-Z ]*<)?MAILER-DAEMON@', header_from) and envelope.mail_from.strip("<>") == "": log.warning("Ignoring mailer daemon") raise SpecialMailException("Ignoring mailer daemon") if Email.objects.filter(reference=header_message_id).exists(): # break before issue thread is created log.warning("Email already exists") raise Exception("Email already exists") recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower() sender = envelope.mail_from if envelope.mail_from else header_from subject = parsed.get('Subject') if not subject: subject = "No subject" subject = unescape_and_decode_quoted_printable(subject) subject = unescape_and_decode_base64(subject) log.info("search for {}".format(recipient)) log.info("known {}".format(["{} -> {}".format(x.address, x.event.slug) for x in EventAddress.objects.all()])) target_event = find_target_event(recipient) log.info("found {}".format(target_event)) active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event) from hashlib import sha256 random_filename = 'mail-' + sha256(envelope.content).hexdigest() email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) email.save() reply = None if new: # auto reply if new issue references = collect_references(active_issue_thread) if not sender.startswith('noreply'): subject = f"Re: {subject} [#{active_issue_thread.short_uuid()}]" body = '''Your request (#{}) has been received and will be reviewed by our lost&found angels. We are reviewing incoming requests during the event and teardown. Immediately after the event, expect a delay as the \ workload is high. We will not forget about your request and get back in touch once we have updated information on your \ request. Requests for devices, wallets, credit cards or similar items will be handled with priority. If you happen to find your lost item or just want to add additional information, please reply to this email. Please \ do not create a new request. Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid()) reply_email = Email.objects.create( sender=recipient, recipient=sender, body=body, subject=subject, in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) reply = make_reply(reply_email, references, event=target_event.slug if target_event else None) else: # change state if not new if active_issue_thread.state != 'pending_new': active_issue_thread.state = 'pending_open' active_issue_thread.save() return email, new, reply, active_issue_thread class LMTPHandler: async def handle_RCPT(self, server, session, envelope, address, rcpt_options): from core.settings import MAIL_DOMAIN address = address.lower() if not address.endswith('@' + MAIL_DOMAIN): return '550 not relaying to that domain' envelope.rcpt_tos.append(address) return '250 OK' async def handle_DATA(self, server, session, envelope): log = logging.getLogger('mail.log') log.setLevel(logging.DEBUG) log.info('Message from %s' % envelope.mail_from) log.info('Message for %s' % envelope.rcpt_tos) log.info('CTX for {}, {}'.format(server, session)) log.info('Message data:\n') content = None try: content = envelope.content email, new, reply, thread = await receive_email(envelope, log) log.info(f"Created email {email.id}") systemevent = await database_sync_to_async(SystemEvent.objects.create)(type='email received', reference=email.id) log.info(f"Created system event {systemevent.id}") channel_layer = get_channel_layer() await channel_layer.group_send( 'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id, "message": "email received"}) log.info(f"Sent message to frontend") if new and reply: log.info('Sending message to %s' % reply['To']) await send_smtp(reply) log.info("Sent auto reply") return '250 Message accepted for delivery' except SpecialMailException as e: import uuid random_filename = 'special-' + str(uuid.uuid4()) with open(random_filename, 'wb') as f: f.write(content) log.warning(f"Special mail exception: {e} saved to {random_filename}") return '250 Message accepted for delivery' except Exception as e: from hashlib import sha256 random_filename = 'mail-' + sha256(content).hexdigest() with open(random_filename, 'wb') as f: f.write(content) log.error(f"Saved email to {random_filename} because of error %s (%s)", e, type(e)) return '451 Internal server error'