From 55aa1ec58fb3de29e498a407007fce2f5b7a3f93 Mon Sep 17 00:00:00 2001 From: jedi Date: Thu, 23 Nov 2023 23:17:20 +0100 Subject: [PATCH] experimental mail transport --- core/core/settings.py | 2 + core/lmtp/__init__.py | 0 core/lmtp/protocol.py | 47 ++++++ core/lmtp/socket.py | 31 ++++ core/notify_sessions/routing.py | 3 +- ...requirements.txt => requirements.prod.txt} | 1 + core/server.py | 134 +++++++++--------- deploy/ansible/inventory.yml.sample | 1 + deploy/ansible/playbooks/deploy-c3lf-sys3.yml | 33 ++++- .../ansible/playbooks/templates/django.env.j2 | 1 + .../ansible/playbooks/templates/postfix.cf.j2 | 50 +++++++ 11 files changed, 235 insertions(+), 68 deletions(-) create mode 100644 core/lmtp/__init__.py create mode 100644 core/lmtp/protocol.py create mode 100644 core/lmtp/socket.py rename core/{requirements.txt => requirements.prod.txt} (97%) create mode 100644 deploy/ansible/playbooks/templates/postfix.cf.j2 diff --git a/core/core/settings.py b/core/core/settings.py index 8f3c5b8..615ecdc 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -31,6 +31,8 @@ DEBUG = True ALLOWED_HOSTS = [os.getenv('HTTP_HOST', 'localhost')] +MAIL_DOMAIN = os.getenv('MAIL_DOMAIN', 'localhost') + SYSTEM3_VERSION = "0.0.0-dev.0" # Application definition diff --git a/core/lmtp/__init__.py b/core/lmtp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/lmtp/protocol.py b/core/lmtp/protocol.py new file mode 100644 index 0000000..d422aab --- /dev/null +++ b/core/lmtp/protocol.py @@ -0,0 +1,47 @@ +import asyncio +import logging + + +def make_reply(message, to, subject): + from email.message import EmailMessage + from core.settings import MAIL_DOMAIN + + reply = EmailMessage() + reply["From"] = "noreply@" + MAIL_DOMAIN + reply["To"] = to + reply["Subject"] = subject + reply.set_content(message) + + return reply + + +async def send_smtp(message): + import aiosmtplib + log = logging.getLogger('mail.log') + log.info('Sending message to %s' % message['To']) + + await aiosmtplib.send(message, hostname="127.0.0.1", port=1025) + + +class LMTPHandler: + async def handle_RCPT(self, server, session, envelope, address, rcpt_options): + from core.settings import MAIL_DOMAIN + if not address.endswith('@' + MAIL_DOMAIN): + return '550 not relaying to that domain' + envelope.rcpt_tos.append(address) + return '250 OK' + + async def handle_DATA(self, server, session, envelope): + log = logging.getLogger('mail.log') + log.info('Message from %s' % envelope.mail_from) + log.info('Message for %s' % envelope.rcpt_tos) + log.info('Message data:\n') + for ln in envelope.content.decode('utf8', errors='replace').splitlines(): + log.info(f'> {ln}'.strip()) + log.info('End of message') + + await send_smtp(make_reply("Thank you for your message.", envelope.mail_from, 'Message received')) + + # asyncio.create_task(send_reply()) + + return '250 Message accepted for delivery' diff --git a/core/lmtp/socket.py b/core/lmtp/socket.py new file mode 100644 index 0000000..312b916 --- /dev/null +++ b/core/lmtp/socket.py @@ -0,0 +1,31 @@ +from abc import ABCMeta + +from aiosmtpd.controller import BaseController, UnixSocketMixin +from aiosmtpd.lmtp import LMTP + + +class BaseAsyncController(BaseController, metaclass=ABCMeta): + def __init__( + self, + handler, + loop, + **SMTP_parameters, + ): + super().__init__( + handler, + loop, + **SMTP_parameters, + ) + + def serve(self): + return self._create_server() + + +class UnixSocketLMTPController(UnixSocketMixin, BaseAsyncController): + def factory(self): + return LMTP(self.handler) + + def _trigger_server(self): # pragma: no-unixsock + # Prevent confusion on which _trigger_server() to invoke. + # Or so LGTM.com claimed + UnixSocketMixin._trigger_server(self) diff --git a/core/notify_sessions/routing.py b/core/notify_sessions/routing.py index 64f561e..1aa0190 100644 --- a/core/notify_sessions/routing.py +++ b/core/notify_sessions/routing.py @@ -3,6 +3,5 @@ from django.urls import path from .consumers import NotifyConsumer websocket_urlpatterns = [ - path('ws/notify/', NotifyConsumer.as_asgi()), + path('ws/2/notify/', NotifyConsumer.as_asgi()), ] - diff --git a/core/requirements.txt b/core/requirements.prod.txt similarity index 97% rename from core/requirements.txt rename to core/requirements.prod.txt index bf4da2a..2212d3c 100644 --- a/core/requirements.txt +++ b/core/requirements.prod.txt @@ -1,4 +1,5 @@ aiosmtpd==1.4.4.post2 +aiosmtplib==3.0.1 anyio==4.0.0 asgiref==3.7.2 async-timeout==4.0.3 diff --git a/core/server.py b/core/server.py index 407d0ba..69946b9 100644 --- a/core/server.py +++ b/core/server.py @@ -1,55 +1,29 @@ #!/usr/bin/env python3 import asyncio import logging +import os import uvicorn -from aiosmtpd.controller import Controller, UnixSocketController -from aiosmtpd.lmtp import LMTP + +from lmtp.protocol import LMTPHandler +from lmtp.socket import UnixSocketLMTPController -async def handle_echo(reader, writer): - # session = Session() - data = await reader.read(100) - message = data.decode() - addr = writer.get_extra_info('peername') - - print(f"Received {message!r} from {addr!r}") - - print(f"Send: {message!r}") - writer.write(data) - await writer.drain() - - print("Close the connection") - writer.close() - await writer.wait_closed() - - -class ExampleHandler: - async def handle_RCPT(self, server, session, envelope, address, rcpt_options): - if not address.endswith('@example.com'): - return '550 not relaying to that domain' - envelope.rcpt_tos.append(address) - return '250 OK' - - async def handle_DATA(self, server, session, envelope): - print('Message from %s' % envelope.mail_from) - print('Message for %s' % envelope.rcpt_tos) - print('Message data:\n') - for ln in envelope.content.decode('utf8', errors='replace').splitlines(): - print(f'> {ln}'.strip()) - print() - print('End of message') - return '250 Message accepted for delivery' - - -class LTMPController(Controller): - def factory(self): - return LMTP(self.handler) - - -class UnixSocketLMTPController(UnixSocketController): - def factory(self): - return LMTP(self.handler) +# async def handle_echo(reader, writer): +# # session = Session() +# data = await reader.read(100) +# message = data.decode() +# addr = writer.get_extra_info('peername') +# +# print(f"Received {message!r} from {addr!r}") +# +# print(f"Send: {message!r}") +# writer.write(data) +# await writer.drain() +# +# print("Close the connection") +# writer.close() +# await writer.wait_closed() class UvicornServer(uvicorn.Server): @@ -57,7 +31,7 @@ class UvicornServer(uvicorn.Server): pass -async def web(): +async def web(loop): log_config = uvicorn.config.LOGGING_CONFIG log_config["handlers"]["default"] = {"class": "logging.FileHandler", "filename": "web.log", "formatter": "default"} log_config["handlers"]["access"] = {"class": "logging.FileHandler", "filename": "web-access.log", @@ -67,24 +41,43 @@ async def web(): await server.serve() -async def tcp(): - log = logging.getLogger('test') - log.info("Starting TCP server") - server = await asyncio.start_unix_server(handle_echo, path='test.sock') +# async def tcp(loop): +# log = logging.getLogger('test.log') +# log.addHandler(logging.FileHandler('test.log')) +# log.setLevel(logging.DEBUG) +# log.info("Starting TCP server") +# server = await asyncio.start_unix_server(handle_echo, path='test.sock') +# +# addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets) +# log.info(f'Serving on {addrs}') +# +# async with server: +# await server.serve_forever() +# log.info("TCP done") + + +async def lmtp(loop): + import grp + log = logging.getLogger('mail.log') + log.addHandler(logging.FileHandler('mail.log')) + # log.setLevel(logging.WARNING) + log.setLevel(logging.INFO) + log.info("Starting LMTP server") + server = await UnixSocketLMTPController(LMTPHandler(), unix_socket='lmtp.sock', loop=loop).serve() addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets) log.info(f'Serving on {addrs}') + try: + os.chmod('lmtp.sock', 0o775) + current_uid = os.getuid() + posix_gid = grp.getgrnam('postfix').gr_gid + os.chown('lmtp.sock', current_uid, posix_gid) + except Exception as e: + log.error(e) + async with server: await server.serve_forever() - log.info("TCP done") - - -async def lmtp(): - log = logging.getLogger('lmtp') - log.info("Starting LMTP server") - cont = UnixSocketLMTPController(ExampleHandler(), unix_socket='lmtp.sock') - cont.start() log.info("LMTP done") @@ -104,18 +97,18 @@ def main(): import sdnotify import signal import setproctitle + import os setproctitle.setproctitle("c3lf-sys3") - logging.basicConfig(filename='server.log', level=logging.DEBUG, encoding='utf-8') - logging.basicConfig(filename='test.log', level=logging.DEBUG, encoding='utf-8') - logging.basicConfig(filename='lmtp.log', level=logging.DEBUG, encoding='utf-8') - log = logging.getLogger() + log = logging.getLogger('server.log') + log.addHandler(logging.FileHandler('server.log')) + log.setLevel(logging.DEBUG) log.info("Starting server") loop = asyncio.get_event_loop() loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop))) loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop))) - loop.create_task(web()) - loop.create_task(tcp()) - loop.create_task(lmtp()) + loop.create_task(web(loop)) + # loop.create_task(tcp(loop)) + loop.create_task(lmtp(loop)) n = sdnotify.SystemdNotifier() n.notify("READY=1") log.info("Server ready") @@ -123,8 +116,19 @@ def main(): loop.run_forever() finally: loop.close() + try: + os.remove("lmtp.sock") + except Exception as e: + log.error(e) + try: + os.remove("web.sock") + except Exception as e: + log.error(e) + log.error(e) logging.info("Server stopped") + logging.shutdown() + if __name__ == '__main__': main() diff --git a/deploy/ansible/inventory.yml.sample b/deploy/ansible/inventory.yml.sample index 87891c7..6ba14ac 100644 --- a/deploy/ansible/inventory.yml.sample +++ b/deploy/ansible/inventory.yml.sample @@ -8,6 +8,7 @@ c3lf-nodes: git_branch: master git_repo: db_password: + mail_domain: main_email: legacy_api_user: legacy_api_password: \ No newline at end of file diff --git a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml index 8c0646a..aaeb20e 100644 --- a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml +++ b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml @@ -274,4 +274,35 @@ service: name: c3lf-sys3 state: started - enabled: yes \ No newline at end of file + enabled: yes + + - name: add postfix to www-data group + user: + name: postfix + groups: www-data + append: yes + notify: + - restart postfix + + - name: add custom transport config + lineinfile: + path: /etc/postfix/master.cf + line: "c3lf-sys3 unix - n n - - lmtp" + state: present + create: yes + notify: + - restart postfix + + - name: configure postfix + template: + src: templates/postfix.cf.j2 + dest: /etc/postfix/main.cf + notify: + - restart postfix + + - name: UFW allow smtp + ufw: + rule: allow + port: 25 + proto: tcp + state: enabled \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index 1246173..72a0c30 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -4,6 +4,7 @@ DB_NAME=c3lf_sys3 DB_USER=c3lf_sys3 DB_PASSWORD={{ db_password }} HTTP_HOST={{ web_domain }} +MAIL_DOMAIN={{ mail_domain }} LEGACY_API_USER={{ legacy_api_user }} LEGACY_API_PASSWORD={{ legacy_api_password }} MEDIA_ROOT=/var/www/c3lf-sys3/userfiles diff --git a/deploy/ansible/playbooks/templates/postfix.cf.j2 b/deploy/ansible/playbooks/templates/postfix.cf.j2 new file mode 100644 index 0000000..724260a --- /dev/null +++ b/deploy/ansible/playbooks/templates/postfix.cf.j2 @@ -0,0 +1,50 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +readme_directory = no + +# See http://www.postfix.org/COMPATIBILITY_README.html -- default to 3.6 on +# fresh installs. +compatibility_level = 3.6 + +# TLS parameters +smtp_use_tls = yes +smtp_force_tls = yes +smtpd_use_tls = yes +smtpd_tls_cert_file=/etc/letsencrypt/live/{{ web_domain }}/fullchain.pem; +smtpd_tls_key_file=/etc/letsencrypt/live/{{ web_domain }}/privkey.pem; +smtpd_tls_security_level=may + +smtp_tls_CApath=/etc/ssl/certs +smtp_tls_security_level=may +smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + + +smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination +myhostname = polaris.c3lf.de +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = /etc/mailname +mydestination = $myhostname, , localhost +relayhost = firefly.lab.or.it +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all +inet_protocols = all + +maillog_file = /var/log/mail.log + +virtual_mailbox_domains = {{ mail_domain }} +virtual_transport=c3lf-sys3:unix:/var/www/c3lf-sys3/lmtp.sock