experimental mail transport

core/core/settings.py

@@ -31,6 +31,8 @@ DEBUG = True
 
 ALLOWED_HOSTS = [os.getenv('HTTP_HOST', 'localhost')]
 
+MAIL_DOMAIN = os.getenv('MAIL_DOMAIN', 'localhost')
+
 SYSTEM3_VERSION = "0.0.0-dev.0"
 
 # Application definition

core/helper.py

@@ -0,0 +1,30 @@
+import asyncio
+import logging
+import signal
+
+loop = None
+
+
+def create_task(coro):
+    global loop
+    loop.create_task(coro)
+
+
+async def shutdown(sig, loop):
+    log = logging.getLogger()
+    log.info(f"Received exit signal {sig.name}...")
+    tasks = [t for t in asyncio.all_tasks() if t is not
+             asyncio.current_task()]
+    [task.cancel() for task in tasks]
+    log.info(f"Cancelling {len(tasks)} outstanding tasks")
+    await asyncio.wait_for(loop.shutdown_asyncgens(), timeout=10)
+    loop.stop()
+    log.info("Shutdown complete.")
+
+
+def init_loop():
+    global loop
+    loop = asyncio.get_event_loop()
+    loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop)))
+    loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop)))
+    return loop

core/lmtp/protocol.py

@@ -0,0 +1,50 @@
+import asyncio
+import logging
+
+from helper import create_task
+
+
+def make_reply(message, to, subject):
+    from email.message import EmailMessage
+    from core.settings import MAIL_DOMAIN
+
+    reply = EmailMessage()
+    reply["From"] = "noreply@" + MAIL_DOMAIN
+    reply["To"] = to
+    reply["Subject"] = subject
+    reply.set_content(message)
+
+    return reply
+
+
+async def send_smtp(message):
+    await asyncio.sleep(30)
+    import aiosmtplib
+    log = logging.getLogger('mail.log')
+    log.info('Sending message to %s' % message['To'])
+
+    await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False)
+
+
+class LMTPHandler:
+    async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
+        from core.settings import MAIL_DOMAIN
+        if not address.endswith('@' + MAIL_DOMAIN):
+            return '550 not relaying to that domain'
+        envelope.rcpt_tos.append(address)
+        return '250 OK'
+
+    async def handle_DATA(self, server, session, envelope):
+        log = logging.getLogger('mail.log')
+        log.info('Message from %s' % envelope.mail_from)
+        log.info('Message for %s' % envelope.rcpt_tos)
+        log.info('Message data:\n')
+        for ln in envelope.content.decode('utf8', errors='replace').splitlines():
+            log.info(f'> {ln}'.strip())
+        log.info('End of message')
+
+        create_task(send_smtp(make_reply("Thank you for your message.", envelope.mail_from, 'Message received')))
+
+        # asyncio.create_task(send_reply())
+
+        return '250 Message accepted for delivery'

core/lmtp/socket.py

@@ -0,0 +1,31 @@
+from abc import ABCMeta
+
+from aiosmtpd.controller import BaseController, UnixSocketMixin
+from aiosmtpd.lmtp import LMTP
+
+
+class BaseAsyncController(BaseController, metaclass=ABCMeta):
+    def __init__(
+            self,
+            handler,
+            loop,
+            **SMTP_parameters,
+    ):
+        super().__init__(
+            handler,
+            loop,
+            **SMTP_parameters,
+        )
+
+    def serve(self):
+        return self._create_server()
+
+
+class UnixSocketLMTPController(UnixSocketMixin, BaseAsyncController):
+    def factory(self):
+        return LMTP(self.handler)
+
+    def _trigger_server(self):  # pragma: no-unixsock
+        # Prevent confusion on which _trigger_server() to invoke.
+        # Or so LGTM.com claimed
+        UnixSocketMixin._trigger_server(self)

core/notify_sessions/routing.py

@@ -3,6 +3,5 @@ from django.urls import path
 from .consumers import NotifyConsumer
 
 websocket_urlpatterns = [
-    path('ws/notify/', NotifyConsumer.as_asgi()),
+    path('ws/2/notify/', NotifyConsumer.as_asgi()),
 ]
-

core/requirements.prod.txt

@@ -1,4 +1,5 @@
 aiosmtpd==1.4.4.post2
+aiosmtplib==3.0.1
 anyio==4.0.0
 asgiref==3.7.2
 async-timeout==4.0.3

core/server.py

@@ -1,55 +1,30 @@
 #!/usr/bin/env python3
 import asyncio
 import logging
+import os
 
 import uvicorn
-from aiosmtpd.controller import Controller, UnixSocketController
+
-from aiosmtpd.lmtp import LMTP
+from helper import init_loop
+from lmtp.protocol import LMTPHandler
+from lmtp.socket import UnixSocketLMTPController
 
 
-async def handle_echo(reader, writer):
+# async def handle_echo(reader, writer):
-    # session = Session()
+#    # session = Session()
-    data = await reader.read(100)
+#    data = await reader.read(100)
-    message = data.decode()
+#    message = data.decode()
-    addr = writer.get_extra_info('peername')
+#    addr = writer.get_extra_info('peername')
-
+#
-    print(f"Received {message!r} from {addr!r}")
+#    print(f"Received {message!r} from {addr!r}")
-
+#
-    print(f"Send: {message!r}")
+#    print(f"Send: {message!r}")
-    writer.write(data)
+#    writer.write(data)
-    await writer.drain()
+#    await writer.drain()
-
+#
-    print("Close the connection")
+#    print("Close the connection")
-    writer.close()
+#    writer.close()
-    await writer.wait_closed()
+#    await writer.wait_closed()
-
-
-class ExampleHandler:
-    async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
-        if not address.endswith('@example.com'):
-            return '550 not relaying to that domain'
-        envelope.rcpt_tos.append(address)
-        return '250 OK'
-
-    async def handle_DATA(self, server, session, envelope):
-        print('Message from %s' % envelope.mail_from)
-        print('Message for %s' % envelope.rcpt_tos)
-        print('Message data:\n')
-        for ln in envelope.content.decode('utf8', errors='replace').splitlines():
-            print(f'> {ln}'.strip())
-        print()
-        print('End of message')
-        return '250 Message accepted for delivery'
-
-
-class LTMPController(Controller):
-    def factory(self):
-        return LMTP(self.handler)
-
-
-class UnixSocketLMTPController(UnixSocketController):
-    def factory(self):
-        return LMTP(self.handler)
 
 
 class UvicornServer(uvicorn.Server):
@@ -57,7 +32,7 @@ class UvicornServer(uvicorn.Server):
         pass
 
 
-async def web():
+async def web(loop):
     log_config = uvicorn.config.LOGGING_CONFIG
     log_config["handlers"]["default"] = {"class": "logging.FileHandler", "filename": "web.log", "formatter": "default"}
     log_config["handlers"]["access"] = {"class": "logging.FileHandler", "filename": "web-access.log",
@@ -67,55 +42,59 @@ async def web():
     await server.serve()
 
 
-async def tcp():
+# async def tcp(loop):
-    log = logging.getLogger('test')
+#    log = logging.getLogger('test.log')
-    log.info("Starting TCP server")
+#    log.addHandler(logging.FileHandler('test.log'))
-    server = await asyncio.start_unix_server(handle_echo, path='test.sock')
+#    log.setLevel(logging.DEBUG)
+#    log.info("Starting TCP server")
+#    server = await asyncio.start_unix_server(handle_echo, path='test.sock')
+#
+#    addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
+#    log.info(f'Serving on {addrs}')
+#
+#    async with server:
+#        await server.serve_forever()
+#    log.info("TCP done")
+
+
+async def lmtp(loop):
+    import grp
+    log = logging.getLogger('mail.log')
+    log.addHandler(logging.FileHandler('mail.log'))
+    # log.setLevel(logging.WARNING)
+    log.setLevel(logging.INFO)
+    log.info("Starting LMTP server")
+    server = await UnixSocketLMTPController(LMTPHandler(), unix_socket='lmtp.sock', loop=loop).serve()
 
     addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
     log.info(f'Serving on {addrs}')
 
+    try:
+        os.chmod('lmtp.sock', 0o775)
+        current_uid = os.getuid()
+        posix_gid = grp.getgrnam('postfix').gr_gid
+        os.chown('lmtp.sock', current_uid, posix_gid)
+    except Exception as e:
+        log.error(e)
+
     async with server:
         await server.serve_forever()
-    log.info("TCP done")
-
-
-async def lmtp():
-    log = logging.getLogger('lmtp')
-    log.info("Starting LMTP server")
-    cont = UnixSocketLMTPController(ExampleHandler(), unix_socket='lmtp.sock')
-    cont.start()
     log.info("LMTP done")
 
 
-async def shutdown(sig, loop):
-    log = logging.getLogger()
-    log.info(f"Received exit signal {sig.name}...")
-    tasks = [t for t in asyncio.all_tasks() if t is not
-             asyncio.current_task()]
-    [task.cancel() for task in tasks]
-    log.info(f"Cancelling {len(tasks)} outstanding tasks")
-    await asyncio.wait_for(loop.shutdown_asyncgens(), timeout=10)
-    loop.stop()
-    log.info("Shutdown complete.")
-
-
 def main():
     import sdnotify
-    import signal
     import setproctitle
+    import os
     setproctitle.setproctitle("c3lf-sys3")
-    logging.basicConfig(filename='server.log', level=logging.DEBUG, encoding='utf-8')
+    log = logging.getLogger('server.log')
-    logging.basicConfig(filename='test.log', level=logging.DEBUG, encoding='utf-8')
+    log.addHandler(logging.FileHandler('server.log'))
-    logging.basicConfig(filename='lmtp.log', level=logging.DEBUG, encoding='utf-8')
+    log.setLevel(logging.DEBUG)
-    log = logging.getLogger()
     log.info("Starting server")
-    loop = asyncio.get_event_loop()
+    loop = init_loop()
-    loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop)))
+    loop.create_task(web(loop))
-    loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop)))
+    #    loop.create_task(tcp(loop))
-    loop.create_task(web())
+    loop.create_task(lmtp(loop))
-    loop.create_task(tcp())
-    loop.create_task(lmtp())
     n = sdnotify.SystemdNotifier()
     n.notify("READY=1")
     log.info("Server ready")
@@ -123,8 +102,19 @@ def main():
         loop.run_forever()
     finally:
         loop.close()
+    try:
+        os.remove("lmtp.sock")
+    except Exception as e:
+        log.error(e)
+    try:
+        os.remove("web.sock")
+    except Exception as e:
+        log.error(e)
+        log.error(e)
     logging.info("Server stopped")
 
+    logging.shutdown()
+
 
 if __name__ == '__main__':
     main()

deploy/ansible/inventory.yml.sample

@@ -8,6 +8,7 @@ c3lf-nodes:
       git_branch: master
       git_repo: <git_repo_url>
       db_password: <db_password>
+      mail_domain: <mail_domain>
       main_email: <main_email>
       legacy_api_user: <legacy_api_user>
       legacy_api_password: <legacy_api_password>

deploy/ansible/playbooks/deploy-c3lf-sys3.yml

@@ -222,7 +222,7 @@
 
         - name: install requirements
           pip:
-            requirements: /var/www/c3lf-sys3/repo/core/requirements.txt
+            requirements: /var/www/c3lf-sys3/repo/core/requirements.prod.txt
             virtualenv: /var/www/c3lf-sys3/venv
             state: present
           when: git_repo.changed == true
@@ -274,4 +274,35 @@
       service:
         name: c3lf-sys3
         state: started
-        enabled: yes
+        enabled: yes
+
+    - name: add postfix to www-data group
+      user:
+        name: postfix
+        groups: www-data
+        append: yes
+      notify:
+        - restart postfix
+
+    - name: add custom transport config
+      lineinfile:
+        path: /etc/postfix/master.cf
+        line: "c3lf-sys3 unix - n n - - lmtp"
+        state: present
+        create: yes
+      notify:
+        - restart postfix
+
+    - name: configure postfix
+      template:
+        src: templates/postfix.cf.j2
+        dest: /etc/postfix/main.cf
+      notify:
+        - restart postfix
+
+    - name: UFW allow smtp
+      ufw:
+        rule: allow
+        port: 25
+        proto: tcp
+        state: enabled

deploy/ansible/playbooks/templates/django.env.j2

@@ -4,6 +4,7 @@ DB_NAME=c3lf_sys3
 DB_USER=c3lf_sys3
 DB_PASSWORD={{ db_password }}
 HTTP_HOST={{ web_domain }}
+MAIL_DOMAIN={{ mail_domain }}
 LEGACY_API_USER={{ legacy_api_user }}
 LEGACY_API_PASSWORD={{ legacy_api_password }}
 MEDIA_ROOT=/var/www/c3lf-sys3/userfiles

deploy/ansible/playbooks/templates/postfix.cf.j2

@@ -0,0 +1,50 @@
+# See /usr/share/postfix/main.cf.dist for a commented, more complete version
+
+
+# Debian specific:  Specifying a file name will cause the first
+# line of that file to be used as the name.  The Debian default
+# is /etc/mailname.
+#myorigin = /etc/mailname
+
+smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU)
+biff = no
+
+# appending .domain is the MUA's job.
+append_dot_mydomain = no
+
+readme_directory = no
+
+# See http://www.postfix.org/COMPATIBILITY_README.html -- default to 3.6 on
+# fresh installs.
+compatibility_level = 3.6
+
+# TLS parameters
+smtp_use_tls = yes
+smtp_force_tls = yes
+smtpd_use_tls = yes
+smtpd_tls_cert_file=/etc/letsencrypt/live/{{ web_domain }}/fullchain.pem
+smtpd_tls_key_file=/etc/letsencrypt/live/{{ web_domain }}/privkey.pem
+smtpd_tls_security_level=may
+
+smtp_tls_CApath=/etc/ssl/certs
+smtp_tls_security_level=may
+smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
+
+
+smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination
+myhostname = polaris.c3lf.de
+alias_maps = hash:/etc/aliases
+alias_database = hash:/etc/aliases
+myorigin = /etc/mailname
+mydestination = $myhostname, , localhost
+relayhost = firefly.lab.or.it
+mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+mailbox_size_limit = 0
+recipient_delimiter = +
+inet_interfaces = all
+inet_protocols = all
+
+maillog_file = /var/log/mail.log
+
+virtual_mailbox_domains = {{ mail_domain }}
+virtual_transport=c3lf-sys3:unix:/var/www/c3lf-sys3/lmtp.sock