switch from WSGI using uWSGI to ASGI using custom event loop based on uvicorn

core/core/asgi.py

@@ -9,8 +9,25 @@ https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
 
 import os
 
+from channels.auth import AuthMiddlewareStack
+from channels.routing import ProtocolTypeRouter, URLRouter
+from channels.security.websocket import AllowedHostsOriginValidator
 from django.core.asgi import get_asgi_application
+from notify_sessions.routing import websocket_urlpatterns
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
 
-application = get_asgi_application()
+django_asgi_app = get_asgi_application()
+
+websocket_asgi_app = AllowedHostsOriginValidator(
+    #AuthMiddlewareStack(
+        URLRouter(
+            websocket_urlpatterns
+        )
+    #)
+)
+
+application = ProtocolTypeRouter({
+    "http": django_asgi_app,
+    "websocket": websocket_asgi_app,
+})

core/core/settings.py

@@ -49,6 +49,7 @@ INSTALLED_APPS = [
     'channels',
     'files',
     'inventory',
+    'notify_sessions',
 ]
 
 REST_FRAMEWORK = {
@@ -183,12 +184,10 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
 CHANNEL_LAYERS = {
     'default': {
-        'BACKEND': 'channels.layers.InMemoryChannelLayer',
+        'BACKEND': 'asgi_redis.RedisChannelLayer',
-        # 'BACKEND': 'asgi_redis.RedisChannelLayer',
+        'CONFIG': {
-        # 'CONFIG': {
+            'hosts': [('localhost', 6379)],
-        #    'hosts': [('localhost', 6379)],
+        },
-        # },
-        'ROUTING': 'example.routing.channel_routing',
     }
 
 }

core/notify_sessions/consumers.py

@@ -0,0 +1,40 @@
+import json
+
+from channels.generic.websocket import AsyncWebsocketConsumer
+
+
+class NotifyConsumer(AsyncWebsocketConsumer):
+    def __init__(self, *args, **kwargs):
+        super().__init__(args, kwargs)
+        self.room_group_name = None
+        self.event_slug = None
+
+    async def connect(self):
+        self.event_slug = self.scope["url_route"]["kwargs"]["event_slug"]
+        self.room_group_name = f"chat_{self.event_slug}"
+
+        # Join room group
+        await self.channel_layer.group_add(self.room_group_name, self.channel_name)
+
+        await self.accept()
+
+    async def disconnect(self, close_code):
+        # Leave room group
+        await self.channel_layer.group_discard(self.room_group_name, self.channel_name)
+
+    # Receive message from WebSocket
+    async def receive(self, text_data):
+        text_data_json = json.loads(text_data)
+        message = text_data_json["message"]
+
+        # Send message to room group
+        await self.channel_layer.group_send(
+            self.room_group_name, {"type": "chat.message", "message": message}
+        )
+
+    # Receive message from room group
+    async def chat_message(self, event):
+        message = event["message"]
+
+        # Send message to WebSocket
+        await self.send(text_data=json.dumps({"message": message}))

core/notify_sessions/routing.py

@@ -0,0 +1,8 @@
+from django.urls import path
+
+from .consumers import NotifyConsumer
+
+websocket_urlpatterns = [
+    path('notify/<str:event_slug>/', NotifyConsumer.as_asgi()),
+]
+

core/requirements.txt

@@ -18,7 +18,6 @@ itypes==1.2.0
 Jinja2==3.1.2
 MarkupSafe==2.1.3
 msgpack-python==0.5.6
-mysqlclient
 openapi-codec==1.3.2
 packaging==23.2
 Pillow==10.1.0
@@ -27,9 +26,13 @@ pytz==2023.3.post1
 PyYAML==6.0.1
 redis
 requests==2.31.0
+sdnotify==0.3.2
+setproctitle==1.3.3
 six==1.16.0
 sqlparse==0.4.4
 typing_extensions==4.8.0
 uritemplate==4.1.1
 urllib3==2.1.0
+uvicorn==0.24.0.post1
+watchfiles==0.21.0
 websockets==12.0

core/server.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+import asyncio
+import logging
+
+import uvicorn
+from aiosmtpd.controller import Controller, UnixSocketController
+from aiosmtpd.lmtp import LMTP
+
+
+async def handle_echo(reader, writer):
+    # session = Session()
+    data = await reader.read(100)
+    message = data.decode()
+    addr = writer.get_extra_info('peername')
+
+    print(f"Received {message!r} from {addr!r}")
+
+    print(f"Send: {message!r}")
+    writer.write(data)
+    await writer.drain()
+
+    print("Close the connection")
+    writer.close()
+    await writer.wait_closed()
+
+
+class ExampleHandler:
+    async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
+        if not address.endswith('@example.com'):
+            return '550 not relaying to that domain'
+        envelope.rcpt_tos.append(address)
+        return '250 OK'
+
+    async def handle_DATA(self, server, session, envelope):
+        print('Message from %s' % envelope.mail_from)
+        print('Message for %s' % envelope.rcpt_tos)
+        print('Message data:\n')
+        for ln in envelope.content.decode('utf8', errors='replace').splitlines():
+            print(f'> {ln}'.strip())
+        print()
+        print('End of message')
+        return '250 Message accepted for delivery'
+
+
+class LTMPController(Controller):
+    def factory(self):
+        return LMTP(self.handler)
+
+
+class UnixSocketLMTPController(UnixSocketController):
+    def factory(self):
+        return LMTP(self.handler)
+
+
+class UvicornServer(uvicorn.Server):
+    def install_signal_handlers(self):
+        pass
+
+
+async def web():
+    log_config = uvicorn.config.LOGGING_CONFIG
+    log_config["handlers"]["default"] = {"class": "logging.FileHandler", "filename": "web.log", "formatter": "default"}
+    log_config["handlers"]["access"] = {"class": "logging.FileHandler", "filename": "web-access.log",
+                                        "formatter": "access"}
+    config = uvicorn.Config("core.asgi:application", uds="web.sock", log_config=log_config)
+    server = UvicornServer(config=config)
+    await server.serve()
+
+
+async def tcp():
+    log = logging.getLogger('test')
+    log.info("Starting TCP server")
+    server = await asyncio.start_unix_server(handle_echo, path='test.sock')
+
+    addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
+    log.info(f'Serving on {addrs}')
+
+    async with server:
+        await server.serve_forever()
+    log.info("TCP done")
+
+
+async def lmtp():
+    log = logging.getLogger('lmtp')
+    log.info("Starting LMTP server")
+    cont = UnixSocketLMTPController(ExampleHandler(), unix_socket='lmtp.sock')
+    cont.start()
+    log.info("LMTP done")
+
+
+async def shutdown(sig, loop):
+    log = logging.getLogger()
+    log.info(f"Received exit signal {sig.name}...")
+    tasks = [t for t in asyncio.all_tasks() if t is not
+             asyncio.current_task()]
+    [task.cancel() for task in tasks]
+    log.info(f"Cancelling {len(tasks)} outstanding tasks")
+    await asyncio.wait_for(loop.shutdown_asyncgens(), timeout=10)
+    loop.stop()
+    log.info("Shutdown complete.")
+
+
+def main():
+    import sdnotify
+    import signal
+    import setproctitle
+    setproctitle.setproctitle("c3lf-sys3")
+    logging.basicConfig(filename='server.log', level=logging.DEBUG, encoding='utf-8')
+    logging.basicConfig(filename='test.log', level=logging.DEBUG, encoding='utf-8')
+    logging.basicConfig(filename='lmtp.log', level=logging.DEBUG, encoding='utf-8')
+    log = logging.getLogger()
+    log.info("Starting server")
+    loop = asyncio.get_event_loop()
+    loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop)))
+    loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop)))
+    loop.create_task(web())
+    loop.create_task(tcp())
+    loop.create_task(lmtp())
+    n = sdnotify.SystemdNotifier()
+    n.notify("READY=1")
+    log.info("Server ready")
+    try:
+        loop.run_forever()
+    finally:
+        loop.close()
+    logging.info("Server stopped")
+
+
+if __name__ == '__main__':
+    main()

deploy/.gitignore

@@ -0,0 +1 @@
+inventory.yml

deploy/ansible/inventory.yml.sample

@@ -0,0 +1,13 @@
+---
+c3lf-nodes:
+  hosts:
+    <name>:
+      ansible_ssh_host: <fqdn>
+      ansible_ssh_user: root
+      web_domain: <https_domain>
+      git_branch: master
+      git_repo: <git_repo_url>
+      db_password: <db_password>
+      main_email: <main_email>
+      legacy_api_user: <legacy_api_user>
+      legacy_api_password: <legacy_api_password>

deploy/ansible/playbooks/deploy-c3lf-sys3.yml

@@ -0,0 +1,277 @@
+- name: 'deploy c3lf-sys3'
+  hosts: 'c3lf-nodes'
+  handlers:
+    - name: restart nginx
+      service:
+        name: nginx
+        state: restarted
+
+    - name: restart postfix
+      service:
+        name: postfix
+        state: restarted
+
+    - name: restart mariadb
+      service:
+        name: mariadb
+        state: restarted
+
+    - name: restart c3lf-sys3
+      service:
+        name: c3lf-sys3
+        state: restarted
+
+  tasks:
+    - name: Update apt-get repo and cache
+      apt: update_cache=yes force_apt_get=yes cache_valid_time=3600
+
+    - name: Upgrade all apt packages
+      apt: upgrade=dist force_apt_get=yes
+
+    - name: Ansible apt-get to install base tools
+      apt:
+        name:
+          - htop
+          - tcpdump
+          - jq
+          - curl
+          - libsensors5
+          - prometheus-node-exporter
+          - openssh-server
+        state: present
+        force_apt_get: yes
+
+    - name: Remove useless packages from the cache
+      apt:
+        autoclean: yes
+
+    - name: Remove dependencies that are no longer required
+      apt:
+        autoremove: yes
+
+    - name: Check if a reboot is needed for debian
+      register: reboot_required_file
+      stat: path=/var/run/reboot-required get_md5=no
+
+    - name: Reboot the Debian or Ubuntu server
+      reboot:
+        msg: "Reboot initiated by Ansible due to kernel updates"
+        connect_timeout: 5
+        reboot_timeout: 300
+        pre_reboot_delay: 0
+        post_reboot_delay: 30
+        test_command: uptime
+      when: reboot_required_file.stat.exists
+
+    - name: Ansible apt-get to install sys3 requirements
+      apt:
+        name:
+          - ufw
+          - fail2ban
+          - nginx
+          - redis
+          - python3
+          - python3-pip
+          - python3-venv
+          - python3-passlib
+          - certbot
+          - python3-certbot-nginx
+          - mariadb-server
+          - python3-dev
+          - default-libmysqlclient-dev
+          - build-essential
+          - postfix
+          - git
+          - pkg-config
+          - npm
+        state: present
+
+    - name: remove default nginx site
+      file:
+        path: /etc/nginx/sites-enabled/default
+        state: absent
+
+    - name: remove default nginx site
+      file:
+        path: /etc/nginx/sites-available/default
+        state: absent
+
+    - name: UFW allow SSH
+      ufw:
+        rule: allow
+        port: 22
+        proto: tcp
+        state: enabled
+
+    - name: UFW logging off
+      ufw:
+        logging: off
+
+    - name: Configure nginx
+      template:
+        src: templates/nginx.conf.j2
+        dest: /etc/nginx/sites-available/c3lf-sys3.conf
+      notify:
+        - restart nginx
+
+    - name: UFW allow http
+      ufw:
+        rule: allow
+        port: 80
+        proto: tcp
+        state: enabled
+
+    - name: UFW allow https
+      ufw:
+        rule: allow
+        port: 443
+        proto: tcp
+        state: enabled
+
+    - name: Check if initial certbot certificate is needed
+      stat:
+        path: /etc/letsencrypt/live/{{web_domain}}/fullchain.pem
+      register: certbot_cert_exists
+
+    - block:
+        - name: stop nginx
+          service:
+            name: nginx
+            state: stopped
+        - name: add certbot domain
+          command: "certbot certonly --standalone -d {{web_domain}} --non-interactive --agree-tos --email {{main_email}}"
+        - name: start nginx
+          service:
+            name: nginx
+            state: started
+      when: certbot_cert_exists.stat.exists == false
+
+    - name: Enable certbot auto renew
+      cron:
+        name: "certbot-auto renew"
+        minute: "0"
+        hour: "12"
+        job: "certbot renew --quiet --no-self-upgrade --nginx --cert-name {{web_domain}}"
+        state: present
+
+    - name: Configure basic auth
+      htpasswd:
+        path: /etc/nginx/conf.d/lf-prod.htpasswd
+        name: "{{ legacy_api_user }}"
+        password: "{{ legacy_api_password }}"
+        state: present
+      notify:
+        - restart nginx
+
+    - name: Enable nginx site
+      file:
+        src: /etc/nginx/sites-available/c3lf-sys3.conf
+        dest: /etc/nginx/sites-enabled/c3lf-sys3.conf
+        state: link
+      notify:
+        - restart nginx
+
+    - name: create database
+      mysql_db:
+        name: c3lf_sys3
+        state: present
+        login_unix_socket: /var/run/mysqld/mysqld.sock
+
+    - name: create database user
+      mysql_user:
+        name: c3lf_sys3
+        password: "{{ db_password }}"
+        priv: "c3lf_sys3.*:ALL"
+        state: present
+        login_unix_socket: /var/run/mysqld/mysqld.sock
+
+    - name: configure webdir
+      file:
+        path: /var/www/c3lf-sys3
+        state: directory
+        owner: www-data
+        group: www-data
+        mode: 0755
+
+    - name: install python app
+      become: true
+      become_user: www-data
+      become_method: su
+      become_flags: '-s /bin/bash'
+      block:
+        - name: create repo dir
+          git:
+            repo: "{{ git_repo }}"
+            dest: /var/www/c3lf-sys3/repo
+            version: "{{ git_branch }}"
+            force: yes
+            recursive: yes
+            single_branch: yes
+          register: git_repo
+          notify:
+            - restart c3lf-sys3
+
+        - name: check if venv exists
+          stat:
+            path: /var/www/c3lf-sys3/venv/bin/python3
+          register: venv_exists
+
+        - name: create venv
+          command: "python3 -m venv /var/www/c3lf-sys3/venv"
+          when: venv_exists.stat.exists == false
+
+        - name: install requirements
+          pip:
+            requirements: /var/www/c3lf-sys3/repo/core/requirements.txt
+            virtualenv: /var/www/c3lf-sys3/venv
+            state: present
+          when: git_repo.changed == true
+          notify:
+            - restart c3lf-sys3
+
+        - name: configure django
+          template:
+            src: templates/django.env.j2
+            dest: /var/www/c3lf-sys3/repo/core/.env
+
+        - name: migrate database
+          command: "/var/www/c3lf-sys3/venv/bin/python /var/www/c3lf-sys3/repo/core/manage.py migrate"
+          when: git_repo.changed == true
+
+        - name: collect static files
+          command: "/var/www/c3lf-sys3/venv/bin/python /var/www/c3lf-sys3/repo/core/manage.py collectstatic --noinput"
+          when: git_repo.changed == true
+
+        - name: js config
+          template:
+            src: templates/config.js.j2
+            dest: /var/www/c3lf-sys3/repo/web/src/config.js
+
+        - name: build install dependencies
+          command:
+            cmd: "npm install"
+            chdir: /var/www/c3lf-sys3/repo/web
+          when: git_repo.changed == true
+
+        - name: build frontend
+          command:
+            cmd: "npm run build"
+            chdir: /var/www/c3lf-sys3/repo/web
+          when: git_repo.changed == true
+
+    - name: add c3lf-sys3 service
+      template:
+        src: templates/c3lf-sys3.service.j2
+        dest: /etc/systemd/system/c3lf-sys3.service
+      notify:
+        - restart c3lf-sys3
+
+    - name: reload systemd
+      systemd:
+        daemon_reload: yes
+
+    - name: start c3lf-sys3 service
+      service:
+        name: c3lf-sys3
+        state: started
+        enabled: yes

deploy/ansible/playbooks/templates/c3lf-sys3.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=standalone c3lf-sys3 server
+After=network.target
+
+[Service]
+Type=notify
+WorkingDirectory=/var/www/c3lf-sys3
+ExecStart=/var/www/c3lf-sys3/venv/bin/python3 /var/www/c3lf-sys3/repo/core/server.py
+Restart=always
+RestartSec=5
+User=www-data
+Group=www-data
+
+[Install]
+WantedBy=multi-user.target

deploy/ansible/playbooks/templates/config.js.j2

@@ -0,0 +1,9 @@
+export default {
+    service: {
+        url: 'https://{{ web_domain }}/api',
+        auth: {
+            username: '{{ legacy_api_user }}',
+            password: '{{ legacy_api_password }}'
+        }
+    }
+};

deploy/ansible/playbooks/templates/django.env.j2

@@ -0,0 +1,10 @@
+DB_HOST=localhost
+DB_PORT=3306
+DB_NAME=c3lf_sys3
+DB_USER=c3lf_sys3
+DB_PASSWORD={{ db_password }}
+HTTP_HOST={{ web_domain }}
+LEGACY_API_USER={{ legacy_api_user }}
+LEGACY_API_PASSWORD={{ legacy_api_password }}
+MEDIA_ROOT=/var/www/c3lf-sys3/userfiles
+STATIC_ROOT=/var/www/c3lf-sys3/staticfiles

deploy/ansible/playbooks/templates/nginx.conf.j2

@@ -0,0 +1,80 @@
+upstream c3lf-sys3 {
+    server unix:/var/www/c3lf-sys3/web.sock;
+}
+
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+
+server {
+
+    server_name {{ web_domain }};
+    client_max_body_size 1024M;
+
+    auth_basic C3LF;
+    auth_basic_user_file conf.d/lf-prod.htpasswd;
+
+    location / {
+        add_header 'Access-Control-Allow-Origin' '*';
+        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE, PUT';
+        add_header 'Content-Security-Policy' "default-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob: wss:;";
+        root /var/www/c3lf-sys3/repo/web/dist;
+        index index.html index.htm index.nginx-debian.html;
+        try_files $uri $uri/ /index.html;
+    }
+
+    location /api {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_pass http://c3lf-sys3;
+    }
+
+    location /admin {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_redirect off;
+        proxy_buffering off;
+        proxy_pass http://c3lf-sys3;
+    }
+
+    location /redirect_media/ {
+        internal;
+        alias /var/www/c3lf-sys3/userfiles/;
+    }
+
+    location /redirect_thumbnail/ {
+        internal;
+        alias /var/www/c3lf-sys3/userfiles/thumbnails/;
+    }
+
+    location /static/ {
+        alias /var/www/c3lf-sys3/staticfiles/;
+    }
+
+    listen 443 ssl http2; # managed by Certbot
+    ssl_certificate /etc/letsencrypt/live/{{ web_domain }}/fullchain.pem; # managed by Certbot
+    ssl_certificate_key /etc/letsencrypt/live/{{ web_domain }}/privkey.pem; # managed by Certbot
+    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+
+}
+
+server {
+    if ($host = {{ web_domain }}) {
+        return 301 https://$host$request_uri;
+    } # managed by Certbot
+
+    server_name {{ web_domain }};
+
+    listen 80;
+    return 404; # managed by Certbot
+}