switch from WSGI using uWSGI to ASGI using custom event loop based on uvicorn

This commit is contained in:
j3d1 2023-11-22 21:41:49 +01:00
parent b103205dfe
commit 6b3cc4c168
15 changed files with 610 additions and 8 deletions

View file

@ -9,8 +9,25 @@ https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/
import os
from channels.auth import AuthMiddlewareStack
from channels.routing import ProtocolTypeRouter, URLRouter
from channels.security.websocket import AllowedHostsOriginValidator
from django.core.asgi import get_asgi_application
from notify_sessions.routing import websocket_urlpatterns
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
application = get_asgi_application()
django_asgi_app = get_asgi_application()
websocket_asgi_app = AllowedHostsOriginValidator(
#AuthMiddlewareStack(
URLRouter(
websocket_urlpatterns
)
#)
)
application = ProtocolTypeRouter({
"http": django_asgi_app,
"websocket": websocket_asgi_app,
})

View file

@ -49,6 +49,7 @@ INSTALLED_APPS = [
'channels',
'files',
'inventory',
'notify_sessions',
]
REST_FRAMEWORK = {
@ -183,12 +184,10 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
CHANNEL_LAYERS = {
'default': {
'BACKEND': 'channels.layers.InMemoryChannelLayer',
# 'BACKEND': 'asgi_redis.RedisChannelLayer',
# 'CONFIG': {
# 'hosts': [('localhost', 6379)],
# },
'ROUTING': 'example.routing.channel_routing',
'BACKEND': 'asgi_redis.RedisChannelLayer',
'CONFIG': {
'hosts': [('localhost', 6379)],
},
}
}

View file

View file

@ -0,0 +1,40 @@
import json
from channels.generic.websocket import AsyncWebsocketConsumer
class NotifyConsumer(AsyncWebsocketConsumer):
def __init__(self, *args, **kwargs):
super().__init__(args, kwargs)
self.room_group_name = None
self.event_slug = None
async def connect(self):
self.event_slug = self.scope["url_route"]["kwargs"]["event_slug"]
self.room_group_name = f"chat_{self.event_slug}"
# Join room group
await self.channel_layer.group_add(self.room_group_name, self.channel_name)
await self.accept()
async def disconnect(self, close_code):
# Leave room group
await self.channel_layer.group_discard(self.room_group_name, self.channel_name)
# Receive message from WebSocket
async def receive(self, text_data):
text_data_json = json.loads(text_data)
message = text_data_json["message"]
# Send message to room group
await self.channel_layer.group_send(
self.room_group_name, {"type": "chat.message", "message": message}
)
# Receive message from room group
async def chat_message(self, event):
message = event["message"]
# Send message to WebSocket
await self.send(text_data=json.dumps({"message": message}))

View file

@ -0,0 +1,8 @@
from django.urls import path
from .consumers import NotifyConsumer
websocket_urlpatterns = [
path('notify/<str:event_slug>/', NotifyConsumer.as_asgi()),
]

View file

@ -18,7 +18,6 @@ itypes==1.2.0
Jinja2==3.1.2
MarkupSafe==2.1.3
msgpack-python==0.5.6
mysqlclient
openapi-codec==1.3.2
packaging==23.2
Pillow==10.1.0
@ -27,9 +26,13 @@ pytz==2023.3.post1
PyYAML==6.0.1
redis
requests==2.31.0
sdnotify==0.3.2
setproctitle==1.3.3
six==1.16.0
sqlparse==0.4.4
typing_extensions==4.8.0
uritemplate==4.1.1
urllib3==2.1.0
uvicorn==0.24.0.post1
watchfiles==0.21.0
websockets==12.0

130
core/server.py Normal file
View file

@ -0,0 +1,130 @@
#!/usr/bin/env python3
import asyncio
import logging
import uvicorn
from aiosmtpd.controller import Controller, UnixSocketController
from aiosmtpd.lmtp import LMTP
async def handle_echo(reader, writer):
# session = Session()
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print(f"Received {message!r} from {addr!r}")
print(f"Send: {message!r}")
writer.write(data)
await writer.drain()
print("Close the connection")
writer.close()
await writer.wait_closed()
class ExampleHandler:
async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
if not address.endswith('@example.com'):
return '550 not relaying to that domain'
envelope.rcpt_tos.append(address)
return '250 OK'
async def handle_DATA(self, server, session, envelope):
print('Message from %s' % envelope.mail_from)
print('Message for %s' % envelope.rcpt_tos)
print('Message data:\n')
for ln in envelope.content.decode('utf8', errors='replace').splitlines():
print(f'> {ln}'.strip())
print()
print('End of message')
return '250 Message accepted for delivery'
class LTMPController(Controller):
def factory(self):
return LMTP(self.handler)
class UnixSocketLMTPController(UnixSocketController):
def factory(self):
return LMTP(self.handler)
class UvicornServer(uvicorn.Server):
def install_signal_handlers(self):
pass
async def web():
log_config = uvicorn.config.LOGGING_CONFIG
log_config["handlers"]["default"] = {"class": "logging.FileHandler", "filename": "web.log", "formatter": "default"}
log_config["handlers"]["access"] = {"class": "logging.FileHandler", "filename": "web-access.log",
"formatter": "access"}
config = uvicorn.Config("core.asgi:application", uds="web.sock", log_config=log_config)
server = UvicornServer(config=config)
await server.serve()
async def tcp():
log = logging.getLogger('test')
log.info("Starting TCP server")
server = await asyncio.start_unix_server(handle_echo, path='test.sock')
addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
log.info(f'Serving on {addrs}')
async with server:
await server.serve_forever()
log.info("TCP done")
async def lmtp():
log = logging.getLogger('lmtp')
log.info("Starting LMTP server")
cont = UnixSocketLMTPController(ExampleHandler(), unix_socket='lmtp.sock')
cont.start()
log.info("LMTP done")
async def shutdown(sig, loop):
log = logging.getLogger()
log.info(f"Received exit signal {sig.name}...")
tasks = [t for t in asyncio.all_tasks() if t is not
asyncio.current_task()]
[task.cancel() for task in tasks]
log.info(f"Cancelling {len(tasks)} outstanding tasks")
await asyncio.wait_for(loop.shutdown_asyncgens(), timeout=10)
loop.stop()
log.info("Shutdown complete.")
def main():
import sdnotify
import signal
import setproctitle
setproctitle.setproctitle("c3lf-sys3")
logging.basicConfig(filename='server.log', level=logging.DEBUG, encoding='utf-8')
logging.basicConfig(filename='test.log', level=logging.DEBUG, encoding='utf-8')
logging.basicConfig(filename='lmtp.log', level=logging.DEBUG, encoding='utf-8')
log = logging.getLogger()
log.info("Starting server")
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop)))
loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop)))
loop.create_task(web())
loop.create_task(tcp())
loop.create_task(lmtp())
n = sdnotify.SystemdNotifier()
n.notify("READY=1")
log.info("Server ready")
try:
loop.run_forever()
finally:
loop.close()
logging.info("Server stopped")
if __name__ == '__main__':
main()

1
deploy/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
inventory.yml

View file

@ -0,0 +1,13 @@
---
c3lf-nodes:
hosts:
<name>:
ansible_ssh_host: <fqdn>
ansible_ssh_user: root
web_domain: <https_domain>
git_branch: master
git_repo: <git_repo_url>
db_password: <db_password>
main_email: <main_email>
legacy_api_user: <legacy_api_user>
legacy_api_password: <legacy_api_password>

View file

@ -0,0 +1,277 @@
- name: 'deploy c3lf-sys3'
hosts: 'c3lf-nodes'
handlers:
- name: restart nginx
service:
name: nginx
state: restarted
- name: restart postfix
service:
name: postfix
state: restarted
- name: restart mariadb
service:
name: mariadb
state: restarted
- name: restart c3lf-sys3
service:
name: c3lf-sys3
state: restarted
tasks:
- name: Update apt-get repo and cache
apt: update_cache=yes force_apt_get=yes cache_valid_time=3600
- name: Upgrade all apt packages
apt: upgrade=dist force_apt_get=yes
- name: Ansible apt-get to install base tools
apt:
name:
- htop
- tcpdump
- jq
- curl
- libsensors5
- prometheus-node-exporter
- openssh-server
state: present
force_apt_get: yes
- name: Remove useless packages from the cache
apt:
autoclean: yes
- name: Remove dependencies that are no longer required
apt:
autoremove: yes
- name: Check if a reboot is needed for debian
register: reboot_required_file
stat: path=/var/run/reboot-required get_md5=no
- name: Reboot the Debian or Ubuntu server
reboot:
msg: "Reboot initiated by Ansible due to kernel updates"
connect_timeout: 5
reboot_timeout: 300
pre_reboot_delay: 0
post_reboot_delay: 30
test_command: uptime
when: reboot_required_file.stat.exists
- name: Ansible apt-get to install sys3 requirements
apt:
name:
- ufw
- fail2ban
- nginx
- redis
- python3
- python3-pip
- python3-venv
- python3-passlib
- certbot
- python3-certbot-nginx
- mariadb-server
- python3-dev
- default-libmysqlclient-dev
- build-essential
- postfix
- git
- pkg-config
- npm
state: present
- name: remove default nginx site
file:
path: /etc/nginx/sites-enabled/default
state: absent
- name: remove default nginx site
file:
path: /etc/nginx/sites-available/default
state: absent
- name: UFW allow SSH
ufw:
rule: allow
port: 22
proto: tcp
state: enabled
- name: UFW logging off
ufw:
logging: off
- name: Configure nginx
template:
src: templates/nginx.conf.j2
dest: /etc/nginx/sites-available/c3lf-sys3.conf
notify:
- restart nginx
- name: UFW allow http
ufw:
rule: allow
port: 80
proto: tcp
state: enabled
- name: UFW allow https
ufw:
rule: allow
port: 443
proto: tcp
state: enabled
- name: Check if initial certbot certificate is needed
stat:
path: /etc/letsencrypt/live/{{web_domain}}/fullchain.pem
register: certbot_cert_exists
- block:
- name: stop nginx
service:
name: nginx
state: stopped
- name: add certbot domain
command: "certbot certonly --standalone -d {{web_domain}} --non-interactive --agree-tos --email {{main_email}}"
- name: start nginx
service:
name: nginx
state: started
when: certbot_cert_exists.stat.exists == false
- name: Enable certbot auto renew
cron:
name: "certbot-auto renew"
minute: "0"
hour: "12"
job: "certbot renew --quiet --no-self-upgrade --nginx --cert-name {{web_domain}}"
state: present
- name: Configure basic auth
htpasswd:
path: /etc/nginx/conf.d/lf-prod.htpasswd
name: "{{ legacy_api_user }}"
password: "{{ legacy_api_password }}"
state: present
notify:
- restart nginx
- name: Enable nginx site
file:
src: /etc/nginx/sites-available/c3lf-sys3.conf
dest: /etc/nginx/sites-enabled/c3lf-sys3.conf
state: link
notify:
- restart nginx
- name: create database
mysql_db:
name: c3lf_sys3
state: present
login_unix_socket: /var/run/mysqld/mysqld.sock
- name: create database user
mysql_user:
name: c3lf_sys3
password: "{{ db_password }}"
priv: "c3lf_sys3.*:ALL"
state: present
login_unix_socket: /var/run/mysqld/mysqld.sock
- name: configure webdir
file:
path: /var/www/c3lf-sys3
state: directory
owner: www-data
group: www-data
mode: 0755
- name: install python app
become: true
become_user: www-data
become_method: su
become_flags: '-s /bin/bash'
block:
- name: create repo dir
git:
repo: "{{ git_repo }}"
dest: /var/www/c3lf-sys3/repo
version: "{{ git_branch }}"
force: yes
recursive: yes
single_branch: yes
register: git_repo
notify:
- restart c3lf-sys3
- name: check if venv exists
stat:
path: /var/www/c3lf-sys3/venv/bin/python3
register: venv_exists
- name: create venv
command: "python3 -m venv /var/www/c3lf-sys3/venv"
when: venv_exists.stat.exists == false
- name: install requirements
pip:
requirements: /var/www/c3lf-sys3/repo/core/requirements.txt
virtualenv: /var/www/c3lf-sys3/venv
state: present
when: git_repo.changed == true
notify:
- restart c3lf-sys3
- name: configure django
template:
src: templates/django.env.j2
dest: /var/www/c3lf-sys3/repo/core/.env
- name: migrate database
command: "/var/www/c3lf-sys3/venv/bin/python /var/www/c3lf-sys3/repo/core/manage.py migrate"
when: git_repo.changed == true
- name: collect static files
command: "/var/www/c3lf-sys3/venv/bin/python /var/www/c3lf-sys3/repo/core/manage.py collectstatic --noinput"
when: git_repo.changed == true
- name: js config
template:
src: templates/config.js.j2
dest: /var/www/c3lf-sys3/repo/web/src/config.js
- name: build install dependencies
command:
cmd: "npm install"
chdir: /var/www/c3lf-sys3/repo/web
when: git_repo.changed == true
- name: build frontend
command:
cmd: "npm run build"
chdir: /var/www/c3lf-sys3/repo/web
when: git_repo.changed == true
- name: add c3lf-sys3 service
template:
src: templates/c3lf-sys3.service.j2
dest: /etc/systemd/system/c3lf-sys3.service
notify:
- restart c3lf-sys3
- name: reload systemd
systemd:
daemon_reload: yes
- name: start c3lf-sys3 service
service:
name: c3lf-sys3
state: started
enabled: yes

View file

@ -0,0 +1,15 @@
[Unit]
Description=standalone c3lf-sys3 server
After=network.target
[Service]
Type=notify
WorkingDirectory=/var/www/c3lf-sys3
ExecStart=/var/www/c3lf-sys3/venv/bin/python3 /var/www/c3lf-sys3/repo/core/server.py
Restart=always
RestartSec=5
User=www-data
Group=www-data
[Install]
WantedBy=multi-user.target

View file

@ -0,0 +1,9 @@
export default {
service: {
url: 'https://{{ web_domain }}/api',
auth: {
username: '{{ legacy_api_user }}',
password: '{{ legacy_api_password }}'
}
}
};

View file

@ -0,0 +1,10 @@
DB_HOST=localhost
DB_PORT=3306
DB_NAME=c3lf_sys3
DB_USER=c3lf_sys3
DB_PASSWORD={{ db_password }}
HTTP_HOST={{ web_domain }}
LEGACY_API_USER={{ legacy_api_user }}
LEGACY_API_PASSWORD={{ legacy_api_password }}
MEDIA_ROOT=/var/www/c3lf-sys3/userfiles
STATIC_ROOT=/var/www/c3lf-sys3/staticfiles

View file

@ -0,0 +1,80 @@
upstream c3lf-sys3 {
server unix:/var/www/c3lf-sys3/web.sock;
}
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
server_name {{ web_domain }};
client_max_body_size 1024M;
auth_basic C3LF;
auth_basic_user_file conf.d/lf-prod.htpasswd;
location / {
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE, PUT';
add_header 'Content-Security-Policy' "default-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob: wss:;";
root /var/www/c3lf-sys3/repo/web/dist;
index index.html index.htm index.nginx-debian.html;
try_files $uri $uri/ /index.html;
}
location /api {
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_redirect off;
proxy_buffering off;
proxy_pass http://c3lf-sys3;
}
location /admin {
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_redirect off;
proxy_buffering off;
proxy_pass http://c3lf-sys3;
}
location /redirect_media/ {
internal;
alias /var/www/c3lf-sys3/userfiles/;
}
location /redirect_thumbnail/ {
internal;
alias /var/www/c3lf-sys3/userfiles/thumbnails/;
}
location /static/ {
alias /var/www/c3lf-sys3/staticfiles/;
}
listen 443 ssl http2; # managed by Certbot
ssl_certificate /etc/letsencrypt/live/{{ web_domain }}/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/{{ web_domain }}/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}
server {
if ($host = {{ web_domain }}) {
return 301 https://$host$request_uri;
} # managed by Certbot
server_name {{ web_domain }};
listen 80;
return 404; # managed by Certbot
}