diff --git a/core/core/asgi.py b/core/core/asgi.py index 3b35c6b..da744a2 100644 --- a/core/core/asgi.py +++ b/core/core/asgi.py @@ -9,8 +9,25 @@ https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ import os +from channels.auth import AuthMiddlewareStack +from channels.routing import ProtocolTypeRouter, URLRouter +from channels.security.websocket import AllowedHostsOriginValidator from django.core.asgi import get_asgi_application +from notify_sessions.routing import websocket_urlpatterns os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') -application = get_asgi_application() +django_asgi_app = get_asgi_application() + +websocket_asgi_app = AllowedHostsOriginValidator( + #AuthMiddlewareStack( + URLRouter( + websocket_urlpatterns + ) + #) +) + +application = ProtocolTypeRouter({ + "http": django_asgi_app, + "websocket": websocket_asgi_app, +}) diff --git a/core/core/settings.py b/core/core/settings.py index c0e1291..e752a50 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -49,6 +49,7 @@ INSTALLED_APPS = [ 'channels', 'files', 'inventory', + 'notify_sessions', ] REST_FRAMEWORK = { @@ -183,12 +184,10 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' CHANNEL_LAYERS = { 'default': { - 'BACKEND': 'channels.layers.InMemoryChannelLayer', - # 'BACKEND': 'asgi_redis.RedisChannelLayer', - # 'CONFIG': { - # 'hosts': [('localhost', 6379)], - # }, - 'ROUTING': 'example.routing.channel_routing', + 'BACKEND': 'asgi_redis.RedisChannelLayer', + 'CONFIG': { + 'hosts': [('localhost', 6379)], + }, } } diff --git a/core/notify_sessions/__init__.py b/core/notify_sessions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/notify_sessions/consumers.py b/core/notify_sessions/consumers.py new file mode 100644 index 0000000..6284c9e --- /dev/null +++ b/core/notify_sessions/consumers.py @@ -0,0 +1,40 @@ +import json + +from channels.generic.websocket import AsyncWebsocketConsumer + + +class NotifyConsumer(AsyncWebsocketConsumer): + def __init__(self, *args, **kwargs): + super().__init__(args, kwargs) + self.room_group_name = None + self.event_slug = None + + async def connect(self): + self.event_slug = self.scope["url_route"]["kwargs"]["event_slug"] + self.room_group_name = f"chat_{self.event_slug}" + + # Join room group + await self.channel_layer.group_add(self.room_group_name, self.channel_name) + + await self.accept() + + async def disconnect(self, close_code): + # Leave room group + await self.channel_layer.group_discard(self.room_group_name, self.channel_name) + + # Receive message from WebSocket + async def receive(self, text_data): + text_data_json = json.loads(text_data) + message = text_data_json["message"] + + # Send message to room group + await self.channel_layer.group_send( + self.room_group_name, {"type": "chat.message", "message": message} + ) + + # Receive message from room group + async def chat_message(self, event): + message = event["message"] + + # Send message to WebSocket + await self.send(text_data=json.dumps({"message": message})) \ No newline at end of file diff --git a/core/notify_sessions/migrations/__init__.py b/core/notify_sessions/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/notify_sessions/routing.py b/core/notify_sessions/routing.py new file mode 100644 index 0000000..45e7428 --- /dev/null +++ b/core/notify_sessions/routing.py @@ -0,0 +1,8 @@ +from django.urls import path + +from .consumers import NotifyConsumer + +websocket_urlpatterns = [ + path('notify//', NotifyConsumer.as_asgi()), +] + diff --git a/core/requirements.txt b/core/requirements.txt index 0f42b18..5666822 100644 --- a/core/requirements.txt +++ b/core/requirements.txt @@ -18,7 +18,6 @@ itypes==1.2.0 Jinja2==3.1.2 MarkupSafe==2.1.3 msgpack-python==0.5.6 -mysqlclient openapi-codec==1.3.2 packaging==23.2 Pillow==10.1.0 @@ -27,9 +26,13 @@ pytz==2023.3.post1 PyYAML==6.0.1 redis requests==2.31.0 +sdnotify==0.3.2 +setproctitle==1.3.3 six==1.16.0 sqlparse==0.4.4 typing_extensions==4.8.0 uritemplate==4.1.1 urllib3==2.1.0 +uvicorn==0.24.0.post1 +watchfiles==0.21.0 websockets==12.0 diff --git a/core/server.py b/core/server.py new file mode 100644 index 0000000..407d0ba --- /dev/null +++ b/core/server.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +import asyncio +import logging + +import uvicorn +from aiosmtpd.controller import Controller, UnixSocketController +from aiosmtpd.lmtp import LMTP + + +async def handle_echo(reader, writer): + # session = Session() + data = await reader.read(100) + message = data.decode() + addr = writer.get_extra_info('peername') + + print(f"Received {message!r} from {addr!r}") + + print(f"Send: {message!r}") + writer.write(data) + await writer.drain() + + print("Close the connection") + writer.close() + await writer.wait_closed() + + +class ExampleHandler: + async def handle_RCPT(self, server, session, envelope, address, rcpt_options): + if not address.endswith('@example.com'): + return '550 not relaying to that domain' + envelope.rcpt_tos.append(address) + return '250 OK' + + async def handle_DATA(self, server, session, envelope): + print('Message from %s' % envelope.mail_from) + print('Message for %s' % envelope.rcpt_tos) + print('Message data:\n') + for ln in envelope.content.decode('utf8', errors='replace').splitlines(): + print(f'> {ln}'.strip()) + print() + print('End of message') + return '250 Message accepted for delivery' + + +class LTMPController(Controller): + def factory(self): + return LMTP(self.handler) + + +class UnixSocketLMTPController(UnixSocketController): + def factory(self): + return LMTP(self.handler) + + +class UvicornServer(uvicorn.Server): + def install_signal_handlers(self): + pass + + +async def web(): + log_config = uvicorn.config.LOGGING_CONFIG + log_config["handlers"]["default"] = {"class": "logging.FileHandler", "filename": "web.log", "formatter": "default"} + log_config["handlers"]["access"] = {"class": "logging.FileHandler", "filename": "web-access.log", + "formatter": "access"} + config = uvicorn.Config("core.asgi:application", uds="web.sock", log_config=log_config) + server = UvicornServer(config=config) + await server.serve() + + +async def tcp(): + log = logging.getLogger('test') + log.info("Starting TCP server") + server = await asyncio.start_unix_server(handle_echo, path='test.sock') + + addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets) + log.info(f'Serving on {addrs}') + + async with server: + await server.serve_forever() + log.info("TCP done") + + +async def lmtp(): + log = logging.getLogger('lmtp') + log.info("Starting LMTP server") + cont = UnixSocketLMTPController(ExampleHandler(), unix_socket='lmtp.sock') + cont.start() + log.info("LMTP done") + + +async def shutdown(sig, loop): + log = logging.getLogger() + log.info(f"Received exit signal {sig.name}...") + tasks = [t for t in asyncio.all_tasks() if t is not + asyncio.current_task()] + [task.cancel() for task in tasks] + log.info(f"Cancelling {len(tasks)} outstanding tasks") + await asyncio.wait_for(loop.shutdown_asyncgens(), timeout=10) + loop.stop() + log.info("Shutdown complete.") + + +def main(): + import sdnotify + import signal + import setproctitle + setproctitle.setproctitle("c3lf-sys3") + logging.basicConfig(filename='server.log', level=logging.DEBUG, encoding='utf-8') + logging.basicConfig(filename='test.log', level=logging.DEBUG, encoding='utf-8') + logging.basicConfig(filename='lmtp.log', level=logging.DEBUG, encoding='utf-8') + log = logging.getLogger() + log.info("Starting server") + loop = asyncio.get_event_loop() + loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop))) + loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop))) + loop.create_task(web()) + loop.create_task(tcp()) + loop.create_task(lmtp()) + n = sdnotify.SystemdNotifier() + n.notify("READY=1") + log.info("Server ready") + try: + loop.run_forever() + finally: + loop.close() + logging.info("Server stopped") + + +if __name__ == '__main__': + main() diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 0000000..611281b --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1 @@ +inventory.yml \ No newline at end of file diff --git a/deploy/ansible/inventory.yml.sample b/deploy/ansible/inventory.yml.sample new file mode 100644 index 0000000..87891c7 --- /dev/null +++ b/deploy/ansible/inventory.yml.sample @@ -0,0 +1,13 @@ +--- +c3lf-nodes: + hosts: + : + ansible_ssh_host: + ansible_ssh_user: root + web_domain: + git_branch: master + git_repo: + db_password: + main_email: + legacy_api_user: + legacy_api_password: \ No newline at end of file diff --git a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml new file mode 100644 index 0000000..8c0646a --- /dev/null +++ b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml @@ -0,0 +1,277 @@ +- name: 'deploy c3lf-sys3' + hosts: 'c3lf-nodes' + handlers: + - name: restart nginx + service: + name: nginx + state: restarted + + - name: restart postfix + service: + name: postfix + state: restarted + + - name: restart mariadb + service: + name: mariadb + state: restarted + + - name: restart c3lf-sys3 + service: + name: c3lf-sys3 + state: restarted + + tasks: + - name: Update apt-get repo and cache + apt: update_cache=yes force_apt_get=yes cache_valid_time=3600 + + - name: Upgrade all apt packages + apt: upgrade=dist force_apt_get=yes + + - name: Ansible apt-get to install base tools + apt: + name: + - htop + - tcpdump + - jq + - curl + - libsensors5 + - prometheus-node-exporter + - openssh-server + state: present + force_apt_get: yes + + - name: Remove useless packages from the cache + apt: + autoclean: yes + + - name: Remove dependencies that are no longer required + apt: + autoremove: yes + + - name: Check if a reboot is needed for debian + register: reboot_required_file + stat: path=/var/run/reboot-required get_md5=no + + - name: Reboot the Debian or Ubuntu server + reboot: + msg: "Reboot initiated by Ansible due to kernel updates" + connect_timeout: 5 + reboot_timeout: 300 + pre_reboot_delay: 0 + post_reboot_delay: 30 + test_command: uptime + when: reboot_required_file.stat.exists + + - name: Ansible apt-get to install sys3 requirements + apt: + name: + - ufw + - fail2ban + - nginx + - redis + - python3 + - python3-pip + - python3-venv + - python3-passlib + - certbot + - python3-certbot-nginx + - mariadb-server + - python3-dev + - default-libmysqlclient-dev + - build-essential + - postfix + - git + - pkg-config + - npm + state: present + + - name: remove default nginx site + file: + path: /etc/nginx/sites-enabled/default + state: absent + + - name: remove default nginx site + file: + path: /etc/nginx/sites-available/default + state: absent + + - name: UFW allow SSH + ufw: + rule: allow + port: 22 + proto: tcp + state: enabled + + - name: UFW logging off + ufw: + logging: off + + - name: Configure nginx + template: + src: templates/nginx.conf.j2 + dest: /etc/nginx/sites-available/c3lf-sys3.conf + notify: + - restart nginx + + - name: UFW allow http + ufw: + rule: allow + port: 80 + proto: tcp + state: enabled + + - name: UFW allow https + ufw: + rule: allow + port: 443 + proto: tcp + state: enabled + + - name: Check if initial certbot certificate is needed + stat: + path: /etc/letsencrypt/live/{{web_domain}}/fullchain.pem + register: certbot_cert_exists + + - block: + - name: stop nginx + service: + name: nginx + state: stopped + - name: add certbot domain + command: "certbot certonly --standalone -d {{web_domain}} --non-interactive --agree-tos --email {{main_email}}" + - name: start nginx + service: + name: nginx + state: started + when: certbot_cert_exists.stat.exists == false + + - name: Enable certbot auto renew + cron: + name: "certbot-auto renew" + minute: "0" + hour: "12" + job: "certbot renew --quiet --no-self-upgrade --nginx --cert-name {{web_domain}}" + state: present + + - name: Configure basic auth + htpasswd: + path: /etc/nginx/conf.d/lf-prod.htpasswd + name: "{{ legacy_api_user }}" + password: "{{ legacy_api_password }}" + state: present + notify: + - restart nginx + + - name: Enable nginx site + file: + src: /etc/nginx/sites-available/c3lf-sys3.conf + dest: /etc/nginx/sites-enabled/c3lf-sys3.conf + state: link + notify: + - restart nginx + + - name: create database + mysql_db: + name: c3lf_sys3 + state: present + login_unix_socket: /var/run/mysqld/mysqld.sock + + - name: create database user + mysql_user: + name: c3lf_sys3 + password: "{{ db_password }}" + priv: "c3lf_sys3.*:ALL" + state: present + login_unix_socket: /var/run/mysqld/mysqld.sock + + - name: configure webdir + file: + path: /var/www/c3lf-sys3 + state: directory + owner: www-data + group: www-data + mode: 0755 + + - name: install python app + become: true + become_user: www-data + become_method: su + become_flags: '-s /bin/bash' + block: + - name: create repo dir + git: + repo: "{{ git_repo }}" + dest: /var/www/c3lf-sys3/repo + version: "{{ git_branch }}" + force: yes + recursive: yes + single_branch: yes + register: git_repo + notify: + - restart c3lf-sys3 + + - name: check if venv exists + stat: + path: /var/www/c3lf-sys3/venv/bin/python3 + register: venv_exists + + - name: create venv + command: "python3 -m venv /var/www/c3lf-sys3/venv" + when: venv_exists.stat.exists == false + + - name: install requirements + pip: + requirements: /var/www/c3lf-sys3/repo/core/requirements.txt + virtualenv: /var/www/c3lf-sys3/venv + state: present + when: git_repo.changed == true + notify: + - restart c3lf-sys3 + + - name: configure django + template: + src: templates/django.env.j2 + dest: /var/www/c3lf-sys3/repo/core/.env + + - name: migrate database + command: "/var/www/c3lf-sys3/venv/bin/python /var/www/c3lf-sys3/repo/core/manage.py migrate" + when: git_repo.changed == true + + - name: collect static files + command: "/var/www/c3lf-sys3/venv/bin/python /var/www/c3lf-sys3/repo/core/manage.py collectstatic --noinput" + when: git_repo.changed == true + + - name: js config + template: + src: templates/config.js.j2 + dest: /var/www/c3lf-sys3/repo/web/src/config.js + + - name: build install dependencies + command: + cmd: "npm install" + chdir: /var/www/c3lf-sys3/repo/web + when: git_repo.changed == true + + - name: build frontend + command: + cmd: "npm run build" + chdir: /var/www/c3lf-sys3/repo/web + when: git_repo.changed == true + + - name: add c3lf-sys3 service + template: + src: templates/c3lf-sys3.service.j2 + dest: /etc/systemd/system/c3lf-sys3.service + notify: + - restart c3lf-sys3 + + - name: reload systemd + systemd: + daemon_reload: yes + + - name: start c3lf-sys3 service + service: + name: c3lf-sys3 + state: started + enabled: yes \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/c3lf-sys3.service.j2 b/deploy/ansible/playbooks/templates/c3lf-sys3.service.j2 new file mode 100644 index 0000000..ee90111 --- /dev/null +++ b/deploy/ansible/playbooks/templates/c3lf-sys3.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=standalone c3lf-sys3 server +After=network.target + +[Service] +Type=notify +WorkingDirectory=/var/www/c3lf-sys3 +ExecStart=/var/www/c3lf-sys3/venv/bin/python3 /var/www/c3lf-sys3/repo/core/server.py +Restart=always +RestartSec=5 +User=www-data +Group=www-data + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/config.js.j2 b/deploy/ansible/playbooks/templates/config.js.j2 new file mode 100644 index 0000000..b95ac7e --- /dev/null +++ b/deploy/ansible/playbooks/templates/config.js.j2 @@ -0,0 +1,9 @@ +export default { + service: { + url: 'https://{{ web_domain }}/api', + auth: { + username: '{{ legacy_api_user }}', + password: '{{ legacy_api_password }}' + } + } +}; \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 new file mode 100644 index 0000000..1246173 --- /dev/null +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -0,0 +1,10 @@ +DB_HOST=localhost +DB_PORT=3306 +DB_NAME=c3lf_sys3 +DB_USER=c3lf_sys3 +DB_PASSWORD={{ db_password }} +HTTP_HOST={{ web_domain }} +LEGACY_API_USER={{ legacy_api_user }} +LEGACY_API_PASSWORD={{ legacy_api_password }} +MEDIA_ROOT=/var/www/c3lf-sys3/userfiles +STATIC_ROOT=/var/www/c3lf-sys3/staticfiles diff --git a/deploy/ansible/playbooks/templates/nginx.conf.j2 b/deploy/ansible/playbooks/templates/nginx.conf.j2 new file mode 100644 index 0000000..154aaf9 --- /dev/null +++ b/deploy/ansible/playbooks/templates/nginx.conf.j2 @@ -0,0 +1,80 @@ +upstream c3lf-sys3 { + server unix:/var/www/c3lf-sys3/web.sock; +} + +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + + server_name {{ web_domain }}; + client_max_body_size 1024M; + + auth_basic C3LF; + auth_basic_user_file conf.d/lf-prod.htpasswd; + + location / { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, DELETE, PUT'; + add_header 'Content-Security-Policy' "default-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob: wss:;"; + root /var/www/c3lf-sys3/repo/web/dist; + index index.html index.htm index.nginx-debian.html; + try_files $uri $uri/ /index.html; + } + + location /api { + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://c3lf-sys3; + } + + location /admin { + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_redirect off; + proxy_buffering off; + proxy_pass http://c3lf-sys3; + } + + location /redirect_media/ { + internal; + alias /var/www/c3lf-sys3/userfiles/; + } + + location /redirect_thumbnail/ { + internal; + alias /var/www/c3lf-sys3/userfiles/thumbnails/; + } + + location /static/ { + alias /var/www/c3lf-sys3/staticfiles/; + } + + listen 443 ssl http2; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/{{ web_domain }}/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/{{ web_domain }}/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} + +server { + if ($host = {{ web_domain }}) { + return 301 https://$host$request_uri; + } # managed by Certbot + + server_name {{ web_domain }}; + + listen 80; + return 404; # managed by Certbot +}