From 3709b5dd29d101527a6a24aa51ca97559325f5e7 Mon Sep 17 00:00:00 2001 From: jedi Date: Thu, 5 Dec 2024 01:31:06 +0100 Subject: [PATCH] stash --- .forgejo/workflows/test.yml | 11 +++- core/.coveragerc | 2 +- core/authentication/api_v2.py | 4 +- core/authentication/tests/v2/test_users.py | 11 ++++ core/core/globals.py | 8 +-- core/core/settings.py | 32 +++++++--- core/helper.py | 30 ---------- core/integration_tests/main.py | 69 ++++++++++++++++++++++ core/integration_tests/test_bar.py | 1 + core/integration_tests/test_foo.sh | 5 ++ core/mail/protocol.py | 7 ++- core/mail/socket.py | 8 +-- core/server.py | 4 +- deploy/dev/docker-compose.yml | 28 ++------- deploy/testing/docker-compose.yml | 21 ++++++- 15 files changed, 156 insertions(+), 85 deletions(-) delete mode 100644 core/helper.py create mode 100644 core/integration_tests/main.py create mode 100644 core/integration_tests/test_bar.py create mode 100644 core/integration_tests/test_foo.sh diff --git a/.forgejo/workflows/test.yml b/.forgejo/workflows/test.yml index 480e590..14c7591 100644 --- a/.forgejo/workflows/test.yml +++ b/.forgejo/workflows/test.yml @@ -1,3 +1,4 @@ +name: Test on: pull_request: push: @@ -16,6 +17,12 @@ jobs: - name: Install dependencies working-directory: core run: pip3 install -r requirements.dev.txt - - name: Run django tests + - name: Run django tests with coverage working-directory: core - run: python3 manage.py test + run: coverage run manage.py test + - name: Run integration tests with coverage + working-directory: core + run: python3 integration_tests/main.py + - name: Evaluate coverage + working-directory: core + run: coverage report diff --git a/core/.coveragerc b/core/.coveragerc index 14c1fba..d06fc0e 100644 --- a/core/.coveragerc +++ b/core/.coveragerc @@ -8,7 +8,7 @@ skip_covered = True omit = */tests/* */migrations/* + integration_tests/* core/asgi.py - core/wsgi.py core/settings.py manage.py \ No newline at end of file diff --git a/core/authentication/api_v2.py b/core/authentication/api_v2.py index 2547b6d..b7ecfe3 100644 --- a/core/authentication/api_v2.py +++ b/core/authentication/api_v2.py @@ -54,9 +54,9 @@ def registerUser(request): errors['password'] = 'Password is required' if not email: errors['email'] = 'Email is required' - if ExtendedUser.objects.filter(email=email).exists(): + if email and ExtendedUser.objects.filter(email=email).exists(): errors['email'] = 'Email already exists' - if ExtendedUser.objects.filter(username=username).exists(): + if username and ExtendedUser.objects.filter(username=username).exists(): errors['username'] = 'Username already exists' if errors: return Response({'errors': errors}, status=400) diff --git a/core/authentication/tests/v2/test_users.py b/core/authentication/tests/v2/test_users.py index 125be9b..02096ea 100644 --- a/core/authentication/tests/v2/test_users.py +++ b/core/authentication/tests/v2/test_users.py @@ -72,6 +72,17 @@ class UserApiTest(TestCase): self.assertEqual(ExtendedUser.objects.get(username='testuser2').email, 'test2') self.assertTrue(ExtendedUser.objects.get(username='testuser2').check_password('test')) + def test_register_user_fail(self): + anonymous = Client() + response = anonymous.post('/api/2/register/', {'username': 'testuser2', 'password': 'test', 'email': 'test2'}, + content_type='application/json') + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json()['username'], 'testuser2') + self.assertEqual(response.json()['email'], 'test2') + self.assertEqual(len(ExtendedUser.objects.all()), 3) + self.assertEqual(ExtendedUser.objects.get(username='testuser2').email, 'test2') + self.assertTrue(ExtendedUser.objects.get(username='testuser2').check_password('test')) + def test_register_user_duplicate(self): anonymous = Client() response = anonymous.post('/api/2/register/', {'username': 'testuser', 'password': 'test', 'email': 'test2'}, diff --git a/core/core/globals.py b/core/core/globals.py index d84a7a0..eae7884 100644 --- a/core/core/globals.py +++ b/core/core/globals.py @@ -2,12 +2,7 @@ import asyncio import logging import signal -loop = asyncio.get_event_loop() - - -def create_task(coro): - global loop - loop.create_task(coro) +loop = None async def shutdown(sig, loop): @@ -24,6 +19,7 @@ async def shutdown(sig, loop): def init_loop(): global loop + loop = asyncio.get_event_loop() loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop))) loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop))) return loop diff --git a/core/core/settings.py b/core/core/settings.py index 5a8f20f..e9d6dea 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -124,19 +124,12 @@ TEMPLATES = [ }, ] -WSGI_APPLICATION = 'core.wsgi.application' +ASGI_APPLICATION = 'core.asgi.application' # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases -if 'test' in sys.argv: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', - } - } -else: +if os.getenv('DB_HOST') is not None: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', @@ -149,6 +142,20 @@ else: 'charset': 'utf8mb4', 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'" } + }, + } +elif os.getenv('DB_FILE') is not None: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.getenv('DB_FILE', 'local.db'), + } + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', } } @@ -223,6 +230,13 @@ CHANNEL_LAYERS = { } +OUTBOUND_SMTP = { + 'hostname': os.getenv('OUTBOUND_SMTP_HOST', '127.0.0.1'), + 'port': int(os.getenv('OUTBOUND_SMTP_PORT', '25')), + 'use_tls': False, + 'start_tls': False +} + PROMETHEUS_METRIC_NAMESPACE = 'c3lf' TEST_RUNNER = 'core.test_runner.FastTestRunner' diff --git a/core/helper.py b/core/helper.py deleted file mode 100644 index ae3c53b..0000000 --- a/core/helper.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -import logging -import signal - -loop = None - - -def create_task(coro): - global loop - loop.create_task(coro) - - -async def shutdown(sig, loop): - log = logging.getLogger() - log.info(f"Received exit signal {sig.name}...") - tasks = [t for t in asyncio.all_tasks() if t is not - asyncio.current_task()] - [task.cancel() for task in tasks] - log.info(f"Cancelling {len(tasks)} outstanding tasks") - await asyncio.wait_for(loop.shutdown_asyncgens(), timeout=10) - loop.stop() - log.info("Shutdown complete.") - - -def init_loop(): - global loop - loop = asyncio.get_event_loop() - loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop))) - loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop))) - return loop diff --git a/core/integration_tests/main.py b/core/integration_tests/main.py new file mode 100644 index 0000000..b0ef7a1 --- /dev/null +++ b/core/integration_tests/main.py @@ -0,0 +1,69 @@ +import time +import os +import signal +import re +import sys +import subprocess + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def run(): + while True: + newpid = os.fork() + if newpid == 0: + import coverage + cov = coverage.Coverage() + cov.load() + cov.start() + signal.signal(signal.SIGINT, signal.default_int_handler) + try: + from server import main + main() + except KeyboardInterrupt: + pass + cov.stop() + cov.save() + os._exit(0) + else: + return newpid + + +if __name__ == '__main__': + os.environ["DB_FILE"] = "local.sqlite3" + os.environ["OUTBOUND_SMTP_PORT"] = '1025' + pid = run() + result = subprocess.run(['python3', 'manage.py', 'migrate'], capture_output=True, text=True, env=os.environ) + if result.returncode != 0: + print('Migration failed', file=sys.stderr) + print(result.stdout, file=sys.stderr) + print(result.stderr, file=sys.stderr) + os._exit(1) + time.sleep(2) + script_dir = os.path.dirname(os.path.abspath(__file__)) + pattern = re.compile(r'^test_.*\.(sh|py)$') + dotpy = re.compile(r'^.*\.py$') + dotsh = re.compile(r'^.*\.sh$') + matching_files = [ + filename for filename in os.listdir(script_dir) + if os.path.isfile(os.path.join(script_dir, filename)) and pattern.match(filename) + ] + total = 0 + failed = 0 + for file in matching_files: + file_path = os.path.join(script_dir, file) + if dotpy.match(file): + result = subprocess.run(['python3', file_path], capture_output=True, text=True, env=os.environ) + elif dotsh.match(file): + result = subprocess.run(['bash', file_path], capture_output=True, text=True, env=os.environ) + else: + result = subprocess.run(['bash', '-c', file_path], capture_output=True, text=True, env=os.environ) + print('{} returned {}'.format(file, result.returncode), file=sys.stderr) + if result.returncode != 0: + print(result.stderr, result.stdout, file=sys.stderr) + failed += 1 + total += 1 + time.sleep(1) + os.kill(pid, signal.SIGINT) + print(f'{total - failed} out of {total} tests succeeded, {failed} failed', file=sys.stderr) + os._exit(0 if failed == 0 else 1) diff --git a/core/integration_tests/test_bar.py b/core/integration_tests/test_bar.py new file mode 100644 index 0000000..5f7ce86 --- /dev/null +++ b/core/integration_tests/test_bar.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 \ No newline at end of file diff --git a/core/integration_tests/test_foo.sh b/core/integration_tests/test_foo.sh new file mode 100644 index 0000000..65c2eaf --- /dev/null +++ b/core/integration_tests/test_foo.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +swaks --to user@localhost --socket lmtp.sock --protocol LMTP diff --git a/core/mail/protocol.py b/core/mail/protocol.py index c5aaa4a..668f656 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -88,7 +88,9 @@ def make_reply(reply_email, references=None, event=None): async def send_smtp(message): - await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False) + from core.settings import OUTBOUND_SMTP + if OUTBOUND_SMTP: + await aiosmtplib.send(message, **OUTBOUND_SMTP) def find_active_issue_thread(in_reply_to, address, subject, event): @@ -209,7 +211,8 @@ def receive_email(envelope, log=None): email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), + event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) diff --git a/core/mail/socket.py b/core/mail/socket.py index 312b916..5220795 100644 --- a/core/mail/socket.py +++ b/core/mail/socket.py @@ -25,7 +25,7 @@ class UnixSocketLMTPController(UnixSocketMixin, BaseAsyncController): def factory(self): return LMTP(self.handler) - def _trigger_server(self): # pragma: no-unixsock - # Prevent confusion on which _trigger_server() to invoke. - # Or so LGTM.com claimed - UnixSocketMixin._trigger_server(self) +# def _trigger_server(self): # pragma: no-unixsock +# # Prevent confusion on which _trigger_server() to invoke. +# # Or so LGTM.com claimed +# UnixSocketMixin._trigger_server(self) diff --git a/core/server.py b/core/server.py index d08b595..59dcb3f 100644 --- a/core/server.py +++ b/core/server.py @@ -9,7 +9,7 @@ import uvicorn django.setup() -from helper import init_loop +from core.globals import init_loop from mail.protocol import LMTPHandler from mail.socket import UnixSocketLMTPController @@ -51,7 +51,6 @@ async def lmtp(loop): async with server: await server.serve_forever() - log.info("LMTP done") def main(): @@ -82,7 +81,6 @@ def main(): os.remove("web.sock") except Exception as e: log.error(e) - log.error(e) logging.info("Server stopped") logging.shutdown() diff --git a/deploy/dev/docker-compose.yml b/deploy/dev/docker-compose.yml index dff5ab3..f7c9523 100644 --- a/deploy/dev/docker-compose.yml +++ b/deploy/dev/docker-compose.yml @@ -3,20 +3,15 @@ services: build: context: ../../core dockerfile: ../deploy/dev/Dockerfile.backend - command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000' + command: bash -c 'python manage.py migrate && python testdata.py && python manage.py runserver 0.0.0.0:8000' environment: - HTTP_HOST=core - - DB_HOST=db - - DB_PORT=3306 - - DB_NAME=system3 - - DB_USER=system3 - - DB_PASSWORD=system3 + - DB_FILE=dev.db volumes: - ../../core:/code + - ../testdata.py:/code/testdata.py ports: - "8000:8000" - depends_on: - - db frontend: build: @@ -30,19 +25,4 @@ services: ports: - "8080:8080" depends_on: - - core - - db: - image: mariadb - environment: - MARIADB_RANDOM_ROOT_PASSWORD: true - MARIADB_DATABASE: system3 - MARIADB_USER: system3 - MARIADB_PASSWORD: system3 - volumes: - - mariadb_data:/var/lib/mysql - ports: - - "3306:3306" - -volumes: - mariadb_data: \ No newline at end of file + - core \ No newline at end of file diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml index e93e901..a2ece91 100644 --- a/deploy/testing/docker-compose.yml +++ b/deploy/testing/docker-compose.yml @@ -20,7 +20,7 @@ services: build: context: ../../core dockerfile: ../deploy/testing/Dockerfile.backend - command: bash -c 'python manage.py migrate && python /code/server.py' + command: bash -c 'python manage.py migrate && python testdata.py && python /code/server.py' environment: - HTTP_HOST=core - REDIS_HOST=redis @@ -29,13 +29,16 @@ services: - DB_NAME=system3 - DB_USER=system3 - DB_PASSWORD=system3 + - MAIL_DOMAIN=mail:1025 volumes: - ../../core:/code + - ../testdata.py:/code/testdata.py ports: - "8000:8000" depends_on: - db - redis + - mail frontend: build: @@ -51,5 +54,19 @@ services: depends_on: - core + mail: + image: docker.io/axllent/mailpit + volumes: + - mailpit_data:/data + ports: + - 8025:8025 + - 1025:1025 + environment: + MP_MAX_MESSAGES: 5000 + MP_DATABASE: /data/mailpit.db + MP_SMTP_AUTH_ACCEPT_ANY: 1 + MP_SMTP_AUTH_ALLOW_INSECURE: 1 + volumes: - mariadb_data: \ No newline at end of file + mariadb_data: + mailpit_data: \ No newline at end of file