stash

.forgejo/workflows/test.yml

@@ -1,3 +1,4 @@
+name: Test
 on:
   pull_request:
   push:
@@ -16,6 +17,12 @@ jobs:
       - name: Install dependencies
         working-directory: core
         run: pip3 install -r requirements.dev.txt
-      - name: Run django tests
+      - name: Run django tests with coverage
         working-directory: core
-        run: python3 manage.py test
+        run: coverage run manage.py test
+      - name: Run integration tests with coverage
+        working-directory: core
+        run: python3 integration_tests/main.py
+      - name: Evaluate coverage
+        working-directory: core
+        run: coverage report

core/.coveragerc

@@ -8,7 +8,7 @@ skip_covered = True
 omit =
     */tests/*
     */migrations/*
+    integration_tests/*
     core/asgi.py
-    core/wsgi.py
     core/settings.py
     manage.py

core/authentication/api_v2.py

@@ -54,9 +54,9 @@ def registerUser(request):
             errors['password'] = 'Password is required'
         if not email:
             errors['email'] = 'Email is required'
-        if ExtendedUser.objects.filter(email=email).exists():
+        if email and ExtendedUser.objects.filter(email=email).exists():
             errors['email'] = 'Email already exists'
-        if ExtendedUser.objects.filter(username=username).exists():
+        if username and ExtendedUser.objects.filter(username=username).exists():
             errors['username'] = 'Username already exists'
         if errors:
             return Response({'errors': errors}, status=400)

core/authentication/tests/v2/test_users.py

@@ -72,6 +72,17 @@ class UserApiTest(TestCase):
         self.assertEqual(ExtendedUser.objects.get(username='testuser2').email, 'test2')
         self.assertTrue(ExtendedUser.objects.get(username='testuser2').check_password('test'))
 
+    def test_register_user_fail(self):
+        anonymous = Client()
+        response = anonymous.post('/api/2/register/', {'username': 'testuser2', 'password': 'test', 'email': 'test2'},
+                                  content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(response.json()['username'], 'testuser2')
+        self.assertEqual(response.json()['email'], 'test2')
+        self.assertEqual(len(ExtendedUser.objects.all()), 3)
+        self.assertEqual(ExtendedUser.objects.get(username='testuser2').email, 'test2')
+        self.assertTrue(ExtendedUser.objects.get(username='testuser2').check_password('test'))
+
     def test_register_user_duplicate(self):
         anonymous = Client()
         response = anonymous.post('/api/2/register/', {'username': 'testuser', 'password': 'test', 'email': 'test2'},

core/core/globals.py

@@ -2,12 +2,7 @@ import asyncio
 import logging
 import signal
 
-loop = asyncio.get_event_loop()
+loop = None
-
-
-def create_task(coro):
-    global loop
-    loop.create_task(coro)
 
 
 async def shutdown(sig, loop):
@@ -24,6 +19,7 @@ async def shutdown(sig, loop):
 
 def init_loop():
     global loop
+    loop = asyncio.get_event_loop()
     loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop)))
     loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop)))
     return loop

core/core/settings.py

@@ -124,19 +124,12 @@ TEMPLATES = [
     },
 ]
 
-WSGI_APPLICATION = 'core.wsgi.application'
+ASGI_APPLICATION = 'core.asgi.application'
 
 # Database
 # https://docs.djangoproject.com/en/4.2/ref/settings/#databases
 
-if 'test' in sys.argv:
+if os.getenv('DB_HOST') is not None:
-    DATABASES = {
-        'default': {
-            'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': ':memory:',
-        }
-    }
-else:
     DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.mysql',
@@ -149,6 +142,20 @@ else:
                 'charset': 'utf8mb4',
                 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'"
             }
+        },
+    }
+elif os.getenv('DB_FILE') is not None:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': os.getenv('DB_FILE', 'local.db'),
+        }
+    }
+else:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': ':memory:',
         }
     }
 
@@ -223,6 +230,13 @@ CHANNEL_LAYERS = {
 
 }
 
+OUTBOUND_SMTP = {
+    'hostname': os.getenv('OUTBOUND_SMTP_HOST', '127.0.0.1'),
+    'port': int(os.getenv('OUTBOUND_SMTP_PORT', '25')),
+    'use_tls': False,
+    'start_tls': False
+}
+
 PROMETHEUS_METRIC_NAMESPACE = 'c3lf'
 
 TEST_RUNNER = 'core.test_runner.FastTestRunner'

core/helper.py

@@ -1,30 +0,0 @@
-import asyncio
-import logging
-import signal
-
-loop = None
-
-
-def create_task(coro):
-    global loop
-    loop.create_task(coro)
-
-
-async def shutdown(sig, loop):
-    log = logging.getLogger()
-    log.info(f"Received exit signal {sig.name}...")
-    tasks = [t for t in asyncio.all_tasks() if t is not
-             asyncio.current_task()]
-    [task.cancel() for task in tasks]
-    log.info(f"Cancelling {len(tasks)} outstanding tasks")
-    await asyncio.wait_for(loop.shutdown_asyncgens(), timeout=10)
-    loop.stop()
-    log.info("Shutdown complete.")
-
-
-def init_loop():
-    global loop
-    loop = asyncio.get_event_loop()
-    loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop)))
-    loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop)))
-    return loop

core/integration_tests/main.py

@@ -0,0 +1,69 @@
+import time
+import os
+import signal
+import re
+import sys
+import subprocess
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def run():
+    while True:
+        newpid = os.fork()
+        if newpid == 0:
+            import coverage
+            cov = coverage.Coverage()
+            cov.load()
+            cov.start()
+            signal.signal(signal.SIGINT, signal.default_int_handler)
+            try:
+                from server import main
+                main()
+            except KeyboardInterrupt:
+                pass
+            cov.stop()
+            cov.save()
+            os._exit(0)
+        else:
+            return newpid
+
+
+if __name__ == '__main__':
+    os.environ["DB_FILE"] = "local.sqlite3"
+    os.environ["OUTBOUND_SMTP_PORT"] = '1025'
+    pid = run()
+    result = subprocess.run(['python3', 'manage.py', 'migrate'], capture_output=True, text=True, env=os.environ)
+    if result.returncode != 0:
+        print('Migration failed', file=sys.stderr)
+        print(result.stdout, file=sys.stderr)
+        print(result.stderr, file=sys.stderr)
+        os._exit(1)
+    time.sleep(2)
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    pattern = re.compile(r'^test_.*\.(sh|py)$')
+    dotpy = re.compile(r'^.*\.py$')
+    dotsh = re.compile(r'^.*\.sh$')
+    matching_files = [
+        filename for filename in os.listdir(script_dir)
+        if os.path.isfile(os.path.join(script_dir, filename)) and pattern.match(filename)
+    ]
+    total = 0
+    failed = 0
+    for file in matching_files:
+        file_path = os.path.join(script_dir, file)
+        if dotpy.match(file):
+            result = subprocess.run(['python3', file_path], capture_output=True, text=True, env=os.environ)
+        elif dotsh.match(file):
+            result = subprocess.run(['bash', file_path], capture_output=True, text=True, env=os.environ)
+        else:
+            result = subprocess.run(['bash', '-c', file_path], capture_output=True, text=True, env=os.environ)
+        print('{} returned {}'.format(file, result.returncode), file=sys.stderr)
+        if result.returncode != 0:
+            print(result.stderr, result.stdout, file=sys.stderr)
+            failed += 1
+        total += 1
+    time.sleep(1)
+    os.kill(pid, signal.SIGINT)
+    print(f'{total - failed} out of {total} tests succeeded, {failed} failed', file=sys.stderr)
+    os._exit(0 if failed == 0 else 1)

core/integration_tests/test_bar.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

core/integration_tests/test_foo.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+swaks --to user@localhost --socket lmtp.sock --protocol LMTP

core/mail/protocol.py

@@ -88,7 +88,9 @@ def make_reply(reply_email, references=None, event=None):
 
 
 async def send_smtp(message):
-    await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False)
+    from core.settings import OUTBOUND_SMTP
+    if OUTBOUND_SMTP:
+        await aiosmtplib.send(message, **OUTBOUND_SMTP)
 
 
 def find_active_issue_thread(in_reply_to, address, subject, event):
@@ -209,7 +211,8 @@ def receive_email(envelope, log=None):
 
     email = Email.objects.create(
         sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id,
-        in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event,
+        in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename),
+        event=target_event,
         issue_thread=active_issue_thread)
     for attachment in attachments:
         email.attachments.add(attachment)

core/mail/socket.py

@@ -25,7 +25,7 @@ class UnixSocketLMTPController(UnixSocketMixin, BaseAsyncController):
     def factory(self):
         return LMTP(self.handler)
 
-    def _trigger_server(self):  # pragma: no-unixsock
+#    def _trigger_server(self):  # pragma: no-unixsock
-        # Prevent confusion on which _trigger_server() to invoke.
+#        # Prevent confusion on which _trigger_server() to invoke.
-        # Or so LGTM.com claimed
+#        # Or so LGTM.com claimed
-        UnixSocketMixin._trigger_server(self)
+#        UnixSocketMixin._trigger_server(self)

core/server.py

@@ -9,7 +9,7 @@ import uvicorn
 
 django.setup()
 
-from helper import init_loop
+from core.globals import init_loop
 from mail.protocol import LMTPHandler
 from mail.socket import UnixSocketLMTPController
 
@@ -51,7 +51,6 @@ async def lmtp(loop):
 
     async with server:
         await server.serve_forever()
-    log.info("LMTP done")
 
 
 def main():
@@ -82,7 +81,6 @@ def main():
         os.remove("web.sock")
     except Exception as e:
         log.error(e)
-        log.error(e)
     logging.info("Server stopped")
 
     logging.shutdown()

deploy/dev/docker-compose.yml

@@ -3,20 +3,15 @@ services:
     build:
       context: ../../core
       dockerfile: ../deploy/dev/Dockerfile.backend
-    command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000'
+    command: bash -c 'python manage.py migrate && python testdata.py && python manage.py runserver 0.0.0.0:8000'
     environment:
       - HTTP_HOST=core
-      - DB_HOST=db
+      - DB_FILE=dev.db
-      - DB_PORT=3306
-      - DB_NAME=system3
-      - DB_USER=system3
-      - DB_PASSWORD=system3
     volumes:
       - ../../core:/code
+      - ../testdata.py:/code/testdata.py
     ports:
       - "8000:8000"
-    depends_on:
-      - db
 
   frontend:
     build:
@@ -30,19 +25,4 @@ services:
     ports:
       - "8080:8080"
     depends_on:
-      - core
+      - core
-
-  db:
-    image: mariadb
-    environment:
-      MARIADB_RANDOM_ROOT_PASSWORD: true
-      MARIADB_DATABASE: system3
-      MARIADB_USER: system3
-      MARIADB_PASSWORD: system3
-    volumes:
-      - mariadb_data:/var/lib/mysql
-    ports:
-      - "3306:3306"
-
-volumes:
-  mariadb_data:

deploy/testing/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     build:
       context: ../../core
       dockerfile: ../deploy/testing/Dockerfile.backend
-    command: bash -c 'python manage.py migrate && python /code/server.py'
+    command: bash -c 'python manage.py migrate &&  python testdata.py && python /code/server.py'
     environment:
       - HTTP_HOST=core
       - REDIS_HOST=redis
@@ -29,13 +29,16 @@ services:
       - DB_NAME=system3
       - DB_USER=system3
       - DB_PASSWORD=system3
+      - MAIL_DOMAIN=mail:1025
     volumes:
       - ../../core:/code
+      - ../testdata.py:/code/testdata.py
     ports:
       - "8000:8000"
     depends_on:
       - db
       - redis
+      - mail
 
   frontend:
     build:
@@ -51,5 +54,19 @@ services:
     depends_on:
       - core
 
+  mail:
+    image: docker.io/axllent/mailpit
+    volumes:
+      - mailpit_data:/data
+    ports:
+      - 8025:8025
+      - 1025:1025
+    environment:
+      MP_MAX_MESSAGES: 5000
+      MP_DATABASE: /data/mailpit.db
+      MP_SMTP_AUTH_ACCEPT_ANY: 1
+      MP_SMTP_AUTH_ALLOW_INSECURE: 1
+
 volumes:
-  mariadb_data:
+  mariadb_data:
+  mailpit_data: