add testdata to docker setup. use sqlite in 'dev' scenario and mailpit in 'testing'

core/core/management/commands/update_metrics.py

@@ -1,8 +0,0 @@
-from django.core.management.base import BaseCommand, CommandError
-from core.metrics import update_item_count
-
-class Command(BaseCommand):
-    help = "Refresh the metrics"
-    
-    def handle(self, *args, **options):
-        update_item_count()

core/core/metrics.py

@@ -1,11 +0,0 @@
-from prometheus_client import Gauge
-from django_prometheus import exports
-from inventory.models import *
-
-g_items_total = Gauge('c3lf_items_total', 'Current Total items')
-
-def update_item_count():
-    # Get the count of MyModel objects
-    count = Item.objects.count()
-    # Set the gauge to the current count
-    g_items_total.set(count)

core/core/settings.py

@@ -70,7 +70,6 @@ INSTALLED_APPS = [
     'inventory',
     'mail',
     'notify_sessions',
-    'core'
 ]
 
 REST_FRAMEWORK = {
@@ -125,19 +124,12 @@ TEMPLATES = [
     },
 ]
 
-WSGI_APPLICATION = 'core.wsgi.application'
+ASGI_APPLICATION = 'core.asgi.application'
 
 # Database
 # https://docs.djangoproject.com/en/4.2/ref/settings/#databases
 
-if 'test' in sys.argv:
-    DATABASES = {
-        'default': {
-            'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': ':memory:',
-        }
-    }
-else:
+if os.getenv('DB_HOST') is not None:
     DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.mysql',
@@ -150,6 +142,20 @@ else:
                 'charset': 'utf8mb4',
                 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'"
             }
+        },
+    }
+elif os.getenv('DB_FILE') is not None:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': os.getenv('DB_FILE', 'local.db'),
+        }
+    }
+else:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': ':memory:',
         }
     }
 

core/core/urls.py

@@ -19,8 +19,6 @@ from django.urls import path, include
 
 from .version import get_info
 
-from .metrics import *
-
 urlpatterns = [
     path('djangoadmin/', admin.site.urls),
     path('api/2/', include('inventory.api_v2')),

deploy/ansible/playbooks/deploy-c3lf-sys3.yml

@@ -311,16 +311,6 @@
       notify:
         - restart c3lf-sys3
 
-    - name: add c3lf-sys3-cron service
-      template:
-        src: templates/c3lf-sys3-cron.service.j2
-        dest: /etc/systemd/system/c3lf-sys3-cron.service
-
-    - name: add c3lf-sys3-cron timer
-      template:
-        src: templates/c3lf-sys3-cron.timer.j2
-        dest: /etc/systemd/system/c3lf-sys3-cron.timer
-
     - name: reload systemd
       systemd:
         daemon_reload: yes
@@ -331,12 +321,6 @@
         state: started
         enabled: yes
 
-    - name: start c3lf-sys3 timer
-      service:
-        name: c3lf-sys3-cron.timer
-        state: started
-        enabled: yes
-
     - name: add postfix to www-data group
       user:
         name: postfix
@@ -361,13 +345,6 @@
       notify:
         - restart postfix
 
-    - name: configure rspamd dkim
-      template:
-        src: templates/rspamd-dkim.cf.j2
-        dest: /etc/rspamd/local.d/dkim_signing.conf
-      notify:
-        - restart rspamd
-
     - name: configure rspamd
       copy:
         content: |

deploy/ansible/playbooks/templates/c3lf-sys3-cron.service.j2

@@ -1,13 +0,0 @@
-[Unit]
-Description=c3lf sys3 background service
-After=network.target
-
-[Service]
-User=www-data
-Group=www-data
-WorkingDirectory=/var/www/c3lf-sys3
-ExecStart=/var/www/c3lf-sys3/venv/bin/python3 /var/www/c3lf-sys3/repo/core/manage.py update_metrics
-Environment=DJANGO_SETTINGS_MODULE=core.settings
-
-[Install]
-WantedBy=multi-user.target

deploy/ansible/playbooks/templates/c3lf-sys3-cron.timer.j2

@@ -1,9 +0,0 @@
-[Unit]
-Description=Run Nextcloud cron.php every 5 minutes
-
-[Timer]
-OnCalendar=minutely
-Unit=c3lf-sys3-cron.service
-
-[Install]
-WantedBy=timers.target

deploy/ansible/playbooks/templates/postfix.cf.j2

@@ -37,6 +37,7 @@ alias_maps = hash:/etc/aliases
 alias_database = hash:/etc/aliases
 myorigin = /etc/mailname
 mydestination = $myhostname, , localhost
+relayhost = firefly.lab.or.it
 mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
 mailbox_size_limit = 0
 recipient_delimiter = +

deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2

@@ -1,79 +0,0 @@
-# local.d/dkim_signing.conf
-
-enabled = true;
-
-# If false, messages with empty envelope from are not signed
-allow_envfrom_empty = true;
-
-# If true, envelope/header domain mismatch is ignored
-allow_hdrfrom_mismatch = false;
-
-# If true, multiple from headers are allowed (but only first is used)
-allow_hdrfrom_multiple = false;
-
-# If true, username does not need to contain matching domain
-allow_username_mismatch = false;
-
-# Default path to key, can include '$domain' and '$selector' variables
-path = "/var/lib/rspamd/dkim/$domain.$selector.key";
-
-# Default selector to use
-selector = "dkim";
-
-# If false, messages from authenticated users are not selected for signing
-sign_authenticated = true;
-
-# If false, messages from local networks are not selected for signing
-sign_local = true;
-
-# Map file of IP addresses/subnets to consider for signing
-# sign_networks = "/some/file"; # or url
-
-# Symbol to add when message is signed
-symbol = "DKIM_SIGNED";
-
-# Whether to fallback to global config
-try_fallback = true;
-
-# Domain to use for DKIM signing: can be "header" (MIME From), "envelope" (SMTP From), "recipient" (SMTP To), "auth" (SMTP username) or directly specified domain name
-use_domain = "header";
-
-# Domain to use for DKIM signing when sender is in sign_networks ("header"/"envelope"/"auth")
-#use_domain_sign_networks = "header";
-
-# Domain to use for DKIM signing when sender is a local IP ("header"/"envelope"/"auth")
-#use_domain_sign_local = "header";
-
-# Whether to normalise domains to eSLD
-use_esld = true;
-
-# Whether to get keys from Redis
-use_redis = false;
-
-# Hash for DKIM keys in Redis
-key_prefix = "DKIM_KEYS";
-
-# map of domains -> names of selectors (since rspamd 1.5.3)
-#selector_map = "/etc/rspamd/dkim_selectors.map";
-
-# map of domains -> paths to keys (since rspamd 1.5.3)
-#path_map = "/etc/rspamd/dkim_paths.map";
-
-# If `true` get pubkey from DNS record and check if it matches private key
-check_pubkey = false;
-# Set to `false` if you want to skip signing if public and private keys mismatch
-allow_pubkey_mismatch = true;
-
-# Domain specific settings
-domain {
-  # Domain name is used as key
-  {{ mail_domain }} {
-
-    # Private key path
-    path = "/var/lib/rspamd/dkim/{{ mail_domain }}.key";
-
-    # Selector
-    selector = "ds";
-  }
-}
-

deploy/dev/docker-compose.yml

@@ -3,20 +3,15 @@ services:
     build:
       context: ../../core
       dockerfile: ../deploy/dev/Dockerfile.backend
-    command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000'
+    command: bash -c 'python manage.py migrate && python testdata.py && python manage.py runserver 0.0.0.0:8000'
     environment:
       - HTTP_HOST=core
-      - DB_HOST=db
-      - DB_PORT=3306
-      - DB_NAME=system3
-      - DB_USER=system3
-      - DB_PASSWORD=system3
+      - DB_FILE=dev.db
     volumes:
       - ../../core:/code
+      - ../testdata.py:/code/testdata.py
     ports:
       - "8000:8000"
-    depends_on:
-      - db
 
   frontend:
     build:
@@ -31,18 +26,3 @@ services:
       - "8080:8080"
     depends_on:
       - core
-
-  db:
-    image: mariadb
-    environment:
-      MARIADB_RANDOM_ROOT_PASSWORD: true
-      MARIADB_DATABASE: system3
-      MARIADB_USER: system3
-      MARIADB_PASSWORD: system3
-    volumes:
-      - mariadb_data:/var/lib/mysql
-    ports:
-      - "3306:3306"
-
-volumes:
-  mariadb_data:

deploy/testdata.py

@@ -0,0 +1,88 @@
+import os
+
+
+def setup():
+    from authentication.models import ExtendedUser, EventPermission
+    from inventory.models import Event
+    from django.contrib.auth.models import Permission, Group
+    permissions = ['add_item', 'view_item', 'view_file', 'delete_item', 'change_item']
+    if not ExtendedUser.objects.filter(username='admin').exists():
+        admin = ExtendedUser.objects.create_superuser('admin', 'admin@example.com', 'admin')
+        admin.set_password('admin')
+        admin.user_permissions.add(*Permission.objects.all())
+        admin.save()
+
+    if not ExtendedUser.objects.filter(username='testuser').exists():
+        testuser = ExtendedUser.objects.create_user('testuser', 'testuser@example.com', 'testuser')
+        testuser.set_password('testuser')
+        testuser.user_permissions.add(*Permission.objects.all())
+        testuser.save()
+
+    team = Group.objects.get(name='Team')
+    team.permissions.add(
+        *Permission.objects.all()
+    )
+
+    if not ExtendedUser.objects.filter(username='testuser2').exists():
+        testuser2 = ExtendedUser.objects.create_user('testuser2', 'testuser2@example.com', 'testuser2')
+        testuser2.set_password('testuser2')
+        testuser2.groups.add(team)
+        testuser2.save()
+
+    event1 = Event.objects.get_or_create(id=1, name='first test event', slug='TEST1',
+                                         start='2023-12-18 00:00:00.000000', end='2023-12-27 00:00:00.000000',
+                                         pre_start='2023-12-31 00:00:00.000000', post_end='2024-01-04 00:00:00.000000')[
+        0]
+
+    event2 = Event.objects.get_or_create(id=2, name='second test event', slug='TEST2',
+                                         start='2024-12-18 00:00:00.000000', end='2024-12-27 00:00:00.000000',
+                                         pre_start='2024-12-31 00:00:00.000000', post_end='2025-01-04 00:00:00.000000')[
+        0]
+
+    #    for permission in permissions:
+    #        EventPermission.objects.create(event=event_37c3, user=foo,
+    #                                       permission=Permission.objects.get(codename=permission))
+
+    from tickets.models import IssueThread
+
+    from mail.models import Email
+
+    issue_thread = IssueThread.objects.get_or_create(
+        id=1,
+        name="test",
+        event=Event.objects.get(slug='TEST1')
+    )[0]
+    mail1 = Email.objects.get_or_create(
+        id=1,
+        subject='test subject',
+        body='test',
+        sender='test1@test',
+        recipient='test2@test',
+        issue_thread=issue_thread,
+    )[0]
+    mail1_reply = Email.objects.get_or_create(
+        id=2,
+        subject='Message received',
+        body='Thank you for your message.',
+        sender='test2@test',
+        recipient='test1@test',
+        in_reply_to=mail1.reference,
+        issue_thread=issue_thread,
+    )[0]
+
+
+def main():
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings")
+    import django
+
+    django.setup()
+
+    from django.core.management import call_command
+    call_command('migrate')
+
+    setup()
+    print('testdata initialised')
+
+
+if __name__ == '__main__':
+    main()

deploy/testing/docker-compose.yml

@@ -20,7 +20,7 @@ services:
     build:
       context: ../../core
       dockerfile: ../deploy/testing/Dockerfile.backend
-    command: bash -c 'python manage.py migrate && python /code/server.py'
+    command: bash -c 'python manage.py migrate &&  python testdata.py && python /code/server.py'
     environment:
       - HTTP_HOST=core
       - REDIS_HOST=redis
@@ -29,13 +29,16 @@ services:
       - DB_NAME=system3
       - DB_USER=system3
       - DB_PASSWORD=system3
+      - MAIL_DOMAIN=mail:1025
     volumes:
       - ../../core:/code
+      - ../testdata.py:/code/testdata.py
     ports:
       - "8000:8000"
     depends_on:
       - db
       - redis
+      - mail
 
   frontend:
     build:
@@ -51,5 +54,19 @@ services:
     depends_on:
       - core
 
+  mail:
+    image: docker.io/axllent/mailpit
+    volumes:
+      - mailpit_data:/data
+    ports:
+      - 8025:8025
+      - 1025:1025
+    environment:
+      MP_MAX_MESSAGES: 5000
+      MP_DATABASE: /data/mailpit.db
+      MP_SMTP_AUTH_ACCEPT_ANY: 1
+      MP_SMTP_AUTH_ALLOW_INSECURE: 1
+
 volumes:
-  mariadb_data:
+  mariadb_data:
+  mailpit_data: