cicd: Add testing

core/core/settings.py

@@ -15,9 +15,6 @@ import sys
 import dotenv
 from pathlib import Path
 
-def truthy_str(s):
-    return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', '👍']
-
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -27,10 +24,10 @@ dotenv.load_dotenv(BASE_DIR / '.env')
 # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv')
+SECRET_KEY = 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv'
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = truthy_str(os.getenv('DEBUG_MODE_ACTIVE', 'False'))
+DEBUG = True
 
 ALLOWED_HOSTS = [os.getenv('HTTP_HOST', 'localhost')]
 
@@ -43,8 +40,6 @@ LEGACY_USER_PASSWORD = os.getenv('LEGACY_API_PASSWORD', 'legacy_password')
 
 SYSTEM3_VERSION = "0.0.0-dev.0"
 
-ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False'))
-
 # Application definition
 
 INSTALLED_APPS = [
@@ -55,7 +50,6 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django_extensions',
-    'django_prometheus',
     'rest_framework',
     'knox',
     'drf_yasg',
@@ -91,7 +85,6 @@ SWAGGER_SETTINGS = {
 }
 
 MIDDLEWARE = [
-    'django_prometheus.middleware.PrometheusBeforeMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
@@ -99,7 +92,6 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'django_prometheus.middleware.PrometheusAfterMiddleware',
 ]
 
 ROOT_URLCONF = 'core.urls'
@@ -212,12 +204,10 @@ CHANNEL_LAYERS = {
     'default': {
         'BACKEND': 'channels_redis.core.RedisChannelLayer',
         'CONFIG': {
-            'hosts': [(os.getenv('REDIS_HOST', 'localhost'), 6379)],
+            'hosts': [('localhost', 6379)],
         },
     }
 
 }
 
-PROMETHEUS_METRIC_NAMESPACE = 'c3lf'
-
 TEST_RUNNER = 'core.test_runner.FastTestRunner'

core/core/urls.py

@@ -32,5 +32,4 @@ urlpatterns = [
     path('api/2/', include('notify_sessions.api_v2')),
     path('api/2/', include('authentication.api_v2')),
     path('api/', get_info),
-    path('', include('django_prometheus.urls')),
 ]

core/mail/migrations/0006_email_raw_file.py

@@ -1,36 +0,0 @@
-# Generated by Django 4.2.7 on 2024-11-08 20:37
-from django.core.files.base import ContentFile
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ('mail', '0005_alter_eventaddress_event'),
-    ]
-
-    def move_raw_mails_to_file(apps, schema_editor):
-        Email = apps.get_model('mail', 'Email')
-        for email in Email.objects.all():
-            raw_content = email.raw
-            path = "mail_{}".format(email.id)
-            if len(raw_content):
-                email.raw_file.save(path, ContentFile(raw_content))
-            email.save()
-
-    operations = [
-        migrations.AddField(
-            model_name='email',
-            name='raw_file',
-            field=models.FileField(null=True, upload_to='raw_mail/'),
-        ),
-        migrations.RunPython(move_raw_mails_to_file),
-        migrations.RemoveField(
-            model_name='email',
-            name='raw',
-        ),
-        migrations.AlterField(
-            model_name='email',
-            name='raw_file',
-            field=models.FileField(upload_to='raw_mail/'),
-        ),
-    ]

core/mail/models.py

@@ -3,7 +3,7 @@ import random
 from django.db import models
 from django_softdelete.models import SoftDeleteModel
 
-from core.settings import MAIL_DOMAIN, ACTIVE_SPAM_TRAINING
+from core.settings import MAIL_DOMAIN
 from files.models import AbstractFile
 from inventory.models import Event
 from tickets.models import IssueThread
@@ -18,7 +18,7 @@ class Email(SoftDeleteModel):
     recipient = models.CharField(max_length=255)
     reference = models.CharField(max_length=255, null=True, unique=True)
     in_reply_to = models.CharField(max_length=255, null=True)
-    raw_file = models.FileField(upload_to='raw_mail/')
+    raw = models.TextField()
     issue_thread = models.ForeignKey(IssueThread, models.SET_NULL, null=True, related_name='emails')
     event = models.ForeignKey(Event, models.SET_NULL, null=True)
 
@@ -28,18 +28,6 @@ class Email(SoftDeleteModel):
             self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>'
             self.save()
 
-    def train_spam(self):
-        if ACTIVE_SPAM_TRAINING:
-            import subprocess
-            path = self.raw_file.path
-            subprocess.run(["rspamc", "learn_spam", path])
-
-    def train_ham(self):
-        if ACTIVE_SPAM_TRAINING:
-            import subprocess
-            path = self.raw_file.path
-            subprocess.run(["rspamc", "learn_ham", path])
-
 
 class EventAddress(models.Model):
     id = models.AutoField(primary_key=True)

core/mail/protocol.py

@@ -206,7 +206,7 @@ def receive_email(envelope, log=None):
 
     email = Email.objects.create(
         sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id,
-        in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content), event=target_event,
+        in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event,
         issue_thread=active_issue_thread)
     for attachment in attachments:
         email.attachments.add(attachment)

core/requirements.dev.txt

@@ -73,5 +73,3 @@ watchfiles==0.21.0
 websockets==12.0
 yarl==1.9.4
 zope.interface==6.1
-django-prometheus==2.3.1
-prometheus_client==0.21.0

core/requirements.prod.txt

@@ -41,5 +41,3 @@ urllib3==2.1.0
 uvicorn==0.24.0.post1
 watchfiles==0.21.0
 websockets==12.0
-django-prometheus==2.3.1
-prometheus_client==0.21.0

core/tickets/migrations/0011_train_old_spam.py

@@ -1,31 +0,0 @@
-# Generated by Django 4.2.7 on 2024-06-23 02:17
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ('mail', '0006_email_raw_file'),
-        ('tickets', '0010_issuethread_event_itemrelation_and_more'),
-    ]
-
-    def train_old_mails(apps, schema_editor):
-        from tickets.models import IssueThread
-        for t in IssueThread.objects.all():
-            try:
-                state = t.state
-                i = 0
-                for e in t.emails.all():
-                    if e.raw_file:
-                        if state == 'closed_spam' and i == 0:
-                            e.train_spam()
-                        else:
-                            e.train_ham()
-                    i += 1
-            except:
-                pass
-
-    operations = [
-        migrations.RunPython(train_old_mails),
-    ]

core/tickets/models.py

@@ -60,8 +60,6 @@ class IssueThread(SoftDeleteModel):
         if self.state == value:
             return
         self.state_changes.create(state=value)
-        if value == 'closed_spam' and self.emails.exists():
-            self.emails.first().train_spam()
 
     @property
     def assigned_to(self):

deploy/ansible/inventory.yml.sample

@@ -11,6 +11,4 @@ c3lf-nodes:
       mail_domain: <mail_domain>
       main_email: <main_email>
       legacy_api_user: <legacy_api_user>
-      legacy_api_password: <legacy_api_password>
-      debug_mode_active: false
-      django_secret_key: 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv'
+      legacy_api_password: <legacy_api_password>

deploy/ansible/playbooks/templates/django.env.j2

@@ -1,4 +1,3 @@
-REDIS_HOST=localhost
 DB_HOST=localhost
 DB_PORT=3306
 DB_NAME=c3lf_sys3
@@ -10,6 +9,3 @@ LEGACY_API_USER={{ legacy_api_user }}
 LEGACY_API_PASSWORD={{ legacy_api_password }}
 MEDIA_ROOT=/var/www/c3lf-sys3/userfiles
 STATIC_ROOT=/var/www/c3lf-sys3/staticfiles
-ACTIVE_SPAM_TRAINING=True
-DEBUG_MODE_ACTIVE={{ debug_mode_active }}
-DJANGO_SECRET_KEY={{ django_secret_key }}

deploy/testing/Dockerfile.backend

@@ -1,11 +0,0 @@
-FROM python:3.11-bookworm
-LABEL authors="lagertonne"
-
-ENV PYTHONUNBUFFERED 1
-RUN mkdir /code
-WORKDIR /code
-COPY requirements.prod.txt /code/
-RUN apt update && apt install -y mariadb-client
-RUN pip install -r requirements.prod.txt
-RUN pip install mysqlclient
-COPY .. /code/

deploy/testing/Dockerfile.frontend

@@ -1,6 +0,0 @@
-FROM docker.io/node:22
-
-RUN mkdir /web
-WORKDIR /web
-COPY package.json /web/
-RUN npm install

deploy/testing/docker-compose.yml

@@ -1,55 +0,0 @@
-services:
-  redis:
-    image: redis
-    ports:
-      - "6379:6379"
-
-  db:
-    image: mariadb
-    environment:
-      MARIADB_RANDOM_ROOT_PASSWORD: true
-      MARIADB_DATABASE: system3
-      MARIADB_USER: system3
-      MARIADB_PASSWORD: system3
-    volumes:
-      - mariadb_data:/var/lib/mysql
-    ports:
-      - "3306:3306"
-
-  core:
-    build:
-      context: ../../core
-      dockerfile: ../deploy/testing/Dockerfile.backend
-    command: bash -c 'python manage.py migrate && python /code/server.py'
-    environment:
-      - HTTP_HOST=core
-      - REDIS_HOST=redis
-      - DB_HOST=db
-      - DB_PORT=3306
-      - DB_NAME=system3
-      - DB_USER=system3
-      - DB_PASSWORD=system3
-    volumes:
-      - ../../core:/code
-    ports:
-      - "8000:8000"
-    depends_on:
-      - db
-      - redis
-
-  frontend:
-    build:
-      context: ../../web
-      dockerfile: ../deploy/testing/Dockerfile.frontend
-    command: npm run serve
-    volumes:
-      - ../../web:/web:ro
-      - /web/node_modules
-      - ./vue.config.js:/web/vue.config.js
-    ports:
-      - "8080:8080"
-    depends_on:
-      - core
-
-volumes:
-  mariadb_data:

deploy/testing/vue.config.js

@@ -1,27 +0,0 @@
-// vue.config.js
-
-module.exports = {
-    devServer: {
-        headers: {
-            "Access-Control-Allow-Origin": "*",
-            "Access-Control-Allow-Headers": "*",
-            "Access-Control-Allow-Methods": "*"
-        },
-        proxy: {
-            '^/media/2': {
-                target: 'http://core:8000/',
-            },
-            '^/api/2': {
-                target: 'http://core:8000/',
-            },
-            '^/api/1': {
-                target: 'http://core:8000/',
-            },
-            '^/ws/2': {
-                target: 'http://core:8000/',
-                ws: true,
-                logLevel: 'debug',
-            },
-        }
-    }
-}

web/src/views/Ticket.vue

@@ -125,10 +125,6 @@ export default {
     },
     mounted() {
 		this.scheduleAfterInit(() => [Promise.all([this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), this.fetchShippingVouchers()]).then(()=>{
-			if (this.ticket.state == "pending_new"){
-				this.selected_state = "pending_open";
-				this.changeTicketStatus(this.ticket)
-			};
 			this.selected_state = this.ticket.state;
 			this.selected_assignee = this.ticket.assigned_to
 		})]);