match incoming mail to event

.forgejo/issue_template/bug.yml

@@ -0,0 +1,35 @@
+name: Bug Report
+about: File a bug report
+labels:
+  - Kind/Bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+      placeholder: Tell us what you see!
+    validations:
+      required: true
+  - type: dropdown
+    id: browsers
+    attributes:
+      label: What browsers are you seeing the problem on?
+      multiple: true
+      options:
+        - Firefox (Windows)
+        - Firefox (MacOS)
+        - Firefox (Linux)
+        - Firefox (Android)
+        - Firefox (iOS)
+        - Chrome (Windows)
+        - Chrome (MacOS)
+        - Chrome (Linux)
+        - Chrome (Android)
+        - Chrome (iOS)
+        - Safari
+        - Microsoft Edge

.forgejo/issue_template/feature.yml

@@ -0,0 +1,27 @@
+name: 'New Feature'
+about: 'This template is for new features'
+labels:
+  - Kind/Feature
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Before creating a Feature Ticket, please check for duplicates.
+  - type: markdown
+    attributes:
+      value: |
+        ### Implementation Checklist
+        - [ ] concept
+        - [ ] frontend
+        - [ ] backend
+        - [ ] unittests
+        - [ ] tested on staging
+    visible: [ content ]
+  - type: textarea
+    id: description
+    attributes:
+      label: 'Feature Description'
+      description: 'Explain the the feature.'
+      placeholder: Description
+    validations:
+      required: true

.forgejo/workflows/pull_request.yml

@@ -0,0 +1,20 @@
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: docker
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache-dependency-path: '**/requirements.dev.txt'
+      - name: Install dependencies
+        working-directory: core
+        run: pip3 install -r requirements.dev.txt
+      - name: Run django tests
+        working-directory: core
+        run: python3 manage.py test

.forgejo/workflows/testing.yml

@@ -0,0 +1,60 @@
+on:
+  push:
+    branches:
+      - testing
+
+jobs:
+  test:
+    runs-on: docker
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache-dependency-path: '**/requirements.dev.txt'
+      - name: Install dependencies
+        working-directory: core
+        run: pip3 install -r requirements.dev.txt
+      - name: Run django tests
+        working-directory: core
+        run: python3 manage.py test
+
+  deploy:
+    needs: [test]
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install ansible
+        run: |
+          apt update -y
+          apt install python3-pip -y
+          python3 -m pip install ansible
+          python3 -m pip install ansible-lint
+
+      - name: Populate relevant files
+        run: |
+          mkdir ~/.ssh
+          echo "${{ secrets.C3LF_SSH_TESTING }}" > ~/.ssh/id_ed25519
+          chmod 0600 ~/.ssh/id_ed25519
+          ls -lah ~/.ssh
+          command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y )
+          eval $(ssh-agent -s)
+          ssh-add ~/.ssh/id_ed25519
+          echo "andromeda.lab.or.it ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDXPoO0PE+B9PYwbGaLo98zhbmjAkp6eBtVeZe43v/+T" >> ~/.ssh/known_hosts
+          mkdir /etc/ansible
+          echo "${{ secrets.C3LF_INVENTORY_TESTING }}" > /etc/ansible/hosts
+
+      - name: Check ansible version
+        run: |
+          ansible --version
+
+      - name: List ansible hosts
+        run: |
+          ansible -m ping Andromeda
+
+      - name: Deploy testing
+        run: |
+          cd deploy/ansible
+          ansible-playbook playbooks/deploy-c3lf-sys3.yml

core/core/settings.py

@@ -15,6 +15,9 @@ import sys
 import dotenv
 from pathlib import Path
 
+def truthy_str(s):
+    return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', '👍']
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -24,10 +27,10 @@ dotenv.load_dotenv(BASE_DIR / '.env')
 # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv'
+SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = truthy_str(os.getenv('DEBUG_MODE_ACTIVE', 'False'))
 
 ALLOWED_HOSTS = [os.getenv('HTTP_HOST', 'localhost')]
 
@@ -40,6 +43,8 @@ LEGACY_USER_PASSWORD = os.getenv('LEGACY_API_PASSWORD', 'legacy_password')
 
 SYSTEM3_VERSION = "0.0.0-dev.0"
 
+ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False'))
+
 # Application definition
 
 INSTALLED_APPS = [
@@ -50,6 +55,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django_extensions',
+    'django_prometheus',
     'rest_framework',
     'knox',
     'drf_yasg',
@@ -85,6 +91,7 @@ SWAGGER_SETTINGS = {
 }
 
 MIDDLEWARE = [
+    'django_prometheus.middleware.PrometheusBeforeMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
@@ -92,6 +99,7 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django_prometheus.middleware.PrometheusAfterMiddleware',
 ]
 
 ROOT_URLCONF = 'core.urls'
@@ -204,10 +212,12 @@ CHANNEL_LAYERS = {
     'default': {
         'BACKEND': 'channels_redis.core.RedisChannelLayer',
         'CONFIG': {
-            'hosts': [('localhost', 6379)],
+            'hosts': [(os.getenv('REDIS_HOST', 'localhost'), 6379)],
         },
     }
 
 }
 
+PROMETHEUS_METRIC_NAMESPACE = 'c3lf'
+
 TEST_RUNNER = 'core.test_runner.FastTestRunner'

core/core/urls.py

@@ -32,4 +32,5 @@ urlpatterns = [
     path('api/2/', include('notify_sessions.api_v2')),
     path('api/2/', include('authentication.api_v2')),
     path('api/', get_info),
+    path('', include('django_prometheus.urls')),
 ]

core/core/wsgi.py

@@ -1,16 +0,0 @@
-"""
-WSGI config for core project.
-
-It exposes the WSGI callable as a module-level variable named ``application``.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/
-"""
-
-import os
-
-from django.core.wsgi import get_wsgi_application
-
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
-
-application = get_wsgi_application()

core/mail/migrations/0006_email_raw_file.py

@@ -0,0 +1,36 @@
+# Generated by Django 4.2.7 on 2024-11-08 20:37
+from django.core.files.base import ContentFile
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('mail', '0005_alter_eventaddress_event'),
+    ]
+
+    def move_raw_mails_to_file(apps, schema_editor):
+        Email = apps.get_model('mail', 'Email')
+        for email in Email.objects.all():
+            raw_content = email.raw
+            path = "mail_{}".format(email.id)
+            if len(raw_content):
+                email.raw_file.save(path, ContentFile(raw_content))
+            email.save()
+
+    operations = [
+        migrations.AddField(
+            model_name='email',
+            name='raw_file',
+            field=models.FileField(null=True, upload_to='raw_mail/'),
+        ),
+        migrations.RunPython(move_raw_mails_to_file),
+        migrations.RemoveField(
+            model_name='email',
+            name='raw',
+        ),
+        migrations.AlterField(
+            model_name='email',
+            name='raw_file',
+            field=models.FileField(upload_to='raw_mail/'),
+        ),
+    ]

core/mail/models.py

@@ -3,7 +3,7 @@ import random
 from django.db import models
 from django_softdelete.models import SoftDeleteModel
 
-from core.settings import MAIL_DOMAIN
+from core.settings import MAIL_DOMAIN, ACTIVE_SPAM_TRAINING
 from files.models import AbstractFile
 from inventory.models import Event
 from tickets.models import IssueThread
@@ -18,7 +18,7 @@ class Email(SoftDeleteModel):
     recipient = models.CharField(max_length=255)
     reference = models.CharField(max_length=255, null=True, unique=True)
     in_reply_to = models.CharField(max_length=255, null=True)
-    raw = models.TextField()
+    raw_file = models.FileField(upload_to='raw_mail/')
     issue_thread = models.ForeignKey(IssueThread, models.SET_NULL, null=True, related_name='emails')
     event = models.ForeignKey(Event, models.SET_NULL, null=True)
 
@@ -28,6 +28,18 @@ class Email(SoftDeleteModel):
             self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>'
             self.save()
 
+    def train_spam(self):
+        if ACTIVE_SPAM_TRAINING and self.raw_file.path:
+            import subprocess
+            path = self.raw_file.path
+            subprocess.run(["rspamc", "learn_spam", path])
+
+    def train_ham(self):
+        if ACTIVE_SPAM_TRAINING and self.raw_file.path:
+            import subprocess
+            path = self.raw_file.path
+            subprocess.run(["rspamc", "learn_ham", path])
+
 
 class EventAddress(models.Model):
     id = models.AutoField(primary_key=True)

core/mail/protocol.py

@@ -91,7 +91,7 @@ async def send_smtp(message):
     await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False)
 
 
-def find_active_issue_thread(in_reply_to, address, subject):
+def find_active_issue_thread(in_reply_to, address, subject, event):
     from re import match
     uuid_match = match(r'^ticket\+([a-f0-9-]{36})@', address)
     if uuid_match:
@@ -102,7 +102,7 @@ def find_active_issue_thread(in_reply_to, address, subject):
     if reply_to.exists():
         return reply_to.first().issue_thread, False
     else:
-        issue = IssueThread.objects.create(name=subject)
+        issue = IssueThread.objects.create(name=subject, event=event)
         return issue, True
 
 
@@ -202,11 +202,14 @@ def receive_email(envelope, log=None):
     subject = unescape_and_decode_base64(subject)
     target_event = find_target_event(recipient)
 
-    active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject)
+    active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event)
+
+    from hashlib import sha256
+    random_filename = 'mail-' + sha256(envelope.content).hexdigest()
 
     email = Email.objects.create(
         sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id,
-        in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event,
+        in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event,
         issue_thread=active_issue_thread)
     for attachment in attachments:
         email.attachments.add(attachment)

core/mail/tests/v2/test_mails.py

@@ -142,6 +142,7 @@ class LMTPHandlerTestCase(TestCase):  # TODO replace with less hacky test
         aiosmtplib.send.assert_called_once()
         self.assertEqual('test ä', Email.objects.all()[0].subject)
         self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body)
+        self.assertTrue( Email.objects.all()[0].raw_file.path)
 
     def test_handle_quoted_printable_2(self):
         from aiosmtpd.smtp import Envelope
@@ -162,6 +163,7 @@ class LMTPHandlerTestCase(TestCase):  # TODO replace with less hacky test
         aiosmtplib.send.assert_called_once()
         self.assertEqual('suche_Mütze', Email.objects.all()[0].subject)
         self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body)
+        self.assertTrue( Email.objects.all()[0].raw_file.path)
 
     def test_handle_base64(self):
         from aiosmtpd.smtp import Envelope
@@ -182,6 +184,7 @@ class LMTPHandlerTestCase(TestCase):  # TODO replace with less hacky test
         aiosmtplib.send.assert_called_once()
         self.assertEqual('test', Email.objects.all()[0].subject)
         self.assertEqual('Text mit Base64-Kodierung: äöüß', Email.objects.all()[0].body)
+        self.assertTrue( Email.objects.all()[0].raw_file.path)
 
     def test_handle_client_reply(self):
         issue_thread = IssueThread.objects.create(
@@ -229,6 +232,7 @@ class LMTPHandlerTestCase(TestCase):  # TODO replace with less hacky test
         self.assertEqual(IssueThread.objects.all()[0].name, 'test')
         self.assertEqual(IssueThread.objects.all()[0].state, 'pending_new')
         self.assertEqual(IssueThread.objects.all()[0].assigned_to, None)
+        self.assertTrue( Email.objects.all()[2].raw_file.path)
 
     def test_handle_client_reply_2(self):
         issue_thread = IssueThread.objects.create(
@@ -281,6 +285,7 @@ class LMTPHandlerTestCase(TestCase):  # TODO replace with less hacky test
         self.assertEqual(IssueThread.objects.all()[0].name, 'test')
         self.assertEqual(IssueThread.objects.all()[0].state, 'pending_open')
         self.assertEqual(IssueThread.objects.all()[0].assigned_to, None)
+        self.assertTrue( Email.objects.all()[2].raw_file.path)
 
     def test_mail_reply(self):
         issue_thread = IssueThread.objects.create(
@@ -384,6 +389,7 @@ class LMTPHandlerTestCase(TestCase):  # TODO replace with less hacky test
         states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0])
         self.assertEqual(1, len(states))
         self.assertEqual('pending_new', states[0].state)
+        self.assertEqual(event, IssueThread.objects.all()[0].event)
 
     def test_mail_html_body(self):
         from aiosmtpd.smtp import Envelope

core/requirements.dev.txt

@@ -73,3 +73,5 @@ watchfiles==0.21.0
 websockets==12.0
 yarl==1.9.4
 zope.interface==6.1
+django-prometheus==2.3.1
+prometheus_client==0.21.0

core/requirements.prod.txt

@@ -41,3 +41,5 @@ urllib3==2.1.0
 uvicorn==0.24.0.post1
 watchfiles==0.21.0
 websockets==12.0
+django-prometheus==2.3.1
+prometheus_client==0.21.0

core/tickets/migrations/0011_train_old_spam.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.7 on 2024-06-23 02:17
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('mail', '0006_email_raw_file'),
+        ('tickets', '0010_issuethread_event_itemrelation_and_more'),
+    ]
+
+    def train_old_mails(apps, schema_editor):
+        from tickets.models import IssueThread
+        for t in IssueThread.objects.all():
+            try:
+                state = t.state
+                i = 0
+                for e in t.emails.all():
+                    if e.raw_file:
+                        if state == 'closed_spam' and i == 0:
+                            e.train_spam()
+                        else:
+                            e.train_ham()
+                    i += 1
+            except:
+                pass
+
+    operations = [
+        migrations.RunPython(train_old_mails),
+    ]

core/tickets/models.py

@@ -60,6 +60,8 @@ class IssueThread(SoftDeleteModel):
         if self.state == value:
             return
         self.state_changes.create(state=value)
+        if value == 'closed_spam' and self.emails.exists():
+            self.emails.first().train_spam()
 
     @property
     def assigned_to(self):

deploy/ansible/inventory.yml.sample

@@ -12,3 +12,5 @@ c3lf-nodes:
       main_email: <main_email>
       legacy_api_user: <legacy_api_user>
       legacy_api_password: <legacy_api_password>
+      debug_mode_active: false
+      django_secret_key: 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv'

deploy/ansible/playbooks/templates/django.env.j2

@@ -1,3 +1,4 @@
+REDIS_HOST=localhost
 DB_HOST=localhost
 DB_PORT=3306
 DB_NAME=c3lf_sys3
@@ -9,3 +10,6 @@ LEGACY_API_USER={{ legacy_api_user }}
 LEGACY_API_PASSWORD={{ legacy_api_password }}
 MEDIA_ROOT=/var/www/c3lf-sys3/userfiles
 STATIC_ROOT=/var/www/c3lf-sys3/staticfiles
+ACTIVE_SPAM_TRAINING=True
+DEBUG_MODE_ACTIVE={{ debug_mode_active }}
+DJANGO_SECRET_KEY={{ django_secret_key }}

deploy/ansible/playbooks/templates/nginx.conf.j2

@@ -70,6 +70,13 @@ server {
         alias /var/www/c3lf-sys3/staticfiles/;
     }
 
+    location /metrics {
+        allow 95.156.226.90;
+        allow 127.0.0.1;
+        allow ::1;
+        deny all;
+    }
+
     listen 443 ssl http2; # managed by Certbot
     ssl_certificate /etc/letsencrypt/live/{{ web_domain }}/fullchain.pem; # managed by Certbot
     ssl_certificate_key /etc/letsencrypt/live/{{ web_domain }}/privkey.pem; # managed by Certbot

deploy/testing/Dockerfile.backend

@@ -0,0 +1,11 @@
+FROM python:3.11-bookworm
+LABEL authors="lagertonne"
+
+ENV PYTHONUNBUFFERED 1
+RUN mkdir /code
+WORKDIR /code
+COPY requirements.prod.txt /code/
+RUN apt update && apt install -y mariadb-client
+RUN pip install -r requirements.prod.txt
+RUN pip install mysqlclient
+COPY .. /code/

deploy/testing/Dockerfile.frontend

@@ -0,0 +1,6 @@
+FROM docker.io/node:22
+
+RUN mkdir /web
+WORKDIR /web
+COPY package.json /web/
+RUN npm install

deploy/testing/docker-compose.yml

@@ -0,0 +1,55 @@
+services:
+  redis:
+    image: redis
+    ports:
+      - "6379:6379"
+
+  db:
+    image: mariadb
+    environment:
+      MARIADB_RANDOM_ROOT_PASSWORD: true
+      MARIADB_DATABASE: system3
+      MARIADB_USER: system3
+      MARIADB_PASSWORD: system3
+    volumes:
+      - mariadb_data:/var/lib/mysql
+    ports:
+      - "3306:3306"
+
+  core:
+    build:
+      context: ../../core
+      dockerfile: ../deploy/testing/Dockerfile.backend
+    command: bash -c 'python manage.py migrate && python /code/server.py'
+    environment:
+      - HTTP_HOST=core
+      - REDIS_HOST=redis
+      - DB_HOST=db
+      - DB_PORT=3306
+      - DB_NAME=system3
+      - DB_USER=system3
+      - DB_PASSWORD=system3
+    volumes:
+      - ../../core:/code
+    ports:
+      - "8000:8000"
+    depends_on:
+      - db
+      - redis
+
+  frontend:
+    build:
+      context: ../../web
+      dockerfile: ../deploy/testing/Dockerfile.frontend
+    command: npm run serve
+    volumes:
+      - ../../web:/web:ro
+      - /web/node_modules
+      - ./vue.config.js:/web/vue.config.js
+    ports:
+      - "8080:8080"
+    depends_on:
+      - core
+
+volumes:
+  mariadb_data:

deploy/testing/vue.config.js

@@ -0,0 +1,27 @@
+// vue.config.js
+
+module.exports = {
+    devServer: {
+        headers: {
+            "Access-Control-Allow-Origin": "*",
+            "Access-Control-Allow-Headers": "*",
+            "Access-Control-Allow-Methods": "*"
+        },
+        proxy: {
+            '^/media/2': {
+                target: 'http://core:8000/',
+            },
+            '^/api/2': {
+                target: 'http://core:8000/',
+            },
+            '^/api/1': {
+                target: 'http://core:8000/',
+            },
+            '^/ws/2': {
+                target: 'http://core:8000/',
+                ws: true,
+                logLevel: 'debug',
+            },
+        }
+    }
+}

web/src/views/Ticket.vue

@@ -125,6 +125,10 @@ export default {
     },
     mounted() {
 		this.scheduleAfterInit(() => [Promise.all([this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), this.fetchShippingVouchers()]).then(()=>{
+			if (this.ticket.state == "pending_new"){
+				this.selected_state = "pending_open";
+				this.changeTicketStatus(this.ticket)
+			};
 			this.selected_state = this.ticket.state;
 			this.selected_assignee = this.ticket.assigned_to
 		})]);