save raw_mails as file

core/core/settings.py

@@ -15,6 +15,9 @@ import sys
 import dotenv
 from pathlib import Path
 
+def truthy_str(s):
+    return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', '👍']
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -24,10 +27,10 @@ dotenv.load_dotenv(BASE_DIR / '.env')
 # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv'
+SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv')
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = truthy_str(os.getenv('DEBUG_MODE_ACTIVE', 'False'))
 
 PRIMARY_HOST = os.getenv('HTTP_HOST', 'localhost')
 
@@ -42,6 +45,8 @@ LEGACY_USER_PASSWORD = os.getenv('LEGACY_API_PASSWORD', 'legacy_password')
 
 SYSTEM3_VERSION = "0.0.0-dev.0"
 
+ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False'))
+
 TELEGRAM_BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN', '1234567890:ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghi')
 
 TELEGRAM_GROUP_CHAT_ID = os.getenv('TELEGRAM_GROUP_CHAT_ID', '-1234567890')
@@ -56,6 +61,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django_extensions',
+    'django_prometheus',
     'rest_framework',
     'knox',
     'drf_yasg',
@@ -92,6 +98,7 @@ SWAGGER_SETTINGS = {
 }
 
 MIDDLEWARE = [
+    'django_prometheus.middleware.PrometheusBeforeMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
@@ -99,6 +106,7 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django_prometheus.middleware.PrometheusAfterMiddleware',
 ]
 
 ROOT_URLCONF = 'core.urls'
@@ -218,4 +226,6 @@ CHANNEL_LAYERS = {
 
 }
 
+PROMETHEUS_METRIC_NAMESPACE = 'c3lf'
+
 TEST_RUNNER = 'core.test_runner.FastTestRunner'

core/core/urls.py

@@ -33,4 +33,5 @@ urlpatterns = [
     path('api/2/', include('authentication.api_v2')),
     path('api/2/', include('notifications.api_v2')),
     path('api/', get_info),
+    path('', include('django_prometheus.urls')),
 ]

core/mail/migrations/0006_email_raw_file.py

@@ -0,0 +1,36 @@
+# Generated by Django 4.2.7 on 2024-11-08 20:37
+from django.core.files.base import ContentFile
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('mail', '0005_alter_eventaddress_event'),
+    ]
+
+    def move_raw_mails_to_file(apps, schema_editor):
+        Email = apps.get_model('mail', 'Email')
+        for email in Email.objects.all():
+            raw_content = email.raw
+            path = "mail_{}".format(email.id)
+            if len(raw_content):
+                email.raw_file.save(path, ContentFile(raw_content))
+            email.save()
+
+    operations = [
+        migrations.AddField(
+            model_name='email',
+            name='raw_file',
+            field=models.FileField(null=True, upload_to='raw_mail/'),
+        ),
+        migrations.RunPython(move_raw_mails_to_file),
+        migrations.RemoveField(
+            model_name='email',
+            name='raw',
+        ),
+        migrations.AlterField(
+            model_name='email',
+            name='raw_file',
+            field=models.FileField(upload_to='raw_mail/'),
+        ),
+    ]

core/mail/models.py

@@ -3,6 +3,7 @@ import random
 from django.db import models
 from django_softdelete.models import SoftDeleteModel
 
+from core.settings import MAIL_DOMAIN, ACTIVE_SPAM_TRAINING
 from authentication.models import ExtendedUser
 from core.settings import MAIL_DOMAIN
 from files.models import AbstractFile
@@ -29,6 +30,18 @@ class Email(SoftDeleteModel):
             self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>'
             self.save()
 
+    def train_spam(self):
+        if ACTIVE_SPAM_TRAINING:
+            import subprocess
+            path = self.raw_file.path
+            subprocess.run(["rspamc", "learn_spam", path])
+
+    def train_ham(self):
+        if ACTIVE_SPAM_TRAINING:
+            import subprocess
+            path = self.raw_file.path
+            subprocess.run(["rspamc", "learn_ham", path])
+
 
 class EventAddress(models.Model):
     id = models.AutoField(primary_key=True)

core/requirements.dev.txt

@@ -73,3 +73,5 @@ watchfiles==0.21.0
 websockets==12.0
 yarl==1.9.4
 zope.interface==6.1
+django-prometheus==2.3.1
+prometheus_client==0.21.0

core/requirements.prod.txt

@@ -41,3 +41,5 @@ urllib3==2.1.0
 uvicorn==0.24.0.post1
 watchfiles==0.21.0
 websockets==12.0
+django-prometheus==2.3.1
+prometheus_client==0.21.0

core/tickets/migrations/0011_train_old_spam.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.7 on 2024-06-23 02:17
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('mail', '0006_email_raw_file'),
+        ('tickets', '0010_issuethread_event_itemrelation_and_more'),
+    ]
+
+    def train_old_mails(apps, schema_editor):
+        from tickets.models import IssueThread
+        for t in IssueThread.objects.all():
+            try:
+                state = t.state
+                i = 0
+                for e in t.emails.all():
+                    if e.raw_file:
+                        if state == 'closed_spam' and i == 0:
+                            e.train_spam()
+                        else:
+                            e.train_ham()
+                    i += 1
+            except:
+                pass
+
+    operations = [
+        migrations.RunPython(train_old_mails),
+    ]

core/tickets/models.py

@@ -60,6 +60,8 @@ class IssueThread(SoftDeleteModel):
         if self.state == value:
             return
         self.state_changes.create(state=value)
+        if value == 'closed_spam' and self.emails.exists():
+            self.emails.first().train_spam()
 
     @property
     def assigned_to(self):

deploy/ansible/inventory.yml.sample

@@ -11,4 +11,6 @@ c3lf-nodes:
       mail_domain: <mail_domain>
       main_email: <main_email>
       legacy_api_user: <legacy_api_user>
-      legacy_api_password: <legacy_api_password>
+      legacy_api_password: <legacy_api_password>
+      debug_mode_active: false
+      django_secret_key: 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv'

deploy/ansible/playbooks/templates/django.env.j2

@@ -10,5 +10,8 @@ LEGACY_API_USER={{ legacy_api_user }}
 LEGACY_API_PASSWORD={{ legacy_api_password }}
 MEDIA_ROOT=/var/www/c3lf-sys3/userfiles
 STATIC_ROOT=/var/www/c3lf-sys3/staticfiles
+ACTIVE_SPAM_TRAINING=True
+DEBUG_MODE_ACTIVE={{ debug_mode_active }}
+DJANGO_SECRET_KEY={{ django_secret_key }}
 TELEGRAM_GROUP_CHAT_ID={{ telegram_group_chat_id }}
 TELEGRAM_BOT_TOKEN={{ telegram_bot_token }}