From 4272aab6434b8bffd81d356bb387debcd33120f6 Mon Sep 17 00:00:00 2001 From: jedi Date: Fri, 8 Nov 2024 22:23:52 +0100 Subject: [PATCH 1/3] save raw_mails as file --- core/mail/migrations/0006_email_raw_file.py | 36 +++++++++++++++++++++ core/mail/models.py | 2 +- core/mail/protocol.py | 2 +- core/requirements.dev.txt | 2 ++ 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 core/mail/migrations/0006_email_raw_file.py diff --git a/core/mail/migrations/0006_email_raw_file.py b/core/mail/migrations/0006_email_raw_file.py new file mode 100644 index 0000000..1288bcf --- /dev/null +++ b/core/mail/migrations/0006_email_raw_file.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.7 on 2024-11-08 20:37 +from django.core.files.base import ContentFile +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mail', '0005_alter_eventaddress_event'), + ] + + def move_raw_mails_to_file(apps, schema_editor): + Email = apps.get_model('mail', 'Email') + for email in Email.objects.all(): + raw_content = email.raw + email.raw_file = ContentFile(raw_content) + email.raw = None + email.save() + + operations = [ + migrations.AddField( + model_name='email', + name='raw_file', + field=models.FileField(null=True, upload_to='raw_mail/'), + ), + migrations.RunPython(move_raw_mails_to_file), + migrations.RemoveField( + model_name='email', + name='raw', + ), + migrations.AlterField( + model_name='email', + name='raw_file', + field=models.FileField(upload_to='raw_mail/'), + ), + ] diff --git a/core/mail/models.py b/core/mail/models.py index 378854b..65371fe 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -18,7 +18,7 @@ class Email(SoftDeleteModel): recipient = models.CharField(max_length=255) reference = models.CharField(max_length=255, null=True, unique=True) in_reply_to = models.CharField(max_length=255, null=True) - raw = models.TextField() + raw_file = models.FileField(upload_to='raw_mail/') issue_thread = models.ForeignKey(IssueThread, models.SET_NULL, null=True, related_name='emails') event = models.ForeignKey(Event, models.SET_NULL, null=True) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index a87f1a6..926e8c2 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -206,7 +206,7 @@ def receive_email(envelope, log=None): email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content), event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) diff --git a/core/requirements.dev.txt b/core/requirements.dev.txt index 146aa37..3807a6c 100644 --- a/core/requirements.dev.txt +++ b/core/requirements.dev.txt @@ -73,3 +73,5 @@ watchfiles==0.21.0 websockets==12.0 yarl==1.9.4 zope.interface==6.1 +django-prometheus==2.3.1 +prometheus_client==0.21.0 From a6a8b0defe4d435a855a3ea91bc6a646ed2c7f1f Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 9 Nov 2024 00:03:21 +0100 Subject: [PATCH 2/3] add functions to train mails as spam/ham --- core/core/settings.py | 9 +++++++-- core/mail/models.py | 14 +++++++++++++- deploy/ansible/inventory.yml.sample | 4 +++- deploy/ansible/playbooks/templates/django.env.j2 | 3 +++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/core/settings.py b/core/core/settings.py index 5b37ad8..2d4a818 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -15,6 +15,9 @@ import sys import dotenv from pathlib import Path +def truthy_str(s): + return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', '👍'] + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -24,10 +27,10 @@ dotenv.load_dotenv(BASE_DIR / '.env') # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = truthy_str(os.getenv('DEBUG_MODE_ACTIVE', 'False')) ALLOWED_HOSTS = [os.getenv('HTTP_HOST', 'localhost')] @@ -40,6 +43,8 @@ LEGACY_USER_PASSWORD = os.getenv('LEGACY_API_PASSWORD', 'legacy_password') SYSTEM3_VERSION = "0.0.0-dev.0" +ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False')) + # Application definition INSTALLED_APPS = [ diff --git a/core/mail/models.py b/core/mail/models.py index 65371fe..2215fbb 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -3,7 +3,7 @@ import random from django.db import models from django_softdelete.models import SoftDeleteModel -from core.settings import MAIL_DOMAIN +from core.settings import MAIL_DOMAIN, ACTIVE_SPAM_TRAINING from files.models import AbstractFile from inventory.models import Event from tickets.models import IssueThread @@ -28,6 +28,18 @@ class Email(SoftDeleteModel): self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>' self.save() + def train_spam(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_spam", path]) + + def train_ham(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_ham", path]) + class EventAddress(models.Model): id = models.AutoField(primary_key=True) diff --git a/deploy/ansible/inventory.yml.sample b/deploy/ansible/inventory.yml.sample index 6ba14ac..2a50efd 100644 --- a/deploy/ansible/inventory.yml.sample +++ b/deploy/ansible/inventory.yml.sample @@ -11,4 +11,6 @@ c3lf-nodes: mail_domain: main_email: legacy_api_user: - legacy_api_password: \ No newline at end of file + legacy_api_password: + debug_mode_active: false + django_secret_key: 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index a1757db..c9b1c83 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -10,3 +10,6 @@ LEGACY_API_USER={{ legacy_api_user }} LEGACY_API_PASSWORD={{ legacy_api_password }} MEDIA_ROOT=/var/www/c3lf-sys3/userfiles STATIC_ROOT=/var/www/c3lf-sys3/staticfiles +ACTIVE_SPAM_TRAINING=True +DEBUG_MODE_ACTIVE={{ debug_mode_active }} +DJANGO_SECRET_KEY={{ django_secret_key }} From 5a6349c5d3d8bc9a47866901d9c0a0d5fce8910a Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 9 Nov 2024 01:00:53 +0100 Subject: [PATCH 3/3] train spam on state change to 'closed_spam' --- core/mail/migrations/0006_email_raw_file.py | 6 ++-- .../tickets/migrations/0011_train_old_spam.py | 31 +++++++++++++++++++ core/tickets/models.py | 2 ++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 core/tickets/migrations/0011_train_old_spam.py diff --git a/core/mail/migrations/0006_email_raw_file.py b/core/mail/migrations/0006_email_raw_file.py index 1288bcf..4086af8 100644 --- a/core/mail/migrations/0006_email_raw_file.py +++ b/core/mail/migrations/0006_email_raw_file.py @@ -4,7 +4,6 @@ from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ('mail', '0005_alter_eventaddress_event'), ] @@ -13,8 +12,9 @@ class Migration(migrations.Migration): Email = apps.get_model('mail', 'Email') for email in Email.objects.all(): raw_content = email.raw - email.raw_file = ContentFile(raw_content) - email.raw = None + path = "mail_{}".format(email.id) + if len(raw_content): + email.raw_file.save(path, ContentFile(raw_content)) email.save() operations = [ diff --git a/core/tickets/migrations/0011_train_old_spam.py b/core/tickets/migrations/0011_train_old_spam.py new file mode 100644 index 0000000..206cbb4 --- /dev/null +++ b/core/tickets/migrations/0011_train_old_spam.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.7 on 2024-06-23 02:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('mail', '0006_email_raw_file'), + ('tickets', '0010_issuethread_event_itemrelation_and_more'), + ] + + def train_old_mails(apps, schema_editor): + from tickets.models import IssueThread + for t in IssueThread.objects.all(): + try: + state = t.state + i = 0 + for e in t.emails.all(): + if e.raw_file: + if state == 'closed_spam' and i == 0: + e.train_spam() + else: + e.train_ham() + i += 1 + except: + pass + + operations = [ + migrations.RunPython(train_old_mails), + ] diff --git a/core/tickets/models.py b/core/tickets/models.py index db427fe..aff5d6c 100644 --- a/core/tickets/models.py +++ b/core/tickets/models.py @@ -60,6 +60,8 @@ class IssueThread(SoftDeleteModel): if self.state == value: return self.state_changes.create(state=value) + if value == 'closed_spam' and self.emails.exists(): + self.emails.first().train_spam() @property def assigned_to(self):