diff --git a/core/core/settings.py b/core/core/settings.py index c568fcc..ef7461c 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -15,6 +15,9 @@ import sys import dotenv from pathlib import Path +def truthy_str(s): + return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', '👍'] + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -24,10 +27,10 @@ dotenv.load_dotenv(BASE_DIR / '.env') # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = truthy_str(os.getenv('DEBUG_MODE_ACTIVE', 'False')) PRIMARY_HOST = os.getenv('HTTP_HOST', 'localhost') @@ -42,6 +45,8 @@ LEGACY_USER_PASSWORD = os.getenv('LEGACY_API_PASSWORD', 'legacy_password') SYSTEM3_VERSION = "0.0.0-dev.0" +ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False')) + TELEGRAM_BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN', '1234567890:ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghi') TELEGRAM_GROUP_CHAT_ID = os.getenv('TELEGRAM_GROUP_CHAT_ID', '-1234567890') @@ -56,6 +61,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django_extensions', + 'django_prometheus', 'rest_framework', 'knox', 'drf_yasg', @@ -92,6 +98,7 @@ SWAGGER_SETTINGS = { } MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -99,6 +106,7 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware', ] ROOT_URLCONF = 'core.urls' @@ -218,4 +226,6 @@ CHANNEL_LAYERS = { } +PROMETHEUS_METRIC_NAMESPACE = 'c3lf' + TEST_RUNNER = 'core.test_runner.FastTestRunner' diff --git a/core/core/urls.py b/core/core/urls.py index ba3e975..228ef89 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -33,4 +33,5 @@ urlpatterns = [ path('api/2/', include('authentication.api_v2')), path('api/2/', include('notifications.api_v2')), path('api/', get_info), + path('', include('django_prometheus.urls')), ] diff --git a/core/mail/migrations/0006_email_raw_file.py b/core/mail/migrations/0006_email_raw_file.py new file mode 100644 index 0000000..4086af8 --- /dev/null +++ b/core/mail/migrations/0006_email_raw_file.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.7 on 2024-11-08 20:37 +from django.core.files.base import ContentFile +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('mail', '0005_alter_eventaddress_event'), + ] + + def move_raw_mails_to_file(apps, schema_editor): + Email = apps.get_model('mail', 'Email') + for email in Email.objects.all(): + raw_content = email.raw + path = "mail_{}".format(email.id) + if len(raw_content): + email.raw_file.save(path, ContentFile(raw_content)) + email.save() + + operations = [ + migrations.AddField( + model_name='email', + name='raw_file', + field=models.FileField(null=True, upload_to='raw_mail/'), + ), + migrations.RunPython(move_raw_mails_to_file), + migrations.RemoveField( + model_name='email', + name='raw', + ), + migrations.AlterField( + model_name='email', + name='raw_file', + field=models.FileField(upload_to='raw_mail/'), + ), + ] diff --git a/core/mail/models.py b/core/mail/models.py index 2788e9d..8c95e6b 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -3,6 +3,7 @@ import random from django.db import models from django_softdelete.models import SoftDeleteModel +from core.settings import MAIL_DOMAIN, ACTIVE_SPAM_TRAINING from authentication.models import ExtendedUser from core.settings import MAIL_DOMAIN from files.models import AbstractFile @@ -29,6 +30,18 @@ class Email(SoftDeleteModel): self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>' self.save() + def train_spam(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_spam", path]) + + def train_ham(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_ham", path]) + class EventAddress(models.Model): id = models.AutoField(primary_key=True) diff --git a/core/requirements.dev.txt b/core/requirements.dev.txt index 146aa37..3807a6c 100644 --- a/core/requirements.dev.txt +++ b/core/requirements.dev.txt @@ -73,3 +73,5 @@ watchfiles==0.21.0 websockets==12.0 yarl==1.9.4 zope.interface==6.1 +django-prometheus==2.3.1 +prometheus_client==0.21.0 diff --git a/core/requirements.prod.txt b/core/requirements.prod.txt index 14bdc0f..ee69fe7 100644 --- a/core/requirements.prod.txt +++ b/core/requirements.prod.txt @@ -41,3 +41,5 @@ urllib3==2.1.0 uvicorn==0.24.0.post1 watchfiles==0.21.0 websockets==12.0 +django-prometheus==2.3.1 +prometheus_client==0.21.0 diff --git a/core/tickets/migrations/0011_train_old_spam.py b/core/tickets/migrations/0011_train_old_spam.py new file mode 100644 index 0000000..206cbb4 --- /dev/null +++ b/core/tickets/migrations/0011_train_old_spam.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.7 on 2024-06-23 02:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('mail', '0006_email_raw_file'), + ('tickets', '0010_issuethread_event_itemrelation_and_more'), + ] + + def train_old_mails(apps, schema_editor): + from tickets.models import IssueThread + for t in IssueThread.objects.all(): + try: + state = t.state + i = 0 + for e in t.emails.all(): + if e.raw_file: + if state == 'closed_spam' and i == 0: + e.train_spam() + else: + e.train_ham() + i += 1 + except: + pass + + operations = [ + migrations.RunPython(train_old_mails), + ] diff --git a/core/tickets/models.py b/core/tickets/models.py index 8a559ee..be7c82c 100644 --- a/core/tickets/models.py +++ b/core/tickets/models.py @@ -60,6 +60,8 @@ class IssueThread(SoftDeleteModel): if self.state == value: return self.state_changes.create(state=value) + if value == 'closed_spam' and self.emails.exists(): + self.emails.first().train_spam() @property def assigned_to(self): diff --git a/deploy/ansible/inventory.yml.sample b/deploy/ansible/inventory.yml.sample index 6ba14ac..2a50efd 100644 --- a/deploy/ansible/inventory.yml.sample +++ b/deploy/ansible/inventory.yml.sample @@ -11,4 +11,6 @@ c3lf-nodes: mail_domain: main_email: legacy_api_user: - legacy_api_password: \ No newline at end of file + legacy_api_password: + debug_mode_active: false + django_secret_key: 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index 4526d10..748ecf4 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -10,5 +10,8 @@ LEGACY_API_USER={{ legacy_api_user }} LEGACY_API_PASSWORD={{ legacy_api_password }} MEDIA_ROOT=/var/www/c3lf-sys3/userfiles STATIC_ROOT=/var/www/c3lf-sys3/staticfiles +ACTIVE_SPAM_TRAINING=True +DEBUG_MODE_ACTIVE={{ debug_mode_active }} +DJANGO_SECRET_KEY={{ django_secret_key }} TELEGRAM_GROUP_CHAT_ID={{ telegram_group_chat_id }} TELEGRAM_BOT_TOKEN={{ telegram_bot_token }} \ No newline at end of file