From 269f02c2cecfd97b4f0620bc95b1ce17abfdb820 Mon Sep 17 00:00:00 2001 From: lagertonne Date: Wed, 6 Nov 2024 22:21:32 +0100 Subject: [PATCH 1/9] Add django standard metrics --- core/core/settings.py | 5 +++++ core/core/urls.py | 1 + core/requirements.prod.txt | 2 ++ 3 files changed, 8 insertions(+) diff --git a/core/core/settings.py b/core/core/settings.py index db23180..124fcad 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -50,6 +50,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django_extensions', + 'django_prometheus', 'rest_framework', 'knox', 'drf_yasg', @@ -85,6 +86,7 @@ SWAGGER_SETTINGS = { } MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -92,6 +94,7 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware', ] ROOT_URLCONF = 'core.urls' @@ -210,4 +213,6 @@ CHANNEL_LAYERS = { } +PROMETHEUS_METRIC_NAMESPACE = 'c3lf' + TEST_RUNNER = 'core.test_runner.FastTestRunner' diff --git a/core/core/urls.py b/core/core/urls.py index b0161bb..df6e0d0 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -32,4 +32,5 @@ urlpatterns = [ path('api/2/', include('notify_sessions.api_v2')), path('api/2/', include('authentication.api_v2')), path('api/', get_info), + path('', include('django_prometheus.urls')), ] diff --git a/core/requirements.prod.txt b/core/requirements.prod.txt index 14bdc0f..ee69fe7 100644 --- a/core/requirements.prod.txt +++ b/core/requirements.prod.txt @@ -41,3 +41,5 @@ urllib3==2.1.0 uvicorn==0.24.0.post1 watchfiles==0.21.0 websockets==12.0 +django-prometheus==2.3.1 +prometheus_client==0.21.0 From 0c4995db2b1847e5cfe56fbcf00acf76fe4e028f Mon Sep 17 00:00:00 2001 From: jedi Date: Fri, 8 Nov 2024 20:04:43 +0100 Subject: [PATCH 2/9] add docker env for integration testing --- core/core/settings.py | 2 +- .../ansible/playbooks/templates/django.env.j2 | 1 + deploy/testing/Dockerfile.backend | 11 ++++ deploy/testing/Dockerfile.frontend | 6 ++ deploy/testing/docker-compose.yml | 55 +++++++++++++++++++ deploy/testing/vue.config.js | 27 +++++++++ 6 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 deploy/testing/Dockerfile.backend create mode 100644 deploy/testing/Dockerfile.frontend create mode 100644 deploy/testing/docker-compose.yml create mode 100644 deploy/testing/vue.config.js diff --git a/core/core/settings.py b/core/core/settings.py index 124fcad..5b37ad8 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -207,7 +207,7 @@ CHANNEL_LAYERS = { 'default': { 'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': { - 'hosts': [('localhost', 6379)], + 'hosts': [(os.getenv('REDIS_HOST', 'localhost'), 6379)], }, } diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index 72a0c30..a1757db 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -1,3 +1,4 @@ +REDIS_HOST=localhost DB_HOST=localhost DB_PORT=3306 DB_NAME=c3lf_sys3 diff --git a/deploy/testing/Dockerfile.backend b/deploy/testing/Dockerfile.backend new file mode 100644 index 0000000..c968994 --- /dev/null +++ b/deploy/testing/Dockerfile.backend @@ -0,0 +1,11 @@ +FROM python:3.11-bookworm +LABEL authors="lagertonne" + +ENV PYTHONUNBUFFERED 1 +RUN mkdir /code +WORKDIR /code +COPY requirements.prod.txt /code/ +RUN apt update && apt install -y mariadb-client +RUN pip install -r requirements.prod.txt +RUN pip install mysqlclient +COPY .. /code/ \ No newline at end of file diff --git a/deploy/testing/Dockerfile.frontend b/deploy/testing/Dockerfile.frontend new file mode 100644 index 0000000..0a41d1a --- /dev/null +++ b/deploy/testing/Dockerfile.frontend @@ -0,0 +1,6 @@ +FROM docker.io/node:22 + +RUN mkdir /web +WORKDIR /web +COPY package.json /web/ +RUN npm install diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml new file mode 100644 index 0000000..e93e901 --- /dev/null +++ b/deploy/testing/docker-compose.yml @@ -0,0 +1,55 @@ +services: + redis: + image: redis + ports: + - "6379:6379" + + db: + image: mariadb + environment: + MARIADB_RANDOM_ROOT_PASSWORD: true + MARIADB_DATABASE: system3 + MARIADB_USER: system3 + MARIADB_PASSWORD: system3 + volumes: + - mariadb_data:/var/lib/mysql + ports: + - "3306:3306" + + core: + build: + context: ../../core + dockerfile: ../deploy/testing/Dockerfile.backend + command: bash -c 'python manage.py migrate && python /code/server.py' + environment: + - HTTP_HOST=core + - REDIS_HOST=redis + - DB_HOST=db + - DB_PORT=3306 + - DB_NAME=system3 + - DB_USER=system3 + - DB_PASSWORD=system3 + volumes: + - ../../core:/code + ports: + - "8000:8000" + depends_on: + - db + - redis + + frontend: + build: + context: ../../web + dockerfile: ../deploy/testing/Dockerfile.frontend + command: npm run serve + volumes: + - ../../web:/web:ro + - /web/node_modules + - ./vue.config.js:/web/vue.config.js + ports: + - "8080:8080" + depends_on: + - core + +volumes: + mariadb_data: \ No newline at end of file diff --git a/deploy/testing/vue.config.js b/deploy/testing/vue.config.js new file mode 100644 index 0000000..f8f3c26 --- /dev/null +++ b/deploy/testing/vue.config.js @@ -0,0 +1,27 @@ +// vue.config.js + +module.exports = { + devServer: { + headers: { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Headers": "*", + "Access-Control-Allow-Methods": "*" + }, + proxy: { + '^/media/2': { + target: 'http://core:8000/', + }, + '^/api/2': { + target: 'http://core:8000/', + }, + '^/api/1': { + target: 'http://core:8000/', + }, + '^/ws/2': { + target: 'http://core:8000/', + ws: true, + logLevel: 'debug', + }, + } + } +} \ No newline at end of file From 4272aab6434b8bffd81d356bb387debcd33120f6 Mon Sep 17 00:00:00 2001 From: jedi Date: Fri, 8 Nov 2024 22:23:52 +0100 Subject: [PATCH 3/9] save raw_mails as file --- core/mail/migrations/0006_email_raw_file.py | 36 +++++++++++++++++++++ core/mail/models.py | 2 +- core/mail/protocol.py | 2 +- core/requirements.dev.txt | 2 ++ 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 core/mail/migrations/0006_email_raw_file.py diff --git a/core/mail/migrations/0006_email_raw_file.py b/core/mail/migrations/0006_email_raw_file.py new file mode 100644 index 0000000..1288bcf --- /dev/null +++ b/core/mail/migrations/0006_email_raw_file.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.7 on 2024-11-08 20:37 +from django.core.files.base import ContentFile +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mail', '0005_alter_eventaddress_event'), + ] + + def move_raw_mails_to_file(apps, schema_editor): + Email = apps.get_model('mail', 'Email') + for email in Email.objects.all(): + raw_content = email.raw + email.raw_file = ContentFile(raw_content) + email.raw = None + email.save() + + operations = [ + migrations.AddField( + model_name='email', + name='raw_file', + field=models.FileField(null=True, upload_to='raw_mail/'), + ), + migrations.RunPython(move_raw_mails_to_file), + migrations.RemoveField( + model_name='email', + name='raw', + ), + migrations.AlterField( + model_name='email', + name='raw_file', + field=models.FileField(upload_to='raw_mail/'), + ), + ] diff --git a/core/mail/models.py b/core/mail/models.py index 378854b..65371fe 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -18,7 +18,7 @@ class Email(SoftDeleteModel): recipient = models.CharField(max_length=255) reference = models.CharField(max_length=255, null=True, unique=True) in_reply_to = models.CharField(max_length=255, null=True) - raw = models.TextField() + raw_file = models.FileField(upload_to='raw_mail/') issue_thread = models.ForeignKey(IssueThread, models.SET_NULL, null=True, related_name='emails') event = models.ForeignKey(Event, models.SET_NULL, null=True) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index a87f1a6..926e8c2 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -206,7 +206,7 @@ def receive_email(envelope, log=None): email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content), event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) diff --git a/core/requirements.dev.txt b/core/requirements.dev.txt index 146aa37..3807a6c 100644 --- a/core/requirements.dev.txt +++ b/core/requirements.dev.txt @@ -73,3 +73,5 @@ watchfiles==0.21.0 websockets==12.0 yarl==1.9.4 zope.interface==6.1 +django-prometheus==2.3.1 +prometheus_client==0.21.0 From a6a8b0defe4d435a855a3ea91bc6a646ed2c7f1f Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 9 Nov 2024 00:03:21 +0100 Subject: [PATCH 4/9] add functions to train mails as spam/ham --- core/core/settings.py | 9 +++++++-- core/mail/models.py | 14 +++++++++++++- deploy/ansible/inventory.yml.sample | 4 +++- deploy/ansible/playbooks/templates/django.env.j2 | 3 +++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/core/settings.py b/core/core/settings.py index 5b37ad8..2d4a818 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -15,6 +15,9 @@ import sys import dotenv from pathlib import Path +def truthy_str(s): + return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', '👍'] + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -24,10 +27,10 @@ dotenv.load_dotenv(BASE_DIR / '.env') # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = truthy_str(os.getenv('DEBUG_MODE_ACTIVE', 'False')) ALLOWED_HOSTS = [os.getenv('HTTP_HOST', 'localhost')] @@ -40,6 +43,8 @@ LEGACY_USER_PASSWORD = os.getenv('LEGACY_API_PASSWORD', 'legacy_password') SYSTEM3_VERSION = "0.0.0-dev.0" +ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False')) + # Application definition INSTALLED_APPS = [ diff --git a/core/mail/models.py b/core/mail/models.py index 65371fe..2215fbb 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -3,7 +3,7 @@ import random from django.db import models from django_softdelete.models import SoftDeleteModel -from core.settings import MAIL_DOMAIN +from core.settings import MAIL_DOMAIN, ACTIVE_SPAM_TRAINING from files.models import AbstractFile from inventory.models import Event from tickets.models import IssueThread @@ -28,6 +28,18 @@ class Email(SoftDeleteModel): self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>' self.save() + def train_spam(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_spam", path]) + + def train_ham(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_ham", path]) + class EventAddress(models.Model): id = models.AutoField(primary_key=True) diff --git a/deploy/ansible/inventory.yml.sample b/deploy/ansible/inventory.yml.sample index 6ba14ac..2a50efd 100644 --- a/deploy/ansible/inventory.yml.sample +++ b/deploy/ansible/inventory.yml.sample @@ -11,4 +11,6 @@ c3lf-nodes: mail_domain: main_email: legacy_api_user: - legacy_api_password: \ No newline at end of file + legacy_api_password: + debug_mode_active: false + django_secret_key: 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index a1757db..c9b1c83 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -10,3 +10,6 @@ LEGACY_API_USER={{ legacy_api_user }} LEGACY_API_PASSWORD={{ legacy_api_password }} MEDIA_ROOT=/var/www/c3lf-sys3/userfiles STATIC_ROOT=/var/www/c3lf-sys3/staticfiles +ACTIVE_SPAM_TRAINING=True +DEBUG_MODE_ACTIVE={{ debug_mode_active }} +DJANGO_SECRET_KEY={{ django_secret_key }} From 5a6349c5d3d8bc9a47866901d9c0a0d5fce8910a Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 9 Nov 2024 01:00:53 +0100 Subject: [PATCH 5/9] train spam on state change to 'closed_spam' --- core/mail/migrations/0006_email_raw_file.py | 6 ++-- .../tickets/migrations/0011_train_old_spam.py | 31 +++++++++++++++++++ core/tickets/models.py | 2 ++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 core/tickets/migrations/0011_train_old_spam.py diff --git a/core/mail/migrations/0006_email_raw_file.py b/core/mail/migrations/0006_email_raw_file.py index 1288bcf..4086af8 100644 --- a/core/mail/migrations/0006_email_raw_file.py +++ b/core/mail/migrations/0006_email_raw_file.py @@ -4,7 +4,6 @@ from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ('mail', '0005_alter_eventaddress_event'), ] @@ -13,8 +12,9 @@ class Migration(migrations.Migration): Email = apps.get_model('mail', 'Email') for email in Email.objects.all(): raw_content = email.raw - email.raw_file = ContentFile(raw_content) - email.raw = None + path = "mail_{}".format(email.id) + if len(raw_content): + email.raw_file.save(path, ContentFile(raw_content)) email.save() operations = [ diff --git a/core/tickets/migrations/0011_train_old_spam.py b/core/tickets/migrations/0011_train_old_spam.py new file mode 100644 index 0000000..206cbb4 --- /dev/null +++ b/core/tickets/migrations/0011_train_old_spam.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.7 on 2024-06-23 02:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('mail', '0006_email_raw_file'), + ('tickets', '0010_issuethread_event_itemrelation_and_more'), + ] + + def train_old_mails(apps, schema_editor): + from tickets.models import IssueThread + for t in IssueThread.objects.all(): + try: + state = t.state + i = 0 + for e in t.emails.all(): + if e.raw_file: + if state == 'closed_spam' and i == 0: + e.train_spam() + else: + e.train_ham() + i += 1 + except: + pass + + operations = [ + migrations.RunPython(train_old_mails), + ] diff --git a/core/tickets/models.py b/core/tickets/models.py index db427fe..aff5d6c 100644 --- a/core/tickets/models.py +++ b/core/tickets/models.py @@ -60,6 +60,8 @@ class IssueThread(SoftDeleteModel): if self.state == value: return self.state_changes.create(state=value) + if value == 'closed_spam' and self.emails.exists(): + self.emails.first().train_spam() @property def assigned_to(self): From 8831f67f003526f460e2ebd4499ea8b89eb178a9 Mon Sep 17 00:00:00 2001 From: bton Date: Sun, 10 Nov 2024 17:12:53 +0100 Subject: [PATCH 6/9] ticket state changes to pending_open on first view --- web/src/views/Ticket.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index cbb372c..2042090 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -125,6 +125,9 @@ export default { }, mounted() { this.scheduleAfterInit(() => [Promise.all([this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), this.fetchShippingVouchers()]).then(()=>{ + if (this.ticket.state == "pending_new"){ + this.ticket.state = "pending_open" + }; this.selected_state = this.ticket.state; this.selected_assignee = this.ticket.assigned_to })]); From 444c2de16c522a1a0d9413cde186f8504c674c33 Mon Sep 17 00:00:00 2001 From: bton Date: Sun, 10 Nov 2024 19:53:09 +0100 Subject: [PATCH 7/9] change the ticket.state in the backend too --- web/src/views/Ticket.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index 2042090..f666ee8 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -126,7 +126,8 @@ export default { mounted() { this.scheduleAfterInit(() => [Promise.all([this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), this.fetchShippingVouchers()]).then(()=>{ if (this.ticket.state == "pending_new"){ - this.ticket.state = "pending_open" + this.selected_state = "pending_open"; + this.changeTicketStatus(this.ticket) }; this.selected_state = this.ticket.state; this.selected_assignee = this.ticket.assigned_to From 6e9b4019a2b6f51cbf6fdadb1ceb706366b7d555 Mon Sep 17 00:00:00 2001 From: lagertonne Date: Wed, 6 Nov 2024 22:26:48 +0100 Subject: [PATCH 8/9] cicd: Deploy testing automatically --- .forgejo/workflows/testing.yml | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .forgejo/workflows/testing.yml diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml new file mode 100644 index 0000000..2237cf9 --- /dev/null +++ b/.forgejo/workflows/testing.yml @@ -0,0 +1,41 @@ +on: + push: + branches: + - testing +jobs: + deploy: + runs-on: docker + steps: + - uses: actions/checkout@v4 + - name: Install ansible + run: | + apt update -y + apt install python3-pip -y + python3 -m pip install ansible + python3 -m pip install ansible-lint + + - name: Populate relevant files + run: | + mkdir ~/.ssh + echo "${{ secrets.C3LF_SSH_TESTING }}" > ~/.ssh/id_ed25519 + chmod 0600 ~/.ssh/id_ed25519 + ls -lah ~/.ssh + command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y ) + eval $(ssh-agent -s) + ssh-add ~/.ssh/id_ed25519 + echo "andromeda.lab.or.it ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDXPoO0PE+B9PYwbGaLo98zhbmjAkp6eBtVeZe43v/+T" >> ~/.ssh/known_hosts + mkdir /etc/ansible + echo "${{ secrets.C3LF_INVENTORY_TESTING }}" > /etc/ansible/hosts + + - name: Check ansible version + run: | + ansible --version + + - name: List ansible hosts + run: | + ansible -m ping Andromeda + + - name: Deploy testing + run: | + cd deploy/ansible + ansible-playbook playbooks/deploy-c3lf-sys3.yml From 9f65113c3ab51a25d49436491bd1a045a324bb0a Mon Sep 17 00:00:00 2001 From: lagertonne Date: Mon, 11 Nov 2024 20:49:17 +0100 Subject: [PATCH 9/9] cicd: Add testing --- .forgejo/workflows/testing.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 2237cf9..bee7aeb 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -2,7 +2,23 @@ on: push: branches: - testing + - feature/forgejo-actions jobs: + test: + runs-on: docker + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: 'pip' + - name: Install needed python modules + working-directory: core + run: cd core && pip3 install -r requirements.prod.txt + - name: Run django tests + working-directory: core + run: python3 -m manage.py test + deploy: runs-on: docker steps: