diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml new file mode 100644 index 0000000..bee7aeb --- /dev/null +++ b/.forgejo/workflows/testing.yml @@ -0,0 +1,57 @@ +on: + push: + branches: + - testing + - feature/forgejo-actions +jobs: + test: + runs-on: docker + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: 'pip' + - name: Install needed python modules + working-directory: core + run: cd core && pip3 install -r requirements.prod.txt + - name: Run django tests + working-directory: core + run: python3 -m manage.py test + + deploy: + runs-on: docker + steps: + - uses: actions/checkout@v4 + - name: Install ansible + run: | + apt update -y + apt install python3-pip -y + python3 -m pip install ansible + python3 -m pip install ansible-lint + + - name: Populate relevant files + run: | + mkdir ~/.ssh + echo "${{ secrets.C3LF_SSH_TESTING }}" > ~/.ssh/id_ed25519 + chmod 0600 ~/.ssh/id_ed25519 + ls -lah ~/.ssh + command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y ) + eval $(ssh-agent -s) + ssh-add ~/.ssh/id_ed25519 + echo "andromeda.lab.or.it ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDXPoO0PE+B9PYwbGaLo98zhbmjAkp6eBtVeZe43v/+T" >> ~/.ssh/known_hosts + mkdir /etc/ansible + echo "${{ secrets.C3LF_INVENTORY_TESTING }}" > /etc/ansible/hosts + + - name: Check ansible version + run: | + ansible --version + + - name: List ansible hosts + run: | + ansible -m ping Andromeda + + - name: Deploy testing + run: | + cd deploy/ansible + ansible-playbook playbooks/deploy-c3lf-sys3.yml diff --git a/core/core/settings.py b/core/core/settings.py index db23180..2d4a818 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -15,6 +15,9 @@ import sys import dotenv from pathlib import Path +def truthy_str(s): + return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', '👍'] + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -24,10 +27,10 @@ dotenv.load_dotenv(BASE_DIR / '.env') # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = truthy_str(os.getenv('DEBUG_MODE_ACTIVE', 'False')) ALLOWED_HOSTS = [os.getenv('HTTP_HOST', 'localhost')] @@ -40,6 +43,8 @@ LEGACY_USER_PASSWORD = os.getenv('LEGACY_API_PASSWORD', 'legacy_password') SYSTEM3_VERSION = "0.0.0-dev.0" +ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False')) + # Application definition INSTALLED_APPS = [ @@ -50,6 +55,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django_extensions', + 'django_prometheus', 'rest_framework', 'knox', 'drf_yasg', @@ -85,6 +91,7 @@ SWAGGER_SETTINGS = { } MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -92,6 +99,7 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware', ] ROOT_URLCONF = 'core.urls' @@ -204,10 +212,12 @@ CHANNEL_LAYERS = { 'default': { 'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': { - 'hosts': [('localhost', 6379)], + 'hosts': [(os.getenv('REDIS_HOST', 'localhost'), 6379)], }, } } +PROMETHEUS_METRIC_NAMESPACE = 'c3lf' + TEST_RUNNER = 'core.test_runner.FastTestRunner' diff --git a/core/core/urls.py b/core/core/urls.py index b0161bb..df6e0d0 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -32,4 +32,5 @@ urlpatterns = [ path('api/2/', include('notify_sessions.api_v2')), path('api/2/', include('authentication.api_v2')), path('api/', get_info), + path('', include('django_prometheus.urls')), ] diff --git a/core/mail/migrations/0006_email_raw_file.py b/core/mail/migrations/0006_email_raw_file.py new file mode 100644 index 0000000..4086af8 --- /dev/null +++ b/core/mail/migrations/0006_email_raw_file.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.7 on 2024-11-08 20:37 +from django.core.files.base import ContentFile +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('mail', '0005_alter_eventaddress_event'), + ] + + def move_raw_mails_to_file(apps, schema_editor): + Email = apps.get_model('mail', 'Email') + for email in Email.objects.all(): + raw_content = email.raw + path = "mail_{}".format(email.id) + if len(raw_content): + email.raw_file.save(path, ContentFile(raw_content)) + email.save() + + operations = [ + migrations.AddField( + model_name='email', + name='raw_file', + field=models.FileField(null=True, upload_to='raw_mail/'), + ), + migrations.RunPython(move_raw_mails_to_file), + migrations.RemoveField( + model_name='email', + name='raw', + ), + migrations.AlterField( + model_name='email', + name='raw_file', + field=models.FileField(upload_to='raw_mail/'), + ), + ] diff --git a/core/mail/models.py b/core/mail/models.py index 378854b..2215fbb 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -3,7 +3,7 @@ import random from django.db import models from django_softdelete.models import SoftDeleteModel -from core.settings import MAIL_DOMAIN +from core.settings import MAIL_DOMAIN, ACTIVE_SPAM_TRAINING from files.models import AbstractFile from inventory.models import Event from tickets.models import IssueThread @@ -18,7 +18,7 @@ class Email(SoftDeleteModel): recipient = models.CharField(max_length=255) reference = models.CharField(max_length=255, null=True, unique=True) in_reply_to = models.CharField(max_length=255, null=True) - raw = models.TextField() + raw_file = models.FileField(upload_to='raw_mail/') issue_thread = models.ForeignKey(IssueThread, models.SET_NULL, null=True, related_name='emails') event = models.ForeignKey(Event, models.SET_NULL, null=True) @@ -28,6 +28,18 @@ class Email(SoftDeleteModel): self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>' self.save() + def train_spam(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_spam", path]) + + def train_ham(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_ham", path]) + class EventAddress(models.Model): id = models.AutoField(primary_key=True) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index a87f1a6..926e8c2 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -206,7 +206,7 @@ def receive_email(envelope, log=None): email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content), event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) diff --git a/core/requirements.dev.txt b/core/requirements.dev.txt index 146aa37..3807a6c 100644 --- a/core/requirements.dev.txt +++ b/core/requirements.dev.txt @@ -73,3 +73,5 @@ watchfiles==0.21.0 websockets==12.0 yarl==1.9.4 zope.interface==6.1 +django-prometheus==2.3.1 +prometheus_client==0.21.0 diff --git a/core/requirements.prod.txt b/core/requirements.prod.txt index 14bdc0f..ee69fe7 100644 --- a/core/requirements.prod.txt +++ b/core/requirements.prod.txt @@ -41,3 +41,5 @@ urllib3==2.1.0 uvicorn==0.24.0.post1 watchfiles==0.21.0 websockets==12.0 +django-prometheus==2.3.1 +prometheus_client==0.21.0 diff --git a/core/tickets/migrations/0011_train_old_spam.py b/core/tickets/migrations/0011_train_old_spam.py new file mode 100644 index 0000000..206cbb4 --- /dev/null +++ b/core/tickets/migrations/0011_train_old_spam.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.7 on 2024-06-23 02:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('mail', '0006_email_raw_file'), + ('tickets', '0010_issuethread_event_itemrelation_and_more'), + ] + + def train_old_mails(apps, schema_editor): + from tickets.models import IssueThread + for t in IssueThread.objects.all(): + try: + state = t.state + i = 0 + for e in t.emails.all(): + if e.raw_file: + if state == 'closed_spam' and i == 0: + e.train_spam() + else: + e.train_ham() + i += 1 + except: + pass + + operations = [ + migrations.RunPython(train_old_mails), + ] diff --git a/core/tickets/models.py b/core/tickets/models.py index db427fe..aff5d6c 100644 --- a/core/tickets/models.py +++ b/core/tickets/models.py @@ -60,6 +60,8 @@ class IssueThread(SoftDeleteModel): if self.state == value: return self.state_changes.create(state=value) + if value == 'closed_spam' and self.emails.exists(): + self.emails.first().train_spam() @property def assigned_to(self): diff --git a/deploy/ansible/inventory.yml.sample b/deploy/ansible/inventory.yml.sample index 6ba14ac..2a50efd 100644 --- a/deploy/ansible/inventory.yml.sample +++ b/deploy/ansible/inventory.yml.sample @@ -11,4 +11,6 @@ c3lf-nodes: mail_domain: main_email: legacy_api_user: - legacy_api_password: \ No newline at end of file + legacy_api_password: + debug_mode_active: false + django_secret_key: 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index 72a0c30..c9b1c83 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -1,3 +1,4 @@ +REDIS_HOST=localhost DB_HOST=localhost DB_PORT=3306 DB_NAME=c3lf_sys3 @@ -9,3 +10,6 @@ LEGACY_API_USER={{ legacy_api_user }} LEGACY_API_PASSWORD={{ legacy_api_password }} MEDIA_ROOT=/var/www/c3lf-sys3/userfiles STATIC_ROOT=/var/www/c3lf-sys3/staticfiles +ACTIVE_SPAM_TRAINING=True +DEBUG_MODE_ACTIVE={{ debug_mode_active }} +DJANGO_SECRET_KEY={{ django_secret_key }} diff --git a/deploy/testing/Dockerfile.backend b/deploy/testing/Dockerfile.backend new file mode 100644 index 0000000..c968994 --- /dev/null +++ b/deploy/testing/Dockerfile.backend @@ -0,0 +1,11 @@ +FROM python:3.11-bookworm +LABEL authors="lagertonne" + +ENV PYTHONUNBUFFERED 1 +RUN mkdir /code +WORKDIR /code +COPY requirements.prod.txt /code/ +RUN apt update && apt install -y mariadb-client +RUN pip install -r requirements.prod.txt +RUN pip install mysqlclient +COPY .. /code/ \ No newline at end of file diff --git a/deploy/testing/Dockerfile.frontend b/deploy/testing/Dockerfile.frontend new file mode 100644 index 0000000..0a41d1a --- /dev/null +++ b/deploy/testing/Dockerfile.frontend @@ -0,0 +1,6 @@ +FROM docker.io/node:22 + +RUN mkdir /web +WORKDIR /web +COPY package.json /web/ +RUN npm install diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml new file mode 100644 index 0000000..e93e901 --- /dev/null +++ b/deploy/testing/docker-compose.yml @@ -0,0 +1,55 @@ +services: + redis: + image: redis + ports: + - "6379:6379" + + db: + image: mariadb + environment: + MARIADB_RANDOM_ROOT_PASSWORD: true + MARIADB_DATABASE: system3 + MARIADB_USER: system3 + MARIADB_PASSWORD: system3 + volumes: + - mariadb_data:/var/lib/mysql + ports: + - "3306:3306" + + core: + build: + context: ../../core + dockerfile: ../deploy/testing/Dockerfile.backend + command: bash -c 'python manage.py migrate && python /code/server.py' + environment: + - HTTP_HOST=core + - REDIS_HOST=redis + - DB_HOST=db + - DB_PORT=3306 + - DB_NAME=system3 + - DB_USER=system3 + - DB_PASSWORD=system3 + volumes: + - ../../core:/code + ports: + - "8000:8000" + depends_on: + - db + - redis + + frontend: + build: + context: ../../web + dockerfile: ../deploy/testing/Dockerfile.frontend + command: npm run serve + volumes: + - ../../web:/web:ro + - /web/node_modules + - ./vue.config.js:/web/vue.config.js + ports: + - "8080:8080" + depends_on: + - core + +volumes: + mariadb_data: \ No newline at end of file diff --git a/deploy/testing/vue.config.js b/deploy/testing/vue.config.js new file mode 100644 index 0000000..f8f3c26 --- /dev/null +++ b/deploy/testing/vue.config.js @@ -0,0 +1,27 @@ +// vue.config.js + +module.exports = { + devServer: { + headers: { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Headers": "*", + "Access-Control-Allow-Methods": "*" + }, + proxy: { + '^/media/2': { + target: 'http://core:8000/', + }, + '^/api/2': { + target: 'http://core:8000/', + }, + '^/api/1': { + target: 'http://core:8000/', + }, + '^/ws/2': { + target: 'http://core:8000/', + ws: true, + logLevel: 'debug', + }, + } + } +} \ No newline at end of file diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index cbb372c..f666ee8 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -125,6 +125,10 @@ export default { }, mounted() { this.scheduleAfterInit(() => [Promise.all([this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), this.fetchShippingVouchers()]).then(()=>{ + if (this.ticket.state == "pending_new"){ + this.selected_state = "pending_open"; + this.changeTicketStatus(this.ticket) + }; this.selected_state = this.ticket.state; this.selected_assignee = this.ticket.assigned_to })]);