From 0f8462dc7c11ddb0ce2de90dc8d7b3938bbf097a Mon Sep 17 00:00:00 2001 From: jedi Date: Wed, 6 Nov 2024 21:13:44 +0100 Subject: [PATCH 01/22] enforce startup order in docker-compose.yml --- deploy/dev/docker-compose.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/deploy/dev/docker-compose.yml b/deploy/dev/docker-compose.yml index 95e8083..dff5ab3 100644 --- a/deploy/dev/docker-compose.yml +++ b/deploy/dev/docker-compose.yml @@ -6,11 +6,17 @@ services: command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000' environment: - HTTP_HOST=core - #- DATABASE_URL + - DB_HOST=db + - DB_PORT=3306 + - DB_NAME=system3 + - DB_USER=system3 + - DB_PASSWORD=system3 volumes: - ../../core:/code ports: - "8000:8000" + depends_on: + - db frontend: build: @@ -23,6 +29,8 @@ services: - ./vue.config.js:/web/vue.config.js ports: - "8080:8080" + depends_on: + - core db: image: mariadb @@ -30,4 +38,11 @@ services: MARIADB_RANDOM_ROOT_PASSWORD: true MARIADB_DATABASE: system3 MARIADB_USER: system3 - MARIADB_PASSWORD: system3 \ No newline at end of file + MARIADB_PASSWORD: system3 + volumes: + - mariadb_data:/var/lib/mysql + ports: + - "3306:3306" + +volumes: + mariadb_data: \ No newline at end of file From b9cfdf54565066fcd90ea5b3527daa2966dcab71 Mon Sep 17 00:00:00 2001 From: bton Date: Wed, 6 Nov 2024 21:09:06 +0100 Subject: [PATCH 02/22] fixed race contidion on ticket view loading --- web/src/views/Ticket.vue | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index 63f3515..cbb372c 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -79,16 +79,6 @@ export default { shipping_voucher_type: null, } }, - watch: { - ticket(val) { - if (this.selected_state == null) { - this.selected_state = val.state; - } - if (this.selected_assignee == null) { - this.selected_assignee = val.assigned_to - } - } - }, computed: { ...mapState(['tickets', 'state_options', 'users']), ...mapGetters(['availableShippingVoucherTypes']), @@ -134,12 +124,14 @@ export default { }, }, mounted() { - this.scheduleAfterInit(() => [this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), - this.fetchShippingVouchers()]); + this.scheduleAfterInit(() => [Promise.all([this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), this.fetchShippingVouchers()]).then(()=>{ + this.selected_state = this.ticket.state; + this.selected_assignee = this.ticket.assigned_to + })]); } }; \ No newline at end of file + From 269f02c2cecfd97b4f0620bc95b1ce17abfdb820 Mon Sep 17 00:00:00 2001 From: lagertonne Date: Wed, 6 Nov 2024 22:21:32 +0100 Subject: [PATCH 03/22] Add django standard metrics --- core/core/settings.py | 5 +++++ core/core/urls.py | 1 + core/requirements.prod.txt | 2 ++ 3 files changed, 8 insertions(+) diff --git a/core/core/settings.py b/core/core/settings.py index db23180..124fcad 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -50,6 +50,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django_extensions', + 'django_prometheus', 'rest_framework', 'knox', 'drf_yasg', @@ -85,6 +86,7 @@ SWAGGER_SETTINGS = { } MIDDLEWARE = [ + 'django_prometheus.middleware.PrometheusBeforeMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -92,6 +94,7 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django_prometheus.middleware.PrometheusAfterMiddleware', ] ROOT_URLCONF = 'core.urls' @@ -210,4 +213,6 @@ CHANNEL_LAYERS = { } +PROMETHEUS_METRIC_NAMESPACE = 'c3lf' + TEST_RUNNER = 'core.test_runner.FastTestRunner' diff --git a/core/core/urls.py b/core/core/urls.py index b0161bb..df6e0d0 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -32,4 +32,5 @@ urlpatterns = [ path('api/2/', include('notify_sessions.api_v2')), path('api/2/', include('authentication.api_v2')), path('api/', get_info), + path('', include('django_prometheus.urls')), ] diff --git a/core/requirements.prod.txt b/core/requirements.prod.txt index 14bdc0f..ee69fe7 100644 --- a/core/requirements.prod.txt +++ b/core/requirements.prod.txt @@ -41,3 +41,5 @@ urllib3==2.1.0 uvicorn==0.24.0.post1 watchfiles==0.21.0 websockets==12.0 +django-prometheus==2.3.1 +prometheus_client==0.21.0 From 0c4995db2b1847e5cfe56fbcf00acf76fe4e028f Mon Sep 17 00:00:00 2001 From: jedi Date: Fri, 8 Nov 2024 20:04:43 +0100 Subject: [PATCH 04/22] add docker env for integration testing --- core/core/settings.py | 2 +- .../ansible/playbooks/templates/django.env.j2 | 1 + deploy/testing/Dockerfile.backend | 11 ++++ deploy/testing/Dockerfile.frontend | 6 ++ deploy/testing/docker-compose.yml | 55 +++++++++++++++++++ deploy/testing/vue.config.js | 27 +++++++++ 6 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 deploy/testing/Dockerfile.backend create mode 100644 deploy/testing/Dockerfile.frontend create mode 100644 deploy/testing/docker-compose.yml create mode 100644 deploy/testing/vue.config.js diff --git a/core/core/settings.py b/core/core/settings.py index 124fcad..5b37ad8 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -207,7 +207,7 @@ CHANNEL_LAYERS = { 'default': { 'BACKEND': 'channels_redis.core.RedisChannelLayer', 'CONFIG': { - 'hosts': [('localhost', 6379)], + 'hosts': [(os.getenv('REDIS_HOST', 'localhost'), 6379)], }, } diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index 72a0c30..a1757db 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -1,3 +1,4 @@ +REDIS_HOST=localhost DB_HOST=localhost DB_PORT=3306 DB_NAME=c3lf_sys3 diff --git a/deploy/testing/Dockerfile.backend b/deploy/testing/Dockerfile.backend new file mode 100644 index 0000000..c968994 --- /dev/null +++ b/deploy/testing/Dockerfile.backend @@ -0,0 +1,11 @@ +FROM python:3.11-bookworm +LABEL authors="lagertonne" + +ENV PYTHONUNBUFFERED 1 +RUN mkdir /code +WORKDIR /code +COPY requirements.prod.txt /code/ +RUN apt update && apt install -y mariadb-client +RUN pip install -r requirements.prod.txt +RUN pip install mysqlclient +COPY .. /code/ \ No newline at end of file diff --git a/deploy/testing/Dockerfile.frontend b/deploy/testing/Dockerfile.frontend new file mode 100644 index 0000000..0a41d1a --- /dev/null +++ b/deploy/testing/Dockerfile.frontend @@ -0,0 +1,6 @@ +FROM docker.io/node:22 + +RUN mkdir /web +WORKDIR /web +COPY package.json /web/ +RUN npm install diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml new file mode 100644 index 0000000..e93e901 --- /dev/null +++ b/deploy/testing/docker-compose.yml @@ -0,0 +1,55 @@ +services: + redis: + image: redis + ports: + - "6379:6379" + + db: + image: mariadb + environment: + MARIADB_RANDOM_ROOT_PASSWORD: true + MARIADB_DATABASE: system3 + MARIADB_USER: system3 + MARIADB_PASSWORD: system3 + volumes: + - mariadb_data:/var/lib/mysql + ports: + - "3306:3306" + + core: + build: + context: ../../core + dockerfile: ../deploy/testing/Dockerfile.backend + command: bash -c 'python manage.py migrate && python /code/server.py' + environment: + - HTTP_HOST=core + - REDIS_HOST=redis + - DB_HOST=db + - DB_PORT=3306 + - DB_NAME=system3 + - DB_USER=system3 + - DB_PASSWORD=system3 + volumes: + - ../../core:/code + ports: + - "8000:8000" + depends_on: + - db + - redis + + frontend: + build: + context: ../../web + dockerfile: ../deploy/testing/Dockerfile.frontend + command: npm run serve + volumes: + - ../../web:/web:ro + - /web/node_modules + - ./vue.config.js:/web/vue.config.js + ports: + - "8080:8080" + depends_on: + - core + +volumes: + mariadb_data: \ No newline at end of file diff --git a/deploy/testing/vue.config.js b/deploy/testing/vue.config.js new file mode 100644 index 0000000..f8f3c26 --- /dev/null +++ b/deploy/testing/vue.config.js @@ -0,0 +1,27 @@ +// vue.config.js + +module.exports = { + devServer: { + headers: { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Headers": "*", + "Access-Control-Allow-Methods": "*" + }, + proxy: { + '^/media/2': { + target: 'http://core:8000/', + }, + '^/api/2': { + target: 'http://core:8000/', + }, + '^/api/1': { + target: 'http://core:8000/', + }, + '^/ws/2': { + target: 'http://core:8000/', + ws: true, + logLevel: 'debug', + }, + } + } +} \ No newline at end of file From 4272aab6434b8bffd81d356bb387debcd33120f6 Mon Sep 17 00:00:00 2001 From: jedi Date: Fri, 8 Nov 2024 22:23:52 +0100 Subject: [PATCH 05/22] save raw_mails as file --- core/mail/migrations/0006_email_raw_file.py | 36 +++++++++++++++++++++ core/mail/models.py | 2 +- core/mail/protocol.py | 2 +- core/requirements.dev.txt | 2 ++ 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 core/mail/migrations/0006_email_raw_file.py diff --git a/core/mail/migrations/0006_email_raw_file.py b/core/mail/migrations/0006_email_raw_file.py new file mode 100644 index 0000000..1288bcf --- /dev/null +++ b/core/mail/migrations/0006_email_raw_file.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.7 on 2024-11-08 20:37 +from django.core.files.base import ContentFile +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('mail', '0005_alter_eventaddress_event'), + ] + + def move_raw_mails_to_file(apps, schema_editor): + Email = apps.get_model('mail', 'Email') + for email in Email.objects.all(): + raw_content = email.raw + email.raw_file = ContentFile(raw_content) + email.raw = None + email.save() + + operations = [ + migrations.AddField( + model_name='email', + name='raw_file', + field=models.FileField(null=True, upload_to='raw_mail/'), + ), + migrations.RunPython(move_raw_mails_to_file), + migrations.RemoveField( + model_name='email', + name='raw', + ), + migrations.AlterField( + model_name='email', + name='raw_file', + field=models.FileField(upload_to='raw_mail/'), + ), + ] diff --git a/core/mail/models.py b/core/mail/models.py index 378854b..65371fe 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -18,7 +18,7 @@ class Email(SoftDeleteModel): recipient = models.CharField(max_length=255) reference = models.CharField(max_length=255, null=True, unique=True) in_reply_to = models.CharField(max_length=255, null=True) - raw = models.TextField() + raw_file = models.FileField(upload_to='raw_mail/') issue_thread = models.ForeignKey(IssueThread, models.SET_NULL, null=True, related_name='emails') event = models.ForeignKey(Event, models.SET_NULL, null=True) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index a87f1a6..926e8c2 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -206,7 +206,7 @@ def receive_email(envelope, log=None): email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw=envelope.content, event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content), event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) diff --git a/core/requirements.dev.txt b/core/requirements.dev.txt index 146aa37..3807a6c 100644 --- a/core/requirements.dev.txt +++ b/core/requirements.dev.txt @@ -73,3 +73,5 @@ watchfiles==0.21.0 websockets==12.0 yarl==1.9.4 zope.interface==6.1 +django-prometheus==2.3.1 +prometheus_client==0.21.0 From a6a8b0defe4d435a855a3ea91bc6a646ed2c7f1f Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 9 Nov 2024 00:03:21 +0100 Subject: [PATCH 06/22] add functions to train mails as spam/ham --- core/core/settings.py | 9 +++++++-- core/mail/models.py | 14 +++++++++++++- deploy/ansible/inventory.yml.sample | 4 +++- deploy/ansible/playbooks/templates/django.env.j2 | 3 +++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/core/settings.py b/core/core/settings.py index 5b37ad8..2d4a818 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -15,6 +15,9 @@ import sys import dotenv from pathlib import Path +def truthy_str(s): + return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', 'πŸ‘'] + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -24,10 +27,10 @@ dotenv.load_dotenv(BASE_DIR / '.env') # See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv') # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = truthy_str(os.getenv('DEBUG_MODE_ACTIVE', 'False')) ALLOWED_HOSTS = [os.getenv('HTTP_HOST', 'localhost')] @@ -40,6 +43,8 @@ LEGACY_USER_PASSWORD = os.getenv('LEGACY_API_PASSWORD', 'legacy_password') SYSTEM3_VERSION = "0.0.0-dev.0" +ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False')) + # Application definition INSTALLED_APPS = [ diff --git a/core/mail/models.py b/core/mail/models.py index 65371fe..2215fbb 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -3,7 +3,7 @@ import random from django.db import models from django_softdelete.models import SoftDeleteModel -from core.settings import MAIL_DOMAIN +from core.settings import MAIL_DOMAIN, ACTIVE_SPAM_TRAINING from files.models import AbstractFile from inventory.models import Event from tickets.models import IssueThread @@ -28,6 +28,18 @@ class Email(SoftDeleteModel): self.reference = f'<{random.randint(0, 1000000000):09}@{MAIL_DOMAIN}>' self.save() + def train_spam(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_spam", path]) + + def train_ham(self): + if ACTIVE_SPAM_TRAINING: + import subprocess + path = self.raw_file.path + subprocess.run(["rspamc", "learn_ham", path]) + class EventAddress(models.Model): id = models.AutoField(primary_key=True) diff --git a/deploy/ansible/inventory.yml.sample b/deploy/ansible/inventory.yml.sample index 6ba14ac..2a50efd 100644 --- a/deploy/ansible/inventory.yml.sample +++ b/deploy/ansible/inventory.yml.sample @@ -11,4 +11,6 @@ c3lf-nodes: mail_domain: main_email: legacy_api_user: - legacy_api_password: \ No newline at end of file + legacy_api_password: + debug_mode_active: false + django_secret_key: 'django-insecure-tm*$w_14iqbiy-!7(8#ba7j+_@(7@rf2&a^!=shs&$03b%2*rv' \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index a1757db..c9b1c83 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -10,3 +10,6 @@ LEGACY_API_USER={{ legacy_api_user }} LEGACY_API_PASSWORD={{ legacy_api_password }} MEDIA_ROOT=/var/www/c3lf-sys3/userfiles STATIC_ROOT=/var/www/c3lf-sys3/staticfiles +ACTIVE_SPAM_TRAINING=True +DEBUG_MODE_ACTIVE={{ debug_mode_active }} +DJANGO_SECRET_KEY={{ django_secret_key }} From 5a6349c5d3d8bc9a47866901d9c0a0d5fce8910a Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 9 Nov 2024 01:00:53 +0100 Subject: [PATCH 07/22] train spam on state change to 'closed_spam' --- core/mail/migrations/0006_email_raw_file.py | 6 ++-- .../tickets/migrations/0011_train_old_spam.py | 31 +++++++++++++++++++ core/tickets/models.py | 2 ++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 core/tickets/migrations/0011_train_old_spam.py diff --git a/core/mail/migrations/0006_email_raw_file.py b/core/mail/migrations/0006_email_raw_file.py index 1288bcf..4086af8 100644 --- a/core/mail/migrations/0006_email_raw_file.py +++ b/core/mail/migrations/0006_email_raw_file.py @@ -4,7 +4,6 @@ from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ('mail', '0005_alter_eventaddress_event'), ] @@ -13,8 +12,9 @@ class Migration(migrations.Migration): Email = apps.get_model('mail', 'Email') for email in Email.objects.all(): raw_content = email.raw - email.raw_file = ContentFile(raw_content) - email.raw = None + path = "mail_{}".format(email.id) + if len(raw_content): + email.raw_file.save(path, ContentFile(raw_content)) email.save() operations = [ diff --git a/core/tickets/migrations/0011_train_old_spam.py b/core/tickets/migrations/0011_train_old_spam.py new file mode 100644 index 0000000..206cbb4 --- /dev/null +++ b/core/tickets/migrations/0011_train_old_spam.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.7 on 2024-06-23 02:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ('mail', '0006_email_raw_file'), + ('tickets', '0010_issuethread_event_itemrelation_and_more'), + ] + + def train_old_mails(apps, schema_editor): + from tickets.models import IssueThread + for t in IssueThread.objects.all(): + try: + state = t.state + i = 0 + for e in t.emails.all(): + if e.raw_file: + if state == 'closed_spam' and i == 0: + e.train_spam() + else: + e.train_ham() + i += 1 + except: + pass + + operations = [ + migrations.RunPython(train_old_mails), + ] diff --git a/core/tickets/models.py b/core/tickets/models.py index db427fe..aff5d6c 100644 --- a/core/tickets/models.py +++ b/core/tickets/models.py @@ -60,6 +60,8 @@ class IssueThread(SoftDeleteModel): if self.state == value: return self.state_changes.create(state=value) + if value == 'closed_spam' and self.emails.exists(): + self.emails.first().train_spam() @property def assigned_to(self): From 8831f67f003526f460e2ebd4499ea8b89eb178a9 Mon Sep 17 00:00:00 2001 From: bton Date: Sun, 10 Nov 2024 17:12:53 +0100 Subject: [PATCH 08/22] ticket state changes to pending_open on first view --- web/src/views/Ticket.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index cbb372c..2042090 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -125,6 +125,9 @@ export default { }, mounted() { this.scheduleAfterInit(() => [Promise.all([this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), this.fetchShippingVouchers()]).then(()=>{ + if (this.ticket.state == "pending_new"){ + this.ticket.state = "pending_open" + }; this.selected_state = this.ticket.state; this.selected_assignee = this.ticket.assigned_to })]); From 444c2de16c522a1a0d9413cde186f8504c674c33 Mon Sep 17 00:00:00 2001 From: bton Date: Sun, 10 Nov 2024 19:53:09 +0100 Subject: [PATCH 09/22] change the ticket.state in the backend too --- web/src/views/Ticket.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index 2042090..f666ee8 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -126,7 +126,8 @@ export default { mounted() { this.scheduleAfterInit(() => [Promise.all([this.fetchTicketStates(), this.loadTickets(), this.loadUsers(), this.fetchShippingVouchers()]).then(()=>{ if (this.ticket.state == "pending_new"){ - this.ticket.state = "pending_open" + this.selected_state = "pending_open"; + this.changeTicketStatus(this.ticket) }; this.selected_state = this.ticket.state; this.selected_assignee = this.ticket.assigned_to From be02a3e163ce9de01834a83b79b63dfb054e7236 Mon Sep 17 00:00:00 2001 From: lagertonne Date: Wed, 6 Nov 2024 22:26:48 +0100 Subject: [PATCH 10/22] cicd: Deploy testing automatically --- .forgejo/workflows/testing.yml | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .forgejo/workflows/testing.yml diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml new file mode 100644 index 0000000..2237cf9 --- /dev/null +++ b/.forgejo/workflows/testing.yml @@ -0,0 +1,41 @@ +on: + push: + branches: + - testing +jobs: + deploy: + runs-on: docker + steps: + - uses: actions/checkout@v4 + - name: Install ansible + run: | + apt update -y + apt install python3-pip -y + python3 -m pip install ansible + python3 -m pip install ansible-lint + + - name: Populate relevant files + run: | + mkdir ~/.ssh + echo "${{ secrets.C3LF_SSH_TESTING }}" > ~/.ssh/id_ed25519 + chmod 0600 ~/.ssh/id_ed25519 + ls -lah ~/.ssh + command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y ) + eval $(ssh-agent -s) + ssh-add ~/.ssh/id_ed25519 + echo "andromeda.lab.or.it ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDXPoO0PE+B9PYwbGaLo98zhbmjAkp6eBtVeZe43v/+T" >> ~/.ssh/known_hosts + mkdir /etc/ansible + echo "${{ secrets.C3LF_INVENTORY_TESTING }}" > /etc/ansible/hosts + + - name: Check ansible version + run: | + ansible --version + + - name: List ansible hosts + run: | + ansible -m ping Andromeda + + - name: Deploy testing + run: | + cd deploy/ansible + ansible-playbook playbooks/deploy-c3lf-sys3.yml From 5ba4085e603d4d5ccec82a2163f7eadfdf813db1 Mon Sep 17 00:00:00 2001 From: lagertonne Date: Wed, 6 Nov 2024 22:26:48 +0100 Subject: [PATCH 11/22] cicd: Run tests automatically --- .forgejo/workflows/testing.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 2237cf9..dd16880 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -3,7 +3,25 @@ on: branches: - testing jobs: + test: + runs-on: docker + container: + image: ghcr.io/catthehacker/ubuntu:act-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache-dependency-path: '**/requirements.dev.txt' + - name: Install dependencies + working-directory: core + run: pip3 install -r requirements.dev.txt + - name: Run django tests + working-directory: core + run: python3 manage.py test + deploy: + needs: [test] runs-on: docker steps: - uses: actions/checkout@v4 From 63d6b7a5a8fd7aa447ca36bca74d80b09c79d0dd Mon Sep 17 00:00:00 2001 From: lagertonne Date: Tue, 12 Nov 2024 16:51:29 +0100 Subject: [PATCH 12/22] cicd: run on every pull request, but only deploy on testing --- .forgejo/workflows/pull_request.yml | 20 ++++++++++++++++++++ .forgejo/workflows/testing.yml | 1 + 2 files changed, 21 insertions(+) create mode 100644 .forgejo/workflows/pull_request.yml diff --git a/.forgejo/workflows/pull_request.yml b/.forgejo/workflows/pull_request.yml new file mode 100644 index 0000000..1171616 --- /dev/null +++ b/.forgejo/workflows/pull_request.yml @@ -0,0 +1,20 @@ +on: + pull_request: + +jobs: + test: + runs-on: docker + container: + image: ghcr.io/catthehacker/ubuntu:act-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache-dependency-path: '**/requirements.dev.txt' + - name: Install dependencies + working-directory: core + run: pip3 install -r requirements.dev.txt + - name: Run django tests + working-directory: core + run: python3 manage.py test diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index dd16880..3b44d24 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -2,6 +2,7 @@ on: push: branches: - testing + jobs: test: runs-on: docker From 120507512d2ad9abea62170dbe6735a90018af68 Mon Sep 17 00:00:00 2001 From: lagertonne Date: Tue, 12 Nov 2024 17:28:12 +0100 Subject: [PATCH 13/22] deploy: Simple protection for metrics endpoint --- deploy/ansible/playbooks/templates/nginx.conf.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deploy/ansible/playbooks/templates/nginx.conf.j2 b/deploy/ansible/playbooks/templates/nginx.conf.j2 index 608ffd5..3533f37 100644 --- a/deploy/ansible/playbooks/templates/nginx.conf.j2 +++ b/deploy/ansible/playbooks/templates/nginx.conf.j2 @@ -70,6 +70,13 @@ server { alias /var/www/c3lf-sys3/staticfiles/; } + location /metrics { + allow 95.156.226.90; + allow 127.0.0.1; + allow ::1; + deny all; + } + listen 443 ssl http2; # managed by Certbot ssl_certificate /etc/letsencrypt/live/{{ web_domain }}/fullchain.pem; # managed by Certbot ssl_certificate_key /etc/letsencrypt/live/{{ web_domain }}/privkey.pem; # managed by Certbot From 2c609427ec94b48a6336f53651693a432d68cd46 Mon Sep 17 00:00:00 2001 From: jedi Date: Wed, 13 Nov 2024 23:12:17 +0100 Subject: [PATCH 14/22] add simple issue templates --- .forgejo/issue_template/bug.yml | 35 +++++++++++++++++++++++++++++ .forgejo/issue_template/feature.yml | 27 ++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 .forgejo/issue_template/bug.yml create mode 100644 .forgejo/issue_template/feature.yml diff --git a/.forgejo/issue_template/bug.yml b/.forgejo/issue_template/bug.yml new file mode 100644 index 0000000..8b227a0 --- /dev/null +++ b/.forgejo/issue_template/bug.yml @@ -0,0 +1,35 @@ +name: Bug Report +about: File a bug report +labels: + - Kind/Bug +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + validations: + required: true + - type: dropdown + id: browsers + attributes: + label: What browsers are you seeing the problem on? + multiple: true + options: + - Firefox (Windows) + - Firefox (MacOS) + - Firefox (Linux) + - Firefox (Android) + - Firefox (iOS) + - Chrome (Windows) + - Chrome (MacOS) + - Chrome (Linux) + - Chrome (Android) + - Chrome (iOS) + - Safari + - Microsoft Edge \ No newline at end of file diff --git a/.forgejo/issue_template/feature.yml b/.forgejo/issue_template/feature.yml new file mode 100644 index 0000000..c8cf794 --- /dev/null +++ b/.forgejo/issue_template/feature.yml @@ -0,0 +1,27 @@ +name: 'New Feature' +about: 'This template is for new features' +labels: + - Kind/Feature +body: + - type: markdown + attributes: + value: | + Before creating a Feature Ticket, please check for duplicates. + - type: markdown + attributes: + value: | + ### Implementation Checklist + - [ ] concept + - [ ] frontend + - [ ] backend + - [ ] unittests + - [ ] tested on staging + visible: [ content ] + - type: textarea + id: description + attributes: + label: 'Feature Description' + description: 'Explain the the feature.' + placeholder: Description + validations: + required: true \ No newline at end of file From d73bebd5de97bf4a3483398e04c5b4fb86182611 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 18 Nov 2024 02:01:23 +0100 Subject: [PATCH 15/22] match incoming mail to event --- core/core/wsgi.py | 16 ---------------- core/mail/models.py | 4 ++-- core/mail/protocol.py | 11 +++++++---- core/mail/tests/v2/test_mails.py | 6 ++++++ 4 files changed, 15 insertions(+), 22 deletions(-) delete mode 100644 core/core/wsgi.py diff --git a/core/core/wsgi.py b/core/core/wsgi.py deleted file mode 100644 index f44964d..0000000 --- a/core/core/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for core project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings') - -application = get_wsgi_application() diff --git a/core/mail/models.py b/core/mail/models.py index 2215fbb..36cd2b3 100644 --- a/core/mail/models.py +++ b/core/mail/models.py @@ -29,13 +29,13 @@ class Email(SoftDeleteModel): self.save() def train_spam(self): - if ACTIVE_SPAM_TRAINING: + if ACTIVE_SPAM_TRAINING and self.raw_file.path: import subprocess path = self.raw_file.path subprocess.run(["rspamc", "learn_spam", path]) def train_ham(self): - if ACTIVE_SPAM_TRAINING: + if ACTIVE_SPAM_TRAINING and self.raw_file.path: import subprocess path = self.raw_file.path subprocess.run(["rspamc", "learn_ham", path]) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 926e8c2..c5aaa4a 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -91,7 +91,7 @@ async def send_smtp(message): await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False) -def find_active_issue_thread(in_reply_to, address, subject): +def find_active_issue_thread(in_reply_to, address, subject, event): from re import match uuid_match = match(r'^ticket\+([a-f0-9-]{36})@', address) if uuid_match: @@ -102,7 +102,7 @@ def find_active_issue_thread(in_reply_to, address, subject): if reply_to.exists(): return reply_to.first().issue_thread, False else: - issue = IssueThread.objects.create(name=subject) + issue = IssueThread.objects.create(name=subject, event=event) return issue, True @@ -202,11 +202,14 @@ def receive_email(envelope, log=None): subject = unescape_and_decode_base64(subject) target_event = find_target_event(recipient) - active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject) + active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event) + + from hashlib import sha256 + random_filename = 'mail-' + sha256(envelope.content).hexdigest() email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content), event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) diff --git a/core/mail/tests/v2/test_mails.py b/core/mail/tests/v2/test_mails.py index 3df56ca..3b358ca 100644 --- a/core/mail/tests/v2/test_mails.py +++ b/core/mail/tests/v2/test_mails.py @@ -142,6 +142,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('test Γ€', Email.objects.all()[0].subject) self.assertEqual('Text mit Quoted-Printable-Kodierung: Àâüß', Email.objects.all()[0].body) + self.assertTrue( Email.objects.all()[0].raw_file.path) def test_handle_quoted_printable_2(self): from aiosmtpd.smtp import Envelope @@ -162,6 +163,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('suche_MΓΌtze', Email.objects.all()[0].subject) self.assertEqual('Text mit Quoted-Printable-Kodierung: Àâüß', Email.objects.all()[0].body) + self.assertTrue( Email.objects.all()[0].raw_file.path) def test_handle_base64(self): from aiosmtpd.smtp import Envelope @@ -182,6 +184,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('test', Email.objects.all()[0].subject) self.assertEqual('Text mit Base64-Kodierung: Àâüß', Email.objects.all()[0].body) + self.assertTrue( Email.objects.all()[0].raw_file.path) def test_handle_client_reply(self): issue_thread = IssueThread.objects.create( @@ -229,6 +232,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual(IssueThread.objects.all()[0].name, 'test') self.assertEqual(IssueThread.objects.all()[0].state, 'pending_new') self.assertEqual(IssueThread.objects.all()[0].assigned_to, None) + self.assertTrue( Email.objects.all()[2].raw_file.path) def test_handle_client_reply_2(self): issue_thread = IssueThread.objects.create( @@ -281,6 +285,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual(IssueThread.objects.all()[0].name, 'test') self.assertEqual(IssueThread.objects.all()[0].state, 'pending_open') self.assertEqual(IssueThread.objects.all()[0].assigned_to, None) + self.assertTrue( Email.objects.all()[2].raw_file.path) def test_mail_reply(self): issue_thread = IssueThread.objects.create( @@ -384,6 +389,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) self.assertEqual(1, len(states)) self.assertEqual('pending_new', states[0].state) + self.assertEqual(event, IssueThread.objects.all()[0].event) def test_mail_html_body(self): from aiosmtpd.smtp import Envelope From 41b71bd51a7239c959bc352eb102ac131c3e8e43 Mon Sep 17 00:00:00 2001 From: jedi Date: Wed, 13 Nov 2024 23:51:54 +0100 Subject: [PATCH 16/22] partition tickets by event --- core/inventory/api_v2.py | 14 ++++---- core/inventory/serializers.py | 4 ++- core/inventory/tests/v2/test_items.py | 8 ++--- core/tickets/api_v2.py | 14 ++++++-- core/tickets/serializers.py | 5 ++- core/tickets/tests/v2/test_tickets.py | 46 ++++++++++++++++++++++++++- web/src/components/Navbar.vue | 6 +++- web/src/store.js | 8 +++-- 8 files changed, 85 insertions(+), 20 deletions(-) diff --git a/core/inventory/api_v2.py b/core/inventory/api_v2.py index 0a34140..b539e97 100644 --- a/core/inventory/api_v2.py +++ b/core/inventory/api_v2.py @@ -1,4 +1,4 @@ -from django.urls import path +from django.urls import re_path from django.contrib.auth.decorators import permission_required from rest_framework import routers, viewsets from rest_framework.decorators import api_view, permission_classes @@ -40,7 +40,9 @@ def search_items(request, event_slug, query): @permission_classes([IsAuthenticated]) def item(request, event_slug): try: - event = Event.objects.get(slug=event_slug) + event = None + if event_slug != 'none': + event = Event.objects.get(slug=event_slug) if request.method == 'GET': if not request.user.has_event_perm(event, 'view_item'): return Response(status=403) @@ -99,8 +101,8 @@ router.register(r'boxes', ContainerViewSet, basename='boxes') router.register(r'box', ContainerViewSet, basename='boxes') urlpatterns = router.urls + [ - path('/items/', item), - path('/items//', search_items), - path('/item/', item), - path('/item//', item_by_id), + re_path(r'^(?P[\w-]+)/items/$', item, name='item'), + re_path(r'^(?P[\w-]+)/items/(?P[-A-Za-z0-9+/]*={0,3})/$', search_items, name='search_items'), + re_path(r'^(?P[\w-]+)/item/$', item, name='item'), + re_path(r'^(?P[\w-]+)/item/(?P\d+)/$', item_by_id, name='item_by_id'), ] diff --git a/core/inventory/serializers.py b/core/inventory/serializers.py index 5a26623..3c0eb7e 100644 --- a/core/inventory/serializers.py +++ b/core/inventory/serializers.py @@ -39,10 +39,12 @@ class ItemSerializer(serializers.ModelSerializer): box = serializers.SerializerMethodField() file = serializers.SerializerMethodField() returned = serializers.SerializerMethodField(required=False) + event = serializers.SlugRelatedField(slug_field='slug', queryset=Event.objects.all(), + allow_null=True, required=False) class Meta: model = Item - fields = ['cid', 'box', 'uid', 'description', 'file', 'dataImage', 'returned'] + fields = ['cid', 'box', 'uid', 'description', 'file', 'dataImage', 'returned', 'event'] read_only_fields = ['uid'] def get_cid(self, instance): diff --git a/core/inventory/tests/v2/test_items.py b/core/inventory/tests/v2/test_items.py index 056b38c..a955161 100644 --- a/core/inventory/tests/v2/test_items.py +++ b/core/inventory/tests/v2/test_items.py @@ -30,7 +30,7 @@ class ItemTestCase(TestCase): self.assertEqual(response.status_code, 200) self.assertEqual(response.json(), [{'uid': 1, 'description': '1', 'box': 'BOX', 'cid': self.box.cid, 'file': None, - 'returned': False}]) + 'returned': False, 'event': self.event.slug}]) def test_members_with_file(self): import base64 @@ -40,7 +40,7 @@ class ItemTestCase(TestCase): self.assertEqual(response.status_code, 200) self.assertEqual(response.json(), [{'uid': 1, 'description': '1', 'box': 'BOX', 'cid': self.box.cid, 'file': file.hash, - 'returned': False}]) + 'returned': False, 'event': self.event.slug}]) def test_multi_members(self): Item.objects.create(container=self.box, event=self.event, description='1') @@ -55,7 +55,7 @@ class ItemTestCase(TestCase): self.assertEqual(response.status_code, 201) self.assertEqual(response.json(), {'uid': 1, 'description': '1', 'box': 'BOX', 'cid': self.box.cid, 'file': None, - 'returned': False}) + 'returned': False, 'event': self.event.slug}) self.assertEqual(len(Item.objects.all()), 1) self.assertEqual(Item.objects.all()[0].uid, 1) self.assertEqual(Item.objects.all()[0].description, '1') @@ -86,7 +86,7 @@ class ItemTestCase(TestCase): self.assertEqual(response.status_code, 200) self.assertEqual(response.json(), {'uid': 1, 'description': '2', 'box': 'BOX', 'cid': self.box.cid, 'file': None, - 'returned': False}) + 'returned': False, 'event': self.event.slug}) self.assertEqual(len(Item.objects.all()), 1) self.assertEqual(Item.objects.all()[0].uid, 1) self.assertEqual(Item.objects.all()[0].description, '2') diff --git a/core/tickets/api_v2.py b/core/tickets/api_v2.py index 596bd9b..39404f2 100644 --- a/core/tickets/api_v2.py +++ b/core/tickets/api_v2.py @@ -56,7 +56,7 @@ def reply(request, pk): @api_view(['POST']) @permission_classes([IsAuthenticated]) @permission_required('tickets.add_issuethread_manual', raise_exception=True) -def manual_ticket(request): +def manual_ticket(request, event_slug): if 'name' not in request.data: return Response({'status': 'error', 'message': 'missing name'}, status=status.HTTP_400_BAD_REQUEST) if 'sender' not in request.data: @@ -66,8 +66,16 @@ def manual_ticket(request): if 'body' not in request.data: return Response({'status': 'error', 'message': 'missing body'}, status=status.HTTP_400_BAD_REQUEST) + event = None + if event_slug != 'none': + try: + event = Event.objects.get(slug=event_slug) + except: + return Response({'status': 'error', 'message': 'invalid event'}, status=status.HTTP_400_BAD_REQUEST) + issue = IssueThread.objects.create( name=request.data['name'], + event=event, manually_created=True, ) email = Email.objects.create( @@ -122,8 +130,8 @@ router.register(r'tickets', IssueViewSet, basename='issues') router.register(r'shipping_vouchers', ShippingVoucherViewSet, basename='shipping_vouchers') urlpatterns = ([ + re_path(r'tickets/states/$', get_available_states, name='get_available_states'), re_path(r'^tickets/(?P\d+)/reply/$', reply, name='reply'), re_path(r'^tickets/(?P\d+)/comment/$', add_comment, name='add_comment'), - re_path(r'^tickets/manual/$', manual_ticket, name='manual_ticket'), - re_path(r'^tickets/states/$', get_available_states, name='get_available_states'), + re_path(r'^(?P[\w-]+)/tickets/manual/$', manual_ticket, name='manual_ticket'), ] + router.urls) diff --git a/core/tickets/serializers.py b/core/tickets/serializers.py index a980b97..f5b7ef6 100644 --- a/core/tickets/serializers.py +++ b/core/tickets/serializers.py @@ -1,6 +1,7 @@ from rest_framework import serializers from authentication.models import ExtendedUser +from inventory.models import Event from mail.api_v2 import AttachmentSerializer from tickets.models import IssueThread, Comment, STATE_CHOICES, ShippingVoucher from inventory.serializers import ItemSerializer @@ -41,11 +42,13 @@ class IssueSerializer(serializers.ModelSerializer): last_activity = serializers.SerializerMethodField() assigned_to = serializers.SlugRelatedField(slug_field='username', queryset=ExtendedUser.objects.all(), allow_null=True, required=False) + event = serializers.SlugRelatedField(slug_field='slug', queryset=Event.objects.all(), + allow_null=True, required=False) related_items = ItemSerializer(many=True, read_only=True) class Meta: model = IssueThread - fields = ('id', 'timeline', 'name', 'state', 'assigned_to', 'last_activity', 'uuid', 'related_items') + fields = ('id', 'timeline', 'name', 'state', 'assigned_to', 'last_activity', 'uuid', 'related_items', 'event') read_only_fields = ('id', 'timeline', 'last_activity', 'uuid', 'related_items') def to_internal_value(self, data): diff --git a/core/tickets/tests/v2/test_tickets.py b/core/tickets/tests/v2/test_tickets.py index 8223506..9e89ed5 100644 --- a/core/tickets/tests/v2/test_tickets.py +++ b/core/tickets/tests/v2/test_tickets.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta from django.test import TestCase, Client from authentication.models import ExtendedUser +from inventory.models import Event from mail.models import Email, EmailAttachment from tickets.models import IssueThread, StateChange, Comment from django.contrib.auth.models import Permission @@ -16,6 +17,7 @@ class IssueApiTest(TestCase): self.user = ExtendedUser.objects.create_user('testuser', 'test', 'test') self.user.user_permissions.add(*Permission.objects.all()) self.user.save() + self.event = Event.objects.create(slug='evt') self.token = AuthToken.objects.create(user=self.user) self.client = Client(headers={'Authorization': 'Token ' + self.token[1]}) @@ -28,6 +30,7 @@ class IssueApiTest(TestCase): now = datetime.now() issue = IssueThread.objects.create( name="test issue", + event=self.event, ) mail1 = Email.objects.create( subject='test', @@ -61,6 +64,7 @@ class IssueApiTest(TestCase): self.assertEqual(response.json()[0]['id'], issue.id) self.assertEqual(response.json()[0]['name'], "test issue") self.assertEqual(response.json()[0]['state'], "pending_new") + self.assertEqual(response.json()[0]['event'], "evt") self.assertEqual(response.json()[0]['assigned_to'], None) self.assertEqual(response.json()[0]['uuid'], issue.uuid) self.assertEqual(response.json()[0]['last_activity'], comment.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) @@ -93,12 +97,15 @@ class IssueApiTest(TestCase): now = datetime.now() issue1 = IssueThread.objects.create( name="test issue", + event=self.event, ) issue2 = IssueThread.objects.create( name="test issue", + event=self.event, ) issue3 = IssueThread.objects.create( name="test issue", + event=self.event, ) mail1 = Email.objects.create( subject='test', @@ -118,8 +125,11 @@ class IssueApiTest(TestCase): self.assertEqual(200, response.status_code) self.assertEqual(3, len(response.json())) self.assertEqual(issue1.id, response.json()[0]['id']) + self.assertEqual("evt", response.json()[0]['event']) self.assertEqual(issue2.id, response.json()[1]['id']) + self.assertEqual("evt", response.json()[1]['event']) self.assertEqual(issue3.id, response.json()[2]['id']) + self.assertEqual("evt", response.json()[2]['event']) self.assertEqual(issue1.state_changes.first().timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), response.json()[0]['last_activity']) self.assertEqual(mail1.timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'), @@ -153,6 +163,7 @@ class IssueApiTest(TestCase): now = datetime.now() issue = IssueThread.objects.create( name="test issue", + event=self.event, ) mail1 = Email.objects.create( subject='test', @@ -189,6 +200,7 @@ class IssueApiTest(TestCase): self.assertEqual(200, response.status_code) self.assertEqual(1, len(response.json())) self.assertEqual(issue.id, response.json()[0]['id']) + self.assertEqual("evt", response.json()[0]['event']) self.assertEqual('pending_new', response.json()[0]['state']) self.assertEqual('test issue', response.json()[0]['name']) self.assertEqual(None, response.json()[0]['assigned_to']) @@ -230,13 +242,14 @@ class IssueApiTest(TestCase): self.assertEqual(file2.hash, response.json()[0]['timeline'][1]['attachments'][1]['hash']) def test_manual_creation(self): - response = self.client.post('/api/2/tickets/manual/', + response = self.client.post('/api/2/evt/tickets/manual/', {'name': 'test issue', 'sender': 'test', 'recipient': 'test', 'body': 'test'}, content_type='application/json') self.assertEqual(response.status_code, 201) self.assertEqual(response.json()['state'], 'pending_new') self.assertEqual(response.json()['name'], 'test issue') self.assertEqual(response.json()['assigned_to'], None) + self.assertEqual("evt", response.json()['event']) timeline = response.json()['timeline'] self.assertEqual(len(timeline), 2) self.assertEqual(timeline[0]['type'], 'state') @@ -247,9 +260,35 @@ class IssueApiTest(TestCase): self.assertEqual(timeline[1]['subject'], 'test issue') self.assertEqual(timeline[1]['body'], 'test') + def test_manual_creation_none(self): + response = self.client.post('/api/2/none/tickets/manual/', + {'name': 'test issue', 'sender': 'test', 'recipient': 'test', 'body': 'test'}, + content_type='application/json') + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json()['state'], 'pending_new') + self.assertEqual(response.json()['name'], 'test issue') + self.assertEqual(response.json()['assigned_to'], None) + self.assertEqual(None, response.json()['event']) + timeline = response.json()['timeline'] + self.assertEqual(len(timeline), 2) + self.assertEqual(timeline[0]['type'], 'state') + self.assertEqual(timeline[0]['state'], 'pending_new') + self.assertEqual(timeline[1]['type'], 'mail') + self.assertEqual(timeline[1]['sender'], 'test') + self.assertEqual(timeline[1]['recipient'], 'test') + self.assertEqual(timeline[1]['subject'], 'test issue') + self.assertEqual(timeline[1]['body'], 'test') + + def test_manual_creation_invalid(self): + response = self.client.post('/api/2/foobar/tickets/manual/', + {'name': 'test issue', 'sender': 'test', 'recipient': 'test', 'body': 'test'}, + content_type='application/json') + self.assertEqual(response.status_code, 400) + def test_post_comment_altenative(self): issue = IssueThread.objects.create( name="test issue", + event=self.event, ) response = self.client.post(f'/api/2/tickets/{issue.id}/comment/', {'comment': 'test'}) self.assertEqual(response.status_code, 201) @@ -260,6 +299,7 @@ class IssueApiTest(TestCase): def test_post_alt_comment_empty(self): issue = IssueThread.objects.create( name="test issue", + event=self.event, ) response = self.client.post(f'/api/2/tickets/{issue.id}/comment/', {'comment': ''}) self.assertEqual(response.status_code, 400) @@ -267,6 +307,7 @@ class IssueApiTest(TestCase): def test_state_change(self): issue = IssueThread.objects.create( name="test issue", + event=self.event, ) response = self.client.patch(f'/api/2/tickets/{issue.id}/', {'state': 'pending_open'}, content_type='application/json') @@ -284,6 +325,7 @@ class IssueApiTest(TestCase): def test_state_change_invalid_state(self): issue = IssueThread.objects.create( name="test issue", + event=self.event, ) response = self.client.patch(f'/api/2/tickets/{issue.id}/', {'state': 'invalid'}, content_type='application/json') @@ -292,12 +334,14 @@ class IssueApiTest(TestCase): def test_assign_user(self): issue = IssueThread.objects.create( name="test issue", + event=self.event, ) response = self.client.patch(f'/api/2/tickets/{issue.id}/', {'assigned_to': self.user.username}, content_type='application/json') self.assertEqual(200, response.status_code) self.assertEqual('pending_new', response.json()['state']) self.assertEqual('test issue', response.json()['name']) + self.assertEqual("evt", response.json()['event']) self.assertEqual(self.user.username, response.json()['assigned_to']) timeline = response.json()['timeline'] self.assertEqual(2, len(timeline)) diff --git a/web/src/components/Navbar.vue b/web/src/components/Navbar.vue index 686f324..eb7504b 100644 --- a/web/src/components/Navbar.vue +++ b/web/src/components/Navbar.vue @@ -6,7 +6,8 @@ {{ getEventSlug }}
- {{ event.slug }}
@@ -115,6 +116,9 @@ export default { computed: { ...mapState(['events']), ...mapGetters(['getEventSlug', 'getActiveView', "checkPermission", "hasPermissions", "layout", "route"]), + selectableEvents() { + return [{slug: 'all'}, ...this.events, {slug: 'none'}]; + } }, methods: { ...mapActions(['changeEvent', 'changeView']), diff --git a/web/src/store.js b/web/src/store.js index 53d01b6..3140a99 100644 --- a/web/src/store.js +++ b/web/src/store.js @@ -21,7 +21,7 @@ const store = createStore({ state_options: [], shippingVouchers: [], - lastEvent: '37C3', + lastEvent: 'all', lastUsed: {}, searchQuery: '', remember: false, @@ -358,7 +358,8 @@ const store = createStore({ async searchEventItems({commit, getters, state}, query) { const encoded_query = base64.encode(utf8.encode(query)); - const {data, success} = await http.get(`/2/${getters.getEventSlug}/items/${encoded_query}/`, state.user.token); + const {data, success} = await http.get(`/2/${getters.getEventSlug}/items/${encoded_query}/`, + state.user.token); if (data && success) commit('replaceLoadedItems', data); }, @@ -410,7 +411,8 @@ const store = createStore({ async searchEventTickets({commit, getters, state}, query) { const encoded_query = base64.encode(utf8.encode(query)); - const {data, success} = await http.get(`/2/${getters.getEventSlug}/tickets/${encoded_query}/`, state.user.token); + const {data, success} = await http.get(`/2/${getters.getEventSlug}/tickets/${encoded_query}/`, + state.user.token); if (data && success) commit('replaceTickets', data); }, From c9d58191b376718e5b1ff022b1136d6c3e427dd1 Mon Sep 17 00:00:00 2001 From: jedi Date: Sun, 17 Nov 2024 00:16:54 +0100 Subject: [PATCH 17/22] show tickets filtered by active event --- core/core/settings.py | 2 + core/tickets/serializers.py | 2 - web/src/components/Navbar.vue | 17 +- web/src/store.js | 288 ++++++++++++++++------------------ web/src/utils.js | 10 +- web/src/views/Items.vue | 10 +- web/src/views/Ticket.vue | 25 +-- web/src/views/Tickets.vue | 19 ++- 8 files changed, 175 insertions(+), 198 deletions(-) diff --git a/core/core/settings.py b/core/core/settings.py index 2d4a818..6796112 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -15,9 +15,11 @@ import sys import dotenv from pathlib import Path + def truthy_str(s): return s.lower() in ['true', '1', 't', 'y', 'yes', 'yeah', 'yup', 'certainly', 'sure', 'positive', 'uh-huh', 'πŸ‘'] + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent diff --git a/core/tickets/serializers.py b/core/tickets/serializers.py index f5b7ef6..9f14e7e 100644 --- a/core/tickets/serializers.py +++ b/core/tickets/serializers.py @@ -55,8 +55,6 @@ class IssueSerializer(serializers.ModelSerializer): ret = super().to_internal_value(data) if 'state' in data: ret['state'] = data['state'] -# if 'assigned_to' in data: -# ret['assigned_to'] = data['assigned_to'] return ret def validate(self, attrs): diff --git a/web/src/components/Navbar.vue b/web/src/components/Navbar.vue index eb7504b..ccfb0f0 100644 --- a/web/src/components/Navbar.vue +++ b/web/src/components/Navbar.vue @@ -49,12 +49,12 @@ @@ -65,19 +65,6 @@