From 3709b5dd29d101527a6a24aa51ca97559325f5e7 Mon Sep 17 00:00:00 2001 From: jedi Date: Thu, 5 Dec 2024 01:31:06 +0100 Subject: [PATCH 01/41] stash --- .forgejo/workflows/test.yml | 11 +++- core/.coveragerc | 2 +- core/authentication/api_v2.py | 4 +- core/authentication/tests/v2/test_users.py | 11 ++++ core/core/globals.py | 8 +-- core/core/settings.py | 32 +++++++--- core/helper.py | 30 ---------- core/integration_tests/main.py | 69 ++++++++++++++++++++++ core/integration_tests/test_bar.py | 1 + core/integration_tests/test_foo.sh | 5 ++ core/mail/protocol.py | 7 ++- core/mail/socket.py | 8 +-- core/server.py | 4 +- deploy/dev/docker-compose.yml | 28 ++------- deploy/testing/docker-compose.yml | 21 ++++++- 15 files changed, 156 insertions(+), 85 deletions(-) delete mode 100644 core/helper.py create mode 100644 core/integration_tests/main.py create mode 100644 core/integration_tests/test_bar.py create mode 100644 core/integration_tests/test_foo.sh diff --git a/.forgejo/workflows/test.yml b/.forgejo/workflows/test.yml index 480e590..14c7591 100644 --- a/.forgejo/workflows/test.yml +++ b/.forgejo/workflows/test.yml @@ -1,3 +1,4 @@ +name: Test on: pull_request: push: @@ -16,6 +17,12 @@ jobs: - name: Install dependencies working-directory: core run: pip3 install -r requirements.dev.txt - - name: Run django tests + - name: Run django tests with coverage working-directory: core - run: python3 manage.py test + run: coverage run manage.py test + - name: Run integration tests with coverage + working-directory: core + run: python3 integration_tests/main.py + - name: Evaluate coverage + working-directory: core + run: coverage report diff --git a/core/.coveragerc b/core/.coveragerc index 14c1fba..d06fc0e 100644 --- a/core/.coveragerc +++ b/core/.coveragerc @@ -8,7 +8,7 @@ skip_covered = True omit = */tests/* */migrations/* + integration_tests/* core/asgi.py - core/wsgi.py core/settings.py manage.py \ No newline at end of file diff --git a/core/authentication/api_v2.py b/core/authentication/api_v2.py index 2547b6d..b7ecfe3 100644 --- a/core/authentication/api_v2.py +++ b/core/authentication/api_v2.py @@ -54,9 +54,9 @@ def registerUser(request): errors['password'] = 'Password is required' if not email: errors['email'] = 'Email is required' - if ExtendedUser.objects.filter(email=email).exists(): + if email and ExtendedUser.objects.filter(email=email).exists(): errors['email'] = 'Email already exists' - if ExtendedUser.objects.filter(username=username).exists(): + if username and ExtendedUser.objects.filter(username=username).exists(): errors['username'] = 'Username already exists' if errors: return Response({'errors': errors}, status=400) diff --git a/core/authentication/tests/v2/test_users.py b/core/authentication/tests/v2/test_users.py index 125be9b..02096ea 100644 --- a/core/authentication/tests/v2/test_users.py +++ b/core/authentication/tests/v2/test_users.py @@ -72,6 +72,17 @@ class UserApiTest(TestCase): self.assertEqual(ExtendedUser.objects.get(username='testuser2').email, 'test2') self.assertTrue(ExtendedUser.objects.get(username='testuser2').check_password('test')) + def test_register_user_fail(self): + anonymous = Client() + response = anonymous.post('/api/2/register/', {'username': 'testuser2', 'password': 'test', 'email': 'test2'}, + content_type='application/json') + self.assertEqual(response.status_code, 201) + self.assertEqual(response.json()['username'], 'testuser2') + self.assertEqual(response.json()['email'], 'test2') + self.assertEqual(len(ExtendedUser.objects.all()), 3) + self.assertEqual(ExtendedUser.objects.get(username='testuser2').email, 'test2') + self.assertTrue(ExtendedUser.objects.get(username='testuser2').check_password('test')) + def test_register_user_duplicate(self): anonymous = Client() response = anonymous.post('/api/2/register/', {'username': 'testuser', 'password': 'test', 'email': 'test2'}, diff --git a/core/core/globals.py b/core/core/globals.py index d84a7a0..eae7884 100644 --- a/core/core/globals.py +++ b/core/core/globals.py @@ -2,12 +2,7 @@ import asyncio import logging import signal -loop = asyncio.get_event_loop() - - -def create_task(coro): - global loop - loop.create_task(coro) +loop = None async def shutdown(sig, loop): @@ -24,6 +19,7 @@ async def shutdown(sig, loop): def init_loop(): global loop + loop = asyncio.get_event_loop() loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop))) loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop))) return loop diff --git a/core/core/settings.py b/core/core/settings.py index 5a8f20f..e9d6dea 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -124,19 +124,12 @@ TEMPLATES = [ }, ] -WSGI_APPLICATION = 'core.wsgi.application' +ASGI_APPLICATION = 'core.asgi.application' # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases -if 'test' in sys.argv: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', - } - } -else: +if os.getenv('DB_HOST') is not None: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', @@ -149,6 +142,20 @@ else: 'charset': 'utf8mb4', 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'" } + }, + } +elif os.getenv('DB_FILE') is not None: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.getenv('DB_FILE', 'local.db'), + } + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', } } @@ -223,6 +230,13 @@ CHANNEL_LAYERS = { } +OUTBOUND_SMTP = { + 'hostname': os.getenv('OUTBOUND_SMTP_HOST', '127.0.0.1'), + 'port': int(os.getenv('OUTBOUND_SMTP_PORT', '25')), + 'use_tls': False, + 'start_tls': False +} + PROMETHEUS_METRIC_NAMESPACE = 'c3lf' TEST_RUNNER = 'core.test_runner.FastTestRunner' diff --git a/core/helper.py b/core/helper.py deleted file mode 100644 index ae3c53b..0000000 --- a/core/helper.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -import logging -import signal - -loop = None - - -def create_task(coro): - global loop - loop.create_task(coro) - - -async def shutdown(sig, loop): - log = logging.getLogger() - log.info(f"Received exit signal {sig.name}...") - tasks = [t for t in asyncio.all_tasks() if t is not - asyncio.current_task()] - [task.cancel() for task in tasks] - log.info(f"Cancelling {len(tasks)} outstanding tasks") - await asyncio.wait_for(loop.shutdown_asyncgens(), timeout=10) - loop.stop() - log.info("Shutdown complete.") - - -def init_loop(): - global loop - loop = asyncio.get_event_loop() - loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.create_task(shutdown(signal.SIGTERM, loop))) - loop.add_signal_handler(signal.SIGINT, lambda: asyncio.create_task(shutdown(signal.SIGINT, loop))) - return loop diff --git a/core/integration_tests/main.py b/core/integration_tests/main.py new file mode 100644 index 0000000..b0ef7a1 --- /dev/null +++ b/core/integration_tests/main.py @@ -0,0 +1,69 @@ +import time +import os +import signal +import re +import sys +import subprocess + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def run(): + while True: + newpid = os.fork() + if newpid == 0: + import coverage + cov = coverage.Coverage() + cov.load() + cov.start() + signal.signal(signal.SIGINT, signal.default_int_handler) + try: + from server import main + main() + except KeyboardInterrupt: + pass + cov.stop() + cov.save() + os._exit(0) + else: + return newpid + + +if __name__ == '__main__': + os.environ["DB_FILE"] = "local.sqlite3" + os.environ["OUTBOUND_SMTP_PORT"] = '1025' + pid = run() + result = subprocess.run(['python3', 'manage.py', 'migrate'], capture_output=True, text=True, env=os.environ) + if result.returncode != 0: + print('Migration failed', file=sys.stderr) + print(result.stdout, file=sys.stderr) + print(result.stderr, file=sys.stderr) + os._exit(1) + time.sleep(2) + script_dir = os.path.dirname(os.path.abspath(__file__)) + pattern = re.compile(r'^test_.*\.(sh|py)$') + dotpy = re.compile(r'^.*\.py$') + dotsh = re.compile(r'^.*\.sh$') + matching_files = [ + filename for filename in os.listdir(script_dir) + if os.path.isfile(os.path.join(script_dir, filename)) and pattern.match(filename) + ] + total = 0 + failed = 0 + for file in matching_files: + file_path = os.path.join(script_dir, file) + if dotpy.match(file): + result = subprocess.run(['python3', file_path], capture_output=True, text=True, env=os.environ) + elif dotsh.match(file): + result = subprocess.run(['bash', file_path], capture_output=True, text=True, env=os.environ) + else: + result = subprocess.run(['bash', '-c', file_path], capture_output=True, text=True, env=os.environ) + print('{} returned {}'.format(file, result.returncode), file=sys.stderr) + if result.returncode != 0: + print(result.stderr, result.stdout, file=sys.stderr) + failed += 1 + total += 1 + time.sleep(1) + os.kill(pid, signal.SIGINT) + print(f'{total - failed} out of {total} tests succeeded, {failed} failed', file=sys.stderr) + os._exit(0 if failed == 0 else 1) diff --git a/core/integration_tests/test_bar.py b/core/integration_tests/test_bar.py new file mode 100644 index 0000000..5f7ce86 --- /dev/null +++ b/core/integration_tests/test_bar.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 \ No newline at end of file diff --git a/core/integration_tests/test_foo.sh b/core/integration_tests/test_foo.sh new file mode 100644 index 0000000..65c2eaf --- /dev/null +++ b/core/integration_tests/test_foo.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +swaks --to user@localhost --socket lmtp.sock --protocol LMTP diff --git a/core/mail/protocol.py b/core/mail/protocol.py index c5aaa4a..668f656 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -88,7 +88,9 @@ def make_reply(reply_email, references=None, event=None): async def send_smtp(message): - await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False) + from core.settings import OUTBOUND_SMTP + if OUTBOUND_SMTP: + await aiosmtplib.send(message, **OUTBOUND_SMTP) def find_active_issue_thread(in_reply_to, address, subject, event): @@ -209,7 +211,8 @@ def receive_email(envelope, log=None): email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), + event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) diff --git a/core/mail/socket.py b/core/mail/socket.py index 312b916..5220795 100644 --- a/core/mail/socket.py +++ b/core/mail/socket.py @@ -25,7 +25,7 @@ class UnixSocketLMTPController(UnixSocketMixin, BaseAsyncController): def factory(self): return LMTP(self.handler) - def _trigger_server(self): # pragma: no-unixsock - # Prevent confusion on which _trigger_server() to invoke. - # Or so LGTM.com claimed - UnixSocketMixin._trigger_server(self) +# def _trigger_server(self): # pragma: no-unixsock +# # Prevent confusion on which _trigger_server() to invoke. +# # Or so LGTM.com claimed +# UnixSocketMixin._trigger_server(self) diff --git a/core/server.py b/core/server.py index d08b595..59dcb3f 100644 --- a/core/server.py +++ b/core/server.py @@ -9,7 +9,7 @@ import uvicorn django.setup() -from helper import init_loop +from core.globals import init_loop from mail.protocol import LMTPHandler from mail.socket import UnixSocketLMTPController @@ -51,7 +51,6 @@ async def lmtp(loop): async with server: await server.serve_forever() - log.info("LMTP done") def main(): @@ -82,7 +81,6 @@ def main(): os.remove("web.sock") except Exception as e: log.error(e) - log.error(e) logging.info("Server stopped") logging.shutdown() diff --git a/deploy/dev/docker-compose.yml b/deploy/dev/docker-compose.yml index dff5ab3..f7c9523 100644 --- a/deploy/dev/docker-compose.yml +++ b/deploy/dev/docker-compose.yml @@ -3,20 +3,15 @@ services: build: context: ../../core dockerfile: ../deploy/dev/Dockerfile.backend - command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000' + command: bash -c 'python manage.py migrate && python testdata.py && python manage.py runserver 0.0.0.0:8000' environment: - HTTP_HOST=core - - DB_HOST=db - - DB_PORT=3306 - - DB_NAME=system3 - - DB_USER=system3 - - DB_PASSWORD=system3 + - DB_FILE=dev.db volumes: - ../../core:/code + - ../testdata.py:/code/testdata.py ports: - "8000:8000" - depends_on: - - db frontend: build: @@ -30,19 +25,4 @@ services: ports: - "8080:8080" depends_on: - - core - - db: - image: mariadb - environment: - MARIADB_RANDOM_ROOT_PASSWORD: true - MARIADB_DATABASE: system3 - MARIADB_USER: system3 - MARIADB_PASSWORD: system3 - volumes: - - mariadb_data:/var/lib/mysql - ports: - - "3306:3306" - -volumes: - mariadb_data: \ No newline at end of file + - core \ No newline at end of file diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml index e93e901..a2ece91 100644 --- a/deploy/testing/docker-compose.yml +++ b/deploy/testing/docker-compose.yml @@ -20,7 +20,7 @@ services: build: context: ../../core dockerfile: ../deploy/testing/Dockerfile.backend - command: bash -c 'python manage.py migrate && python /code/server.py' + command: bash -c 'python manage.py migrate && python testdata.py && python /code/server.py' environment: - HTTP_HOST=core - REDIS_HOST=redis @@ -29,13 +29,16 @@ services: - DB_NAME=system3 - DB_USER=system3 - DB_PASSWORD=system3 + - MAIL_DOMAIN=mail:1025 volumes: - ../../core:/code + - ../testdata.py:/code/testdata.py ports: - "8000:8000" depends_on: - db - redis + - mail frontend: build: @@ -51,5 +54,19 @@ services: depends_on: - core + mail: + image: docker.io/axllent/mailpit + volumes: + - mailpit_data:/data + ports: + - 8025:8025 + - 1025:1025 + environment: + MP_MAX_MESSAGES: 5000 + MP_DATABASE: /data/mailpit.db + MP_SMTP_AUTH_ACCEPT_ANY: 1 + MP_SMTP_AUTH_ALLOW_INSECURE: 1 + volumes: - mariadb_data: \ No newline at end of file + mariadb_data: + mailpit_data: \ No newline at end of file From 9141752fdfcb39094293c255062c63be42200751 Mon Sep 17 00:00:00 2001 From: jedi Date: Tue, 17 Dec 2024 21:52:57 +0100 Subject: [PATCH 02/41] fix container selection in AddItemModal.vue --- web/src/components/AddItemModal.vue | 55 ++++++++++++++++++------ web/src/components/AddTicketModal.vue | 3 +- web/src/components/EditItem.vue | 50 --------------------- web/src/components/inputs/InputCombo.vue | 6 +-- web/src/views/Item.vue | 2 - web/src/views/Items.vue | 5 +-- 6 files changed, 48 insertions(+), 73 deletions(-) delete mode 100644 web/src/components/EditItem.vue diff --git a/web/src/components/AddItemModal.vue b/web/src/components/AddItemModal.vue index a3c23fd..24bd449 100644 --- a/web/src/components/AddItemModal.vue +++ b/web/src/components/AddItemModal.vue @@ -2,7 +2,29 @@
diff --git a/web/src/components/AddTicketModal.vue b/web/src/components/AddTicketModal.vue index 37d539c..b407670 100644 --- a/web/src/components/AddTicketModal.vue +++ b/web/src/components/AddTicketModal.vue @@ -19,11 +19,10 @@ diff --git a/web/src/components/inputs/InputCombo.vue b/web/src/components/inputs/InputCombo.vue index fc64d42..2a291e0 100644 --- a/web/src/components/inputs/InputCombo.vue +++ b/web/src/components/inputs/InputCombo.vue @@ -43,11 +43,11 @@ export default { props: ['label', 'model', 'nameKey', 'uniqueKey', 'options', 'onOptionAdd'], data: ({options, model, nameKey, uniqueKey}) => ({ internalName: model[nameKey], - selectedOption: options.filter(e => e[uniqueKey] == model[uniqueKey])[0], + selectedOption: options.filter(e => e[uniqueKey] === model[uniqueKey])[0], addingOption: false }), computed: { - isValid: ({options, nameKey, internalName}) => options.some(e => e[nameKey] == internalName), + isValid: ({options, nameKey, internalName}) => options.some(e => e[nameKey] === internalName), sortedOptions: ({ options, nameKey @@ -56,7 +56,7 @@ export default { watch: { internalName(newValue) { if (this.isValid) { - if (!this.selectedOption || newValue != this.selectedOption[this.nameKey]) { + if (!this.selectedOption || newValue !== this.selectedOption[this.nameKey]) { this.selectedOption = this.options.filter(e => e[this.nameKey] === newValue)[0]; } this.model[this.nameKey] = this.selectedOption[this.nameKey]; diff --git a/web/src/views/Item.vue b/web/src/views/Item.vue index d00b8b1..7b9f484 100644 --- a/web/src/views/Item.vue +++ b/web/src/views/Item.vue @@ -112,14 +112,12 @@ import InputString from "@/components/inputs/InputString.vue"; import AuthenticatedImage from "@/components/AuthenticatedImage.vue"; import InputPhoto from "@/components/inputs/InputPhoto.vue"; import Modal from "@/components/Modal.vue"; -import EditItem from "@/components/EditItem.vue"; import AsyncButton from "@/components/inputs/AsyncButton.vue"; export default { name: 'Item', components: { AsyncButton, - EditItem, Modal, InputPhoto, AuthenticatedImage, InputString, InputCombo, AsyncLoader, ClipboardButton, Timeline }, data() { diff --git a/web/src/views/Items.vue b/web/src/views/Items.vue index 21b11d1..30e553f 100644 --- a/web/src/views/Items.vue +++ b/web/src/views/Items.vue @@ -67,11 +67,10 @@ + + \ No newline at end of file diff --git a/web/src/views/TicketSearch.vue b/web/src/views/TicketSearch.vue new file mode 100644 index 0000000..35e7d07 --- /dev/null +++ b/web/src/views/TicketSearch.vue @@ -0,0 +1,102 @@ + + + + + \ No newline at end of file From 0fa52645c2605be8afd3e63f92ccfb068e106131 Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 18 Jan 2025 22:15:04 +0100 Subject: [PATCH 21/41] handle plain base64 as transfer-encoding in incoming mails --- core/mail/protocol.py | 8 ++++++-- core/mail/tests/v2/test_mails.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 7fe6942..00872f0 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -128,10 +128,14 @@ def decode_email_segment(segment, charset, transfer_encoding): decode_as = 'cp1251' elif charset == 'iso-8859-1': decode_as = 'latin1' - segment = unescape_and_decode_quoted_printable(segment) - segment = unescape_and_decode_base64(segment) if transfer_encoding == 'quoted-printable': segment = unescape_simplified_quoted_printable(segment, decode_as) + elif transfer_encoding == 'base64': + import base64 + segment = base64.b64decode(segment).decode('utf-8') + else: + segment = unescape_and_decode_quoted_printable(segment) + segment = unescape_and_decode_base64(segment) return segment diff --git a/core/mail/tests/v2/test_mails.py b/core/mail/tests/v2/test_mails.py index 455faf1..d2f33fe 100644 --- a/core/mail/tests/v2/test_mails.py +++ b/core/mail/tests/v2/test_mails.py @@ -165,7 +165,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body) self.assertTrue(Email.objects.all()[0].raw_file.path) - def test_handle_base64(self): + def test_handle_base64_inline(self): from aiosmtpd.smtp import Envelope from asgiref.sync import async_to_sync import aiosmtplib @@ -186,6 +186,35 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual('Text mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) self.assertTrue(Email.objects.all()[0].raw_file.path) + def test_handle_base64_transfer_encoding(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'''Subject: test +From: test3@test +To: test4@test +Message-ID: <1@test> +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: base64 + +VGVzdCBtaXQgQmFzZTY0LUtvZGllcnVuZzogw6TDtsO8w58=''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 2) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('Test mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) + self.assertTrue(Email.objects.all()[0].raw_file.path) + def test_handle_client_reply(self): issue_thread = IssueThread.objects.create( name="test", From f133ae9e60450d3302a1c962c3e025c754ca52e7 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 17:39:13 +0100 Subject: [PATCH 22/41] allow searching while "all" event is selected --- web/src/store.js | 82 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/web/src/store.js b/web/src/store.js index b276086..e747ae7 100644 --- a/web/src/store.js +++ b/web/src/store.js @@ -77,10 +77,26 @@ const store = createStore({ getEventTickets: (state, getters) => getters.getEventSlug === 'all' ? getters.getAllTickets : getters.getAllTickets.filter(t => t.event === getters.getEventSlug || (t.event == null && getters.getEventSlug === 'none')), isItemsLoaded: (state, getters) => (getters.getEventSlug === 'all' || getters.getEventSlug === 'none') ? !!state.loadedItems : Object.keys(state.loadedItems).includes(getters.getEventSlug), isTicketsLoaded: (state, getters) => (getters.getEventSlug === 'all' || getters.getEventSlug === 'none') ? !!state.loadedTickets : Object.keys(state.loadedTickets).includes(getters.getEventSlug), - getItemsSearchResults: (state, getters) => state.loadedItemSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [], - getTicketsSearchResults: (state, getters) => state.loadedTicketSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [], - isItemsSearchLoaded: (state, getters) => Object.keys(state.loadedItemSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))), - isTicketsSearchLoaded: (state, getters) => Object.keys(state.loadedTicketSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))), + getItemsSearchResults: (state, getters) => { + if (getters.getEventSlug === 'all') { + return state.events.map(e => { + return state.loadedItemSearchResults[e.slug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + }).flat(); + } else { + return state.loadedItemSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + } + }, + getTicketsSearchResults: (state, getters) => { + if (getters.getEventSlug === 'all') { + return state.events.map(e => { + return state.loadedTicketSearchResults[e.slug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + }).flat(); + } else { + return state.loadedTicketSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + } + }, + isItemsSearchLoaded: (state, getters) => Object.keys(state.loadedItemSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))) || getters.getEventSlug === 'all', + isTicketsSearchLoaded: (state, getters) => Object.keys(state.loadedTicketSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))) || getters.getEventSlug === 'all', getActiveView: state => router.currentRoute.value.name || 'items', getFilters: state => router.currentRoute.value.query, getBoxes: state => state.loadedBoxes, @@ -379,26 +395,39 @@ const store = createStore({ }, async loadEventItems({commit, getters, state}) { if (!state.user.token) return; - if (state.fetchedData.items > Date.now() - 1000 * 60 * 60 * 24) return; - try { - const slug = getters.getEventSlug; - const {data, success} = await getters.session.get(`/2/${slug}/items/`); - if (data && success) { - commit('setItems', {slug, items: data}); + const load = async (slug) => { + try { + const {data, success} = await getters.session.get(`/2/${slug}/items/`); + if (data && success) { + commit('setItems', {slug, items: data}); + } + } catch (e) { + console.error("Error loading items"); } - } catch (e) { - console.error("Error loading items"); + } + const slug = getters.getEventSlug; + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); } }, async searchEventItems({commit, getters, state}, query) { const encoded_query = base64.encode(utf8.encode(query)); + const load = async (slug) => { + if (Object.keys(state.loadedItemSearchResults).includes(slug + '/' + encoded_query)) return; + const { + data, success + } = await getters.session.get(`/2/${slug}/items/${encoded_query}/`); + if (data && success) { + commit('setItemSearchResults', {slug, query: encoded_query, items: data}); + } + } const slug = getters.getEventSlug; - if (Object.keys(state.loadedItemSearchResults).includes(slug + '/' + encoded_query)) return; - const { - data, success - } = await getters.session.get(`/2/${slug}/items/${encoded_query}/`); - if (data && success) { - commit('setItemSearchResults', {slug, query: encoded_query, items: data}); + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); } }, async loadBoxes({commit, state, getters}) { @@ -446,12 +475,19 @@ const store = createStore({ }, async searchEventTickets({commit, getters, state}, query) { const encoded_query = base64.encode(utf8.encode(query)); + const load = async (slug) => { + if (Object.keys(state.loadedTicketSearchResults).includes(slug + '/' + encoded_query)) return; + const { + data, success + } = await getters.session.get(`/2/${slug}/tickets/${encoded_query}/`); + if (data && success) commit('setTicketSearchResults', {slug, query: encoded_query, items: data}); + } const slug = getters.getEventSlug; - if (Object.keys(state.loadedTicketSearchResults).includes(slug + '/' + encoded_query)) return; - const { - data, success - } = await getters.session.get(`/2/${slug}/tickets/${encoded_query}/`); - if (data && success) commit('setTicketSearchResults', {slug, query: encoded_query, items: data}); + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); + } }, async sendMail({commit, dispatch, state, getters}, {id, message}) { const {data, success} = await getters.session.post(`/2/tickets/${id}/reply/`, {message}, From 4ea74637a3d7f59accef14b4a2885c34edd3dfc5 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 18:30:42 +0100 Subject: [PATCH 23/41] finally get a grip on utf-8 --- core/mail/protocol.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 00872f0..3639989 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -53,6 +53,12 @@ def unescape_simplified_quoted_printable(s, encoding='utf-8'): return quopri.decodestring(s).decode(encoding) +def decode_inline_encodings(s): + s = unescape_and_decode_quoted_printable(s) + s = unescape_and_decode_base64(s) + return s + + def ascii_strip(s): if not s: return None @@ -134,8 +140,7 @@ def decode_email_segment(segment, charset, transfer_encoding): import base64 segment = base64.b64decode(segment).decode('utf-8') else: - segment = unescape_and_decode_quoted_printable(segment) - segment = unescape_and_decode_base64(segment) + segment = decode_inline_encodings(segment.decode('utf-8')) return segment @@ -160,7 +165,7 @@ def parse_email_body(raw, log=None): segment = part.get_payload() if not segment: continue - segment = decode_email_segment(segment, charset, part.get('Content-Transfer-Encoding')) + segment = decode_email_segment(segment.encode('utf-8'), charset, part.get('Content-Transfer-Encoding')) log.debug(segment) body = body + segment elif 'attachment' in cdispo or 'inline' in cdispo: @@ -193,7 +198,8 @@ def parse_email_body(raw, log=None): else: log.warning("Unknown content type %s", parsed.get_content_type()) body = "Unknown content type" - body = decode_email_segment(body, parsed.get_content_charset(), parsed.get('Content-Transfer-Encoding')) + body = decode_email_segment(body.encode('utf-8'), parsed.get_content_charset(), + parsed.get('Content-Transfer-Encoding')) log.debug(body) return parsed, body, attachments @@ -221,8 +227,9 @@ def receive_email(envelope, log=None): subject = ascii_strip(parsed.get('Subject')) if not subject: subject = "No subject" - subject = unescape_and_decode_quoted_printable(subject) - subject = unescape_and_decode_base64(subject) + subject = decode_inline_encodings(subject) + recipient = decode_inline_encodings(recipient) + sender = decode_inline_encodings(sender) target_event = find_target_event(recipient) active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event) @@ -256,7 +263,7 @@ do not create a new request. Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid()) reply_email = Email.objects.create( - sender=recipient, recipient=sender, body=body, subject=ascii_strip(subject), + sender=recipient, recipient=sender, body=body, subject=subject, in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) reply = make_reply(reply_email, references, event=target_event.slug if target_event else None) else: From fbbf8352cf9fdfe0af06c1b8df1ab92ae4832b50 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 19:43:01 +0100 Subject: [PATCH 24/41] don't report "Internal Server Error" if mail already exists --- core/mail/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 3639989..8fa2c3b 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -220,7 +220,7 @@ def receive_email(envelope, log=None): if Email.objects.filter(reference=header_message_id).exists(): # break before issue thread is created log.warning("Email already exists") - raise Exception("Email already exists") + raise SpecialMailException("Email already exists") recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower() sender = envelope.mail_from if envelope.mail_from else header_from From 2677f4b8b6c1d614e2bcb3ff3278eb5ea46e33c3 Mon Sep 17 00:00:00 2001 From: jedi Date: Sun, 26 Jan 2025 19:56:25 +0100 Subject: [PATCH 25/41] link item to ticket frontend --- web/src/store.js | 9 ++++++++- web/src/views/Ticket.vue | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/web/src/store.js b/web/src/store.js index e747ae7..34650e4 100644 --- a/web/src/store.js +++ b/web/src/store.js @@ -61,7 +61,6 @@ const store = createStore({ '2kg-de': '2kg Paket (DE)', '5kg-de': '5kg Paket (DE)', '10kg-de': '10kg Paket (DE)', - '2kg-eu': '2kg Paket (EU)', '5kg-eu': '5kg Paket (EU)', '10kg-eu': '10kg Paket (EU)', } @@ -564,6 +563,14 @@ const store = createStore({ state.fetchedData.tickets = 0; await Promise.all([dispatch('loadTickets'), dispatch('fetchShippingVouchers')]); } + }, + async linkTicketItem({dispatch, state, getters}, {ticket_id, item_id}) { + const {data, success} = await getters.session.post(`/2/matches/`, {issue_thread: ticket_id, item: item_id}); + if (data && success) { + state.fetchedData.tickets = 0; + state.fetchedData.items = 0; + await Promise.all([dispatch('loadTickets'), dispatch('loadEventItems')]); + } } }, plugins: [persistentStatePlugin({ // TODO change remember to some kind of enable field diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index dd0b413..220ec3f 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -81,6 +81,13 @@ Copy DHL contact to clipboard +
+ + +
- + Save Comment @@ -25,7 +25,7 @@