diff --git a/.forgejo/workflows/deploy_staging.yml b/.forgejo/workflows/deploy_staging.yml index 3b44d24..4c5fcd0 100644 --- a/.forgejo/workflows/deploy_staging.yml +++ b/.forgejo/workflows/deploy_staging.yml @@ -35,7 +35,7 @@ jobs: - name: Populate relevant files run: | - mkdir ~/.ssh + mkdir -p ~/.ssh echo "${{ secrets.C3LF_SSH_TESTING }}" > ~/.ssh/id_ed25519 chmod 0600 ~/.ssh/id_ed25519 ls -lah ~/.ssh @@ -43,7 +43,7 @@ jobs: eval $(ssh-agent -s) ssh-add ~/.ssh/id_ed25519 echo "andromeda.lab.or.it ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDXPoO0PE+B9PYwbGaLo98zhbmjAkp6eBtVeZe43v/+T" >> ~/.ssh/known_hosts - mkdir /etc/ansible + mkdir -p /etc/ansible echo "${{ secrets.C3LF_INVENTORY_TESTING }}" > /etc/ansible/hosts - name: Check ansible version diff --git a/core/core/metrics.py b/core/core/metrics.py new file mode 100644 index 0000000..149829c --- /dev/null +++ b/core/core/metrics.py @@ -0,0 +1,35 @@ +from django.apps import apps +from prometheus_client.core import CounterMetricFamily, REGISTRY +from django.db.models import Case, Value, When, BooleanField, Count +from inventory.models import Item + +class ItemCountCollector(object): + + def collect(self): + counter = CounterMetricFamily("item_count", "Current number of items", labels=['event', 'returned_state']) + + yield counter + + if not apps.models_ready or not apps.apps_ready: + return + + queryset = ( + Item.all_objects + .annotate( + returned=Case( + When(returned_at__isnull=True, then=Value(False)), + default=Value(True), + output_field=BooleanField() + ) + ) + .values('event__slug', 'returned', 'event_id') + .annotate(amount=Count('id')) + .order_by('event__slug', 'returned') # Optional: order by slug and returned + ) + + for e in queryset: + counter.add_metric([e["event__slug"].lower(), str(e["returned"])], e["amount"]) + + yield counter + +REGISTRY.register(ItemCountCollector()) \ No newline at end of file diff --git a/core/core/settings.py b/core/core/settings.py index 94f15eb..805a27b 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -49,10 +49,6 @@ SYSTEM3_VERSION = "0.0.0-dev.0" ACTIVE_SPAM_TRAINING = truthy_str(os.getenv('ACTIVE_SPAM_TRAINING', 'False')) -TELEGRAM_BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN', '1234567890:ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghi') - -TELEGRAM_GROUP_CHAT_ID = os.getenv('TELEGRAM_GROUP_CHAT_ID', '-1234567890') - # Application definition INSTALLED_APPS = [ @@ -69,7 +65,6 @@ INSTALLED_APPS = [ 'drf_yasg', 'channels', 'authentication', - 'notifications', 'files', 'tickets', 'inventory', @@ -129,19 +124,12 @@ TEMPLATES = [ }, ] -WSGI_APPLICATION = 'core.wsgi.application' +ASGI_APPLICATION = 'core.asgi.application' # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases -if 'test' in sys.argv: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', - } - } -else: +if os.getenv('DB_HOST') is not None: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', @@ -154,6 +142,20 @@ else: 'charset': 'utf8mb4', 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'" } + }, + } +elif os.getenv('DB_FILE') is not None: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.getenv('DB_FILE', 'local.db'), + } + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', } } diff --git a/core/core/urls.py b/core/core/urls.py index d224e52..2386891 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -19,6 +19,8 @@ from django.urls import path, include from .version import get_info +from .metrics import * + urlpatterns = [ path('djangoadmin/', admin.site.urls), path('api/2/', include('inventory.api_v2')), @@ -28,7 +30,6 @@ urlpatterns = [ path('api/2/', include('mail.api_v2')), path('api/2/', include('notify_sessions.api_v2')), path('api/2/', include('authentication.api_v2')), - path('api/2/', include('notifications.api_v2')), path('api/', get_info), path('', include('django_prometheus.urls')), ] diff --git a/core/inventory/api_v2.py b/core/inventory/api_v2.py index 326c049..04c1722 100644 --- a/core/inventory/api_v2.py +++ b/core/inventory/api_v2.py @@ -39,13 +39,61 @@ class ItemViewSet(viewsets.ModelViewSet): def filter_items(items, query): query_tokens = query.split(' ') + matches = [] for item in items: value = 0 + if "I#" + str(item.id) in query: + value += 12 + matches.append( + {'type': 'item_id', 'text': f'is exactly {item.id} and matched "I#{item.id}"'}) + elif "#" + str(item.id) in query: + value += 11 + matches.append( + {'type': 'item_id', 'text': f'is exactly {item.id} and matched "#{item.id}"'}) + elif str(item.id) in query: + value += 10 + matches.append({'type': 'item_id', 'text': f'is exactly {item.id}'}) + for issue in item.related_issues: + if "T#" + issue.short_uuid() in query: + value += 8 + matches.append({'type': 'ticket_uuid', + 'text': f'is exactly {issue.short_uuid()} and matched "T#{issue.short_uuid()}"'}) + elif "#" + issue.short_uuid() in query: + value += 5 + matches.append({'type': 'ticket_uuid', + 'text': f'is exactly {issue.short_uuid()} and matched "#{issue.short_uuid()}"'}) + elif issue.short_uuid() in query: + value += 3 + matches.append({'type': 'ticket_uuid', 'text': f'is exactly {issue.short_uuid()}'}) + if "T#" + str(issue.id) in query: + value += 8 + matches.append({'type': 'ticket_id', 'text': f'is exactly {issue.id} and matched "T#{issue.id}"'}) + elif "#" + str(issue.id) in query: + value += 5 + matches.append({'type': 'ticket_id', 'text': f'is exactly {issue.id} and matched "#{issue.id}"'}) + elif str(issue.id) in query: + value += 3 + matches.append({'type': 'ticket_id', 'text': f'is exactly {issue.id}'}) + for comment in issue.comments.all(): + for token in query_tokens: + if token in comment.comment: + value += 1 + matches.append({'type': 'ticket_comment', 'text': f'contains {token}'}) + for token in query_tokens: + if token in issue.name: + value += 1 + matches.append({'type': 'ticket_name', 'text': f'contains {token}'}) for token in query_tokens: if token in item.description: value += 1 + matches.append({'type': 'item_description', 'text': f'contains {token}'}) + for comment in item.comments.all(): + for token in query_tokens: + if token in comment.comment: + value += 1 + matches.append({'type': 'comment', 'text': f'contains {token}'}) if value > 0: - yield {'search_score': value, 'item': item} + yield {'search_score': value, 'item': item, 'search_matches': matches} @api_view(['GET']) diff --git a/core/inventory/serializers.py b/core/inventory/serializers.py index 26a5be4..2aa1135 100644 --- a/core/inventory/serializers.py +++ b/core/inventory/serializers.py @@ -137,10 +137,12 @@ class ItemSerializer(BasicItemSerializer): class SearchResultSerializer(serializers.Serializer): search_score = serializers.IntegerField() + search_matches = serializers.ListField(child=serializers.DictField()) item = ItemSerializer() def to_representation(self, instance): - return {**ItemSerializer(instance['item']).data, 'search_score': instance['search_score']} + return {**ItemSerializer(instance['item']).data, 'search_score': instance['search_score'], + 'search_matches': instance['search_matches']} class Meta: model = Item diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 9c4d19c..258334c 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -7,7 +7,6 @@ from channels.db import database_sync_to_async from django.core.files.base import ContentFile from mail.models import Email, EventAddress, EmailAttachment -from notifications.templates import render_auto_reply from notify_sessions.models import SystemEvent from tickets.models import IssueThread @@ -49,9 +48,21 @@ def unescape_and_decode_base64(s): return decoded -def unescape_simplified_quoted_printable(s): +def unescape_simplified_quoted_printable(s, encoding='utf-8'): import quopri - return quopri.decodestring(s).decode('utf-8') + return quopri.decodestring(s).decode(encoding) + + +def decode_inline_encodings(s): + s = unescape_and_decode_quoted_printable(s) + s = unescape_and_decode_base64(s) + return s + + +def ascii_strip(s): + if not s: + return None + return ''.join([c for c in str(s) if 128 > ord(c) > 31]) def collect_references(issue_thread): @@ -88,27 +99,11 @@ def make_reply(reply_email, references=None, event=None): return reply -def make_notification(message, to, title): # TODO where should replies to this go - from email.message import EmailMessage - from core.settings import MAIL_DOMAIN - notification = EmailMessage() - notification["From"] = "notifications@%s" % MAIL_DOMAIN - notification["To"] = to - notification["Subject"] = f"[C3LF Notification]%s" % title - # notification["Reply-To"] = f"{event}@{MAIL_DOMAIN}" - # notification["In-Reply-To"] = email.reference - # notification["Message-ID"] = email.id + "@" + MAIL_DOMAIN - - notification.set_content(message) - - return notification - - async def send_smtp(message): await aiosmtplib.send(message, hostname="127.0.0.1", port=25, use_tls=False, start_tls=False) -def find_active_issue_thread(in_reply_to, address, subject, event): +def find_active_issue_thread(in_reply_to, address, subject, event, spam=False): from re import match uuid_match = match(r'^ticket\+([a-f0-9-]{36})@', address) if uuid_match: @@ -119,7 +114,8 @@ def find_active_issue_thread(in_reply_to, address, subject, event): if reply_to.exists(): return reply_to.first().issue_thread, False else: - issue = IssueThread.objects.create(name=subject, event=event) + issue = IssueThread.objects.create(name=subject, event=event, + initial_state='pending_suspected_spam' if spam else 'pending_new') return issue, True @@ -133,6 +129,22 @@ def find_target_event(address): return None +def decode_email_segment(segment, charset, transfer_encoding): + decode_as = 'utf-8' + if charset == 'windows-1251': + decode_as = 'cp1251' + elif charset == 'iso-8859-1': + decode_as = 'latin1' + if transfer_encoding == 'quoted-printable': + segment = unescape_simplified_quoted_printable(segment, decode_as) + elif transfer_encoding == 'base64': + import base64 + segment = base64.b64decode(segment).decode('utf-8') + else: + segment = decode_inline_encodings(segment.decode('utf-8')) + return segment + + def parse_email_body(raw, log=None): import email from hashlib import sha256 @@ -144,21 +156,24 @@ def parse_email_body(raw, log=None): if parsed.is_multipart(): for part in parsed.walk(): ctype = part.get_content_type() + charset = part.get_content_charset() cdispo = str(part.get('Content-Disposition')) + if ctype == 'multipart/mixed': + log.debug("Ignoring Multipart %s %s", ctype, cdispo) # skip any text/plain (txt) attachments - if ctype == 'text/plain' and 'attachment' not in cdispo: + elif ctype == 'text/plain' and 'attachment' not in cdispo: segment = part.get_payload() if not segment: continue - segment = unescape_and_decode_quoted_printable(segment) - segment = unescape_and_decode_base64(segment) - if part.get('Content-Transfer-Encoding') == 'quoted-printable': - segment = unescape_simplified_quoted_printable(segment) + segment = decode_email_segment(segment.encode('utf-8'), charset, part.get('Content-Transfer-Encoding')) log.debug(segment) body = body + segment elif 'attachment' in cdispo or 'inline' in cdispo: - file = ContentFile(part.get_payload(decode=True)) + content = part.get_payload(decode=True) + if content is None: + continue + file = ContentFile(content) chash = sha256(file.read()).hexdigest() name = part.get_filename() if name is None: @@ -184,10 +199,8 @@ def parse_email_body(raw, log=None): else: log.warning("Unknown content type %s", parsed.get_content_type()) body = "Unknown content type" - body = unescape_and_decode_quoted_printable(body) - body = unescape_and_decode_base64(body) - if parsed.get('Content-Transfer-Encoding') == 'quoted-printable': - body = unescape_simplified_quoted_printable(body) + body = decode_email_segment(body.encode('utf-8'), parsed.get_content_charset(), + parsed.get('Content-Transfer-Encoding')) log.debug(body) return parsed, body, attachments @@ -199,8 +212,10 @@ def receive_email(envelope, log=None): header_from = parsed.get('From') header_to = parsed.get('To') - header_in_reply_to = parsed.get('In-Reply-To') - header_message_id = parsed.get('Message-ID') + header_in_reply_to = ascii_strip(parsed.get('In-Reply-To')) + header_message_id = ascii_strip(parsed.get('Message-ID')) + maybe_spam = parsed.get('X-Spam') + suspected_spam = (maybe_spam and maybe_spam.lower() == 'yes') if match(r'^([a-zA-Z ]*<)?MAILER-DAEMON@', header_from) and envelope.mail_from.strip("<>") == "": log.warning("Ignoring mailer daemon") @@ -208,25 +223,28 @@ def receive_email(envelope, log=None): if Email.objects.filter(reference=header_message_id).exists(): # break before issue thread is created log.warning("Email already exists") - raise Exception("Email already exists") + raise SpecialMailException("Email already exists") recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower() sender = envelope.mail_from if envelope.mail_from else header_from - subject = parsed.get('Subject') + subject = ascii_strip(parsed.get('Subject')) if not subject: subject = "No subject" - subject = unescape_and_decode_quoted_printable(subject) - subject = unescape_and_decode_base64(subject) + subject = decode_inline_encodings(subject) + recipient = decode_inline_encodings(recipient) + sender = decode_inline_encodings(sender) target_event = find_target_event(recipient) - active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event) + active_issue_thread, new = find_active_issue_thread( + header_in_reply_to, recipient, subject, target_event, suspected_spam) from hashlib import sha256 random_filename = 'mail-' + sha256(envelope.content).hexdigest() email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), + event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) @@ -236,9 +254,18 @@ def receive_email(envelope, log=None): if new: # auto reply if new issue references = collect_references(active_issue_thread) - if not sender.startswith('noreply'): + if not sender.startswith('noreply') and not sender.startswith('no-reply') and not suspected_spam: subject = f"Re: {subject} [#{active_issue_thread.short_uuid()}]" - body = render_auto_reply(active_issue_thread) + body = '''Your request (#{}) has been received and will be reviewed by our lost&found angels. + +We are reviewing incoming requests during the event and teardown. Immediately after the event, expect a delay as the \ +workload is high. We will not forget about your request and get back in touch once we have updated information on your \ +request. Requests for devices, wallets, credit cards or similar items will be handled with priority. + +If you happen to find your lost item or just want to add additional information, please reply to this email. Please \ +do not create a new request. + +Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid()) reply_email = Email.objects.create( sender=recipient, recipient=sender, body=body, subject=subject, in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) @@ -281,19 +308,11 @@ class LMTPHandler: 'general', {"type": "generic.event", "name": "send_message_to_frontend", "event_id": systemevent.id, "message": "email received"}) log.info(f"Sent message to frontend") - if new and reply: log.info('Sending message to %s' % reply['To']) await send_smtp(reply) log.info("Sent auto reply") - if thread: - await channel_layer.group_send( - 'general', {"type": "generic.event", "name": "user_notification", "event_id": systemevent.id, - "ticket_id": thread.id, "new": new}) - else: - print("No thread found") - return '250 Message accepted for delivery' except SpecialMailException as e: import uuid diff --git a/core/mail/tests/v2/test_mails.py b/core/mail/tests/v2/test_mails.py index 3b358ca..95d35cb 100644 --- a/core/mail/tests/v2/test_mails.py +++ b/core/mail/tests/v2/test_mails.py @@ -142,7 +142,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('test ä', Email.objects.all()[0].subject) self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body) - self.assertTrue( Email.objects.all()[0].raw_file.path) + self.assertTrue(Email.objects.all()[0].raw_file.path) def test_handle_quoted_printable_2(self): from aiosmtpd.smtp import Envelope @@ -163,9 +163,9 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('suche_Mütze', Email.objects.all()[0].subject) self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body) - self.assertTrue( Email.objects.all()[0].raw_file.path) + self.assertTrue(Email.objects.all()[0].raw_file.path) - def test_handle_base64(self): + def test_handle_base64_inline(self): from aiosmtpd.smtp import Envelope from asgiref.sync import async_to_sync import aiosmtplib @@ -184,7 +184,36 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('test', Email.objects.all()[0].subject) self.assertEqual('Text mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) - self.assertTrue( Email.objects.all()[0].raw_file.path) + self.assertTrue(Email.objects.all()[0].raw_file.path) + + def test_handle_base64_transfer_encoding(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'''Subject: test +From: test3@test +To: test4@test +Message-ID: <1@test> +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: base64 + +VGVzdCBtaXQgQmFzZTY0LUtvZGllcnVuZzogw6TDtsO8w58=''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 2) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('Test mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) + self.assertTrue(Email.objects.all()[0].raw_file.path) def test_handle_client_reply(self): issue_thread = IssueThread.objects.create( @@ -232,7 +261,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual(IssueThread.objects.all()[0].name, 'test') self.assertEqual(IssueThread.objects.all()[0].state, 'pending_new') self.assertEqual(IssueThread.objects.all()[0].assigned_to, None) - self.assertTrue( Email.objects.all()[2].raw_file.path) + self.assertTrue(Email.objects.all()[2].raw_file.path) def test_handle_client_reply_2(self): issue_thread = IssueThread.objects.create( @@ -285,7 +314,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual(IssueThread.objects.all()[0].name, 'test') self.assertEqual(IssueThread.objects.all()[0].state, 'pending_open') self.assertEqual(IssueThread.objects.all()[0].assigned_to, None) - self.assertTrue( Email.objects.all()[2].raw_file.path) + self.assertTrue(Email.objects.all()[2].raw_file.path) def test_mail_reply(self): issue_thread = IssueThread.objects.create( @@ -783,6 +812,44 @@ dGVzdGltYWdl self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) aiosmtplib.send.assert_called_once() + def test_mail_spam_header(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'''Subject: test +From: test1@test +To: test2@test +Message-ID: <1@test> +X-Spam: Yes + +test''' + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 1) # do not send auto reply if spam is suspected + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_not_called() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('test', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_suspected_spam', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_suspected_spam', states[0].state) + def test_mail_4byte_unicode_emoji(self): from aiosmtpd.smtp import Envelope from asgiref.sync import async_to_sync @@ -887,6 +954,59 @@ hello \xe4\xf6\xfc''' self.assertEqual(1, len(states)) self.assertEqual('pending_new', states[0].state) + def test_mail_windows_1252(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + + aiosmtplib.send = make_mocked_coro() + + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + + envelope.content = b'''Subject: test +From: test1@test +To: test2@test +Message-ID: <1@test> +Content-Type: text/html; charset=windows-1252 +Content-Transfer-Encoding: quoted-printable + +=0D=0Ahello=''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual('250 Message accepted for delivery', result) + self.assertEqual(2, len(Email.objects.all())) + self.assertEqual(1, len(IssueThread.objects.all())) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('\r\nhello', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('test1@test', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) + def test_mail_quoted_printable_transfer_encoding(self): from aiosmtpd.smtp import Envelope from asgiref.sync import async_to_sync @@ -939,3 +1059,146 @@ hello =C3=A4=C3=B6=C3=BC''' states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) self.assertEqual(1, len(states)) self.assertEqual('pending_new', states[0].state) + + def test_text_with_attachment2(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'''Subject: test +From: test1@test +To: test2@test +Message-ID: <1@test> +Content-Type: multipart/mixed; boundary="abc" +Content-Disposition: inline +Content-Transfer-Encoding: 8bit + +--abc +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: 8bit + +test1 + +--abc +Content-Type: image/jpeg; name="test.jpg" +Content-Disposition: attachment; filename="test.jpg" +Content-Transfer-Encoding: base64 +Content-ID: <1> +X-Attachment-Id: 1 + +dGVzdGltYWdl + +--abc--''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 2) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('test1\n', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('test1@test', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) + self.assertEqual(1, len(EmailAttachment.objects.all())) + self.assertEqual(1, EmailAttachment.objects.all()[0].id) + self.assertEqual('image/jpeg', EmailAttachment.objects.all()[0].mime_type) + self.assertEqual('test.jpg', EmailAttachment.objects.all()[0].name) + file_content = EmailAttachment.objects.all()[0].file.read() + self.assertEqual(b'testimage', file_content) + + + def test_text_non_utf8_in_multipart(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + + aiosmtplib.send = make_mocked_coro() + + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + + envelope.content = b'''Subject: test +From: test1@test +To: test2@test +Message-ID: <1@test> +Content-Type: multipart/alternative; boundary="abc" + +--abc +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit + +test1 + +--abc +Content-Type: text/plain; charset=iso-8859-1 +Content-Transfer-Encoding: quoted-printable + +hello =E4 + +--abc +Content-Type: text/plain; charset=windows-1252 +Content-Transfer-Encoding: quoted-printable + +=0D=0Ahello + +--abc--''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 2) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('test1\nhello ä\n\r\nhello\n', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('test1@test', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) diff --git a/core/mail/tests/v2/test_user_notifications.py b/core/mail/tests/v2/test_user_notifications.py deleted file mode 100644 index c4db23f..0000000 --- a/core/mail/tests/v2/test_user_notifications.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.contrib.auth.models import Permission -from django.test import TestCase - -from authentication.models import ExtendedUser -from notifications.models import UserNotificationChannel - - -class UserNotificationTestCase(TestCase): - - def setUp(self): - super().setUp() - self.user = ExtendedUser.objects.create_user('testuser', 'test', 'test') - self.user.user_permissions.add(*Permission.objects.all()) - self.user.save() - self.channel = UserNotificationChannel.objects.create(user=self.user, channel_type='telegram', - channel_target='123456789', - event_filter='*', active=True) - - async def test_telegram_notify(self): - pass diff --git a/core/notifications/admin.py b/core/notifications/admin.py deleted file mode 100644 index 69620a9..0000000 --- a/core/notifications/admin.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.contrib import admin - -from notifications.models import MessageTemplate, UserNotificationChannel - - -class MessageTemplateAdmin(admin.ModelAdmin): - pass - - -class UserNotificationChannelAdmin(admin.ModelAdmin): - pass - - -admin.site.register(MessageTemplate, MessageTemplateAdmin) -admin.site.register(UserNotificationChannel, UserNotificationChannelAdmin) diff --git a/core/notifications/api_v2.py b/core/notifications/api_v2.py deleted file mode 100644 index a9492f5..0000000 --- a/core/notifications/api_v2.py +++ /dev/null @@ -1,51 +0,0 @@ -from django.contrib.auth.decorators import permission_required -from rest_framework import routers, viewsets -from django.urls import re_path -from rest_framework.decorators import api_view, permission_classes -from rest_framework.permissions import IsAuthenticated -from rest_framework.response import Response - -from notifications.models import MessageTemplate, UserNotificationChannel -from rest_framework import serializers - -from notifications.templates import TEMPLATE_VARS -from authentication.serializers import UserSerializer - - -class MessageTemplateSerializer(serializers.ModelSerializer): - class Meta: - model = MessageTemplate - fields = '__all__' - - -class UserNotificationChannelSerializer(serializers.ModelSerializer): - user = UserSerializer() - - class Meta: - model = UserNotificationChannel - fields = '__all__' - - -class MessageTemplateViewSet(viewsets.ModelViewSet): - serializer_class = MessageTemplateSerializer - queryset = MessageTemplate.objects.all() - - -class UserNotificationChannelViewSet(viewsets.ModelViewSet): - serializer_class = UserNotificationChannelSerializer - queryset = UserNotificationChannel.objects.all() - - -@api_view(['GET']) -@permission_classes([IsAuthenticated]) -@permission_required('tickets.add_issuethread_manual', raise_exception=True) # TDOO: change this permission -def get_template_vars(self): - return Response(TEMPLATE_VARS, status=200) - - -router = routers.SimpleRouter() -router.register(r'message_templates', MessageTemplateViewSet) -router.register(r'user_notification_channels', UserNotificationChannelViewSet) -urlpatterns = ([ - re_path('message_template_variables', get_template_vars), - ] + router.urls) diff --git a/core/notifications/defaults.py b/core/notifications/defaults.py deleted file mode 100644 index 812d93e..0000000 --- a/core/notifications/defaults.py +++ /dev/null @@ -1,16 +0,0 @@ -auto_reply_body = '''Your request (#{{ ticket_uuid }}) has been received and will be reviewed by our lost&found angels. - -We are reviewing incoming requests during the event and teardown. Immediately after the event, expect a delay as the \ -workload is high. We will not forget about your request and get back in touch once we have updated information on your \ -request. Requests for devices, wallets, credit cards or similar items will be handled with priority. - -If you happen to find your lost item or just want to add additional information, please reply to this email. Please \ -do not create a new request. - -Your c3lf (Cloakroom + Lost&Found) Team''' - -new_issue_notification = '''New issue "{{ ticket_name | limit_length }}" [{{ ticket_uuid }}] created -{{ ticket_url }}''' - -reply_issue_notification = '''Reply to issue "{{ ticket_name }}" [{{ ticket_uuid }}] (was {{ previous_state_pretty }}) -{{ ticket_url }}''' diff --git a/core/notifications/dispatch.py b/core/notifications/dispatch.py deleted file mode 100644 index 752c342..0000000 --- a/core/notifications/dispatch.py +++ /dev/null @@ -1,85 +0,0 @@ -import asyncio - -from aiohttp.client import ClientSession -from channels.layers import get_channel_layer -from channels.db import database_sync_to_async -from urllib.parse import quote as urlencode - -from core.settings import TELEGRAM_BOT_TOKEN, TELEGRAM_GROUP_CHAT_ID -from mail.protocol import send_smtp, make_notification -from notifications.models import UserNotificationChannel -from notifications.templates import render_notification_new_ticket_async, render_notification_reply_ticket_async -from tickets.models import IssueThread - - -async def http_get(url): - async with ClientSession() as session: - async with session.get(url) as response: - return await response.text() - - -async def telegram_notify(message, chat_id): - encoded_message = urlencode(message) - url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage?chat_id={chat_id}&text={encoded_message}" - return await http_get(url) - - -async def email_notify(message, title, email): - mail = make_notification(message, email, title) - await send_smtp(mail) - - -class NotificationDispatcher: - channel_layer = None - room_group_name = "general" - - def __init__(self): - self.channel_layer = get_channel_layer('default') - if not self.channel_layer: - raise Exception("Could not get channel layer") - - @database_sync_to_async - def get_notification_targets(self): - channels = UserNotificationChannel.objects.filter(active=True) - return list(channels) - - @database_sync_to_async - def get_ticket(self, ticket_id): - return IssueThread.objects.filter(id=ticket_id).select_related('event').first() - - async def run_forever(self): - # Infinite loop to continuously listen for messages - print("Listening for messages...") - channel_name = await self.channel_layer.new_channel() - await self.channel_layer.group_add(self.room_group_name, channel_name) - print("Channel name:", channel_name) - while True: - # Blocking receive to get the message from the channel layer - message = await self.channel_layer.receive(channel_name) - - if (message and 'type' in message and message['type'] == 'generic.event' and 'name' in message and - message['name'] == 'user_notification'): - if 'ticket_id' in message and 'event_id' in message and 'new' in message: - ticket = await self.get_ticket(message['ticket_id']) - await self.dispatch(ticket, message['event_id'], message['new']) - else: - print("Error: Invalid message format") - - async def dispatch(self, ticket, event_id, new): - message = await render_notification_new_ticket_async( - ticket) if new else await render_notification_reply_ticket_async(ticket) - title = f"[#{ticket.short_uuid()}] {ticket.name}" - print("Dispatching message:", message, "with event_id:", event_id) - targets = await self.get_notification_targets() - jobs = [] - jobs.append(telegram_notify(message, TELEGRAM_GROUP_CHAT_ID)) - for target in targets: - if target.channel_type == 'telegram': - print("Sending telegram notification to:", target.channel_target) - jobs.append(telegram_notify(message, target.channel_target)) - elif target.channel_type == 'email': - print("Sending email notification to:", target.channel_target) - jobs.append(email_notify(message, title, target.channel_target)) - else: - print("Unknown channel type:", target.channel_type) - await asyncio.gather(*jobs) diff --git a/core/notifications/migrations/0001_initial.py b/core/notifications/migrations/0001_initial.py deleted file mode 100644 index 4d276eb..0000000 --- a/core/notifications/migrations/0001_initial.py +++ /dev/null @@ -1,51 +0,0 @@ -# Generated by Django 4.2.7 on 2024-05-03 21:02 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - -from notifications.defaults import auto_reply_body, new_issue_notification, reply_issue_notification -from notifications.models import MessageTemplate - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - def create_required_templates(apps, schema_editor): - MessageTemplate.objects.create(name='auto_reply', message=auto_reply_body, marked_required=True) - MessageTemplate.objects.create(name='new_issue_notification', message=new_issue_notification, - marked_required=True) - MessageTemplate.objects.create(name='reply_issue_notification', message=reply_issue_notification, - marked_required=True) - - operations = [ - migrations.CreateModel( - name='MessageTemplate', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=255)), - ('message', models.TextField()), - ('created', models.DateTimeField(auto_now_add=True)), - ('marked_confidential', models.BooleanField(default=False)), - ('marked_required', models.BooleanField(default=False)), - ], - ), - migrations.CreateModel( - name='UserNotificationChannel', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('channel_type', - models.CharField(choices=[('telegram', 'telegram'), ('email', 'email')], max_length=255)), - ('channel_target', models.CharField(max_length=255)), - ('event_filter', models.CharField(max_length=255)), - ('active', models.BooleanField(default=True)), - ('created', models.DateTimeField(auto_now_add=True)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - ), - migrations.RunPython(create_required_templates), - ] diff --git a/core/notifications/migrations/__init__.py b/core/notifications/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/core/notifications/models.py b/core/notifications/models.py deleted file mode 100644 index 9cbfbe5..0000000 --- a/core/notifications/models.py +++ /dev/null @@ -1,29 +0,0 @@ -from django.db import models - -from authentication.models import ExtendedUser - - -class MessageTemplate(models.Model): - name = models.CharField(max_length=255) - message = models.TextField() - created = models.DateTimeField(auto_now_add=True) - marked_confidential = models.BooleanField(default=False) - marked_required = models.BooleanField(default=False) # may not be deleted - - def __str__(self): - return self.name - - -class UserNotificationChannel(models.Model): - user = models.ForeignKey(ExtendedUser, models.CASCADE) - channel_type = models.CharField(choices=[('telegram', 'telegram'), ('email', 'email')], max_length=255) - channel_target = models.CharField(max_length=255) - event_filter = models.CharField(max_length=255) - active = models.BooleanField(default=True) - created = models.DateTimeField(auto_now_add=True) - - def validate_constraints(self, exclude=None): # TODO: email -> emailaddress, telegram -> chatid - return True - - def __str__(self): - return self.user.username + '(' + self.channel_type + ')' diff --git a/core/notifications/templates.py b/core/notifications/templates.py deleted file mode 100644 index af77193..0000000 --- a/core/notifications/templates.py +++ /dev/null @@ -1,69 +0,0 @@ -import jinja2 -from channels.db import database_sync_to_async -from core.settings import PRIMARY_HOST - -from notifications.models import MessageTemplate - -TEMPLATE_VARS = ['ticket_name', 'ticket_uuid', 'ticket_id', 'ticket_url', - 'current_state', 'previous_state', 'current_state_pretty', 'previous_state_pretty', - 'event_slug', 'event_name', - 'username', 'user_nick', - 'web_host'] # TODO customer_name, tracking_code - - -def limit_length(s, length=50): - if len(s) > length: - return s[:(length - 3)] + "..." - return s - - -def ticket_url(ticket): - eventslug = ticket.event.slug if ticket.event else "37C3" # TODO 37C3 should not be hardcoded - return f"https://{PRIMARY_HOST}/{eventslug}/ticket/{ticket.id}/" - - -def render_template(template, **kwargs): - try: - environment = jinja2.Environment() - environment.filters['limit_length'] = limit_length - tmpl = MessageTemplate.objects.get(name=template) - template = environment.from_string(tmpl.message) - return template.render(**kwargs, web_host=PRIMARY_HOST) - except MessageTemplate.DoesNotExist: - return None - - -def get_ticket_vars(ticket): - states = list(ticket.state_changes.order_by('-timestamp')) - return { - 'ticket_name': ticket.name, - 'ticket_uuid': ticket.short_uuid(), - 'ticket_id': ticket.id, - 'ticket_url': ticket_url(ticket), - 'current_state': states[0].state if states else 'none', - 'previous_state': states[1].state if len(states) > 1 else 'none', - 'current_state_pretty': states[0].get_state_display() if states else 'none', - 'previous_state_pretty': states[1].get_state_display() if len(states) > 1 else 'none', - 'event_slug': ticket.event.slug if ticket.event else "37C3", # TODO 37C3 should not be hardcoded - 'event_name': ticket.event.name if ticket.event else "37C3", - } - - -def render_auto_reply(ticket): - return render_template('auto_reply', **get_ticket_vars(ticket)) - - -def render_notification_new_ticket(ticket): - return render_template('new_issue_notification', **get_ticket_vars(ticket)) - - -def render_notification_reply_ticket(ticket): - return render_template('reply_issue_notification', **get_ticket_vars(ticket)) - - -async def render_notification_new_ticket_async(ticket): - return await database_sync_to_async(render_notification_new_ticket)(ticket) - - -async def render_notification_reply_ticket_async(ticket): - return await database_sync_to_async(render_notification_reply_ticket)(ticket) diff --git a/core/notifications/tests/__init__.py b/core/notifications/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/core/server.py b/core/server.py index a09e315..d08b595 100644 --- a/core/server.py +++ b/core/server.py @@ -12,7 +12,6 @@ django.setup() from helper import init_loop from mail.protocol import LMTPHandler from mail.socket import UnixSocketLMTPController -from notifications.dispatch import NotificationDispatcher class UvicornServer(uvicorn.Server): @@ -55,11 +54,6 @@ async def lmtp(loop): log.info("LMTP done") -async def notifications(loop): - dispatcher = NotificationDispatcher() - await dispatcher.run_forever() - - def main(): import sdnotify import setproctitle @@ -73,7 +67,6 @@ def main(): loop.create_task(web(loop)) # loop.create_task(tcp(loop)) loop.create_task(lmtp(loop)) - loop.create_task(notifications(loop)) n = sdnotify.SystemdNotifier() n.notify("READY=1") log.info("Server ready") diff --git a/core/tickets/api_v2.py b/core/tickets/api_v2.py index 99dc008..d5fd5fe 100644 --- a/core/tickets/api_v2.py +++ b/core/tickets/api_v2.py @@ -143,14 +143,71 @@ def add_comment(request, pk): def filter_issues(issues, query): - query_tokens = query.split(' ') + query_tokens = query.lower().split(' ') + matches = [] for issue in issues: value = 0 + if "T#" + issue.short_uuid() in query: + value += 12 + matches.append( + {'type': 'ticket_uuid', 'text': f'is exactly {issue.short_uuid()} and matched "T#{issue.short_uuid()}"'}) + elif "#" + issue.short_uuid() in query: + value += 11 + matches.append( + {'type': 'ticket_uuid', 'text': f'is exactly {issue.short_uuid()} and matched "#{issue.short_uuid()}"'}) + elif issue.short_uuid() in query: + value += 10 + matches.append({'type': 'ticket_uuid', 'text': f'is exactly {issue.short_uuid()}'}) + if "T#" + str(issue.id) in query: + value += 10 + matches.append({'type': 'ticket_id', 'text': f'is exactly {issue.id} and matched "T#{issue.id}"'}) + elif "#" + str(issue.id) in query: + value += 7 + matches.append({'type': 'ticket_id', 'text': f'is exactly {issue.id} and matched "#{issue.id}"'}) + elif str(issue.id) in query: + value += 4 + matches.append({'type': 'ticket_id', 'text': f'is exactly {issue.id}'}) + for item in issue.related_items: + if "I#" + str(item.id) in query: + value += 8 + matches.append({'type': 'item_id', 'text': f'is exactly {item.id} and matched "I#{item.id}"'}) + elif "#" + str(item.id) in query: + value += 5 + matches.append({'type': 'item_id', 'text': f'is exactly {item.id} and matched "#{item.id}"'}) + elif str(item.id) in query: + value += 3 + matches.append({'type': 'item_id', 'text': f'is exactly {item.id}'}) + for token in query_tokens: + if token in item.description.lower(): + value += 1 + matches.append({'type': 'item_description', 'text': f'contains {token}'}) + for comment in item.comments.all(): + for token in query_tokens: + if token in comment.comment.lower(): + value += 1 + matches.append({'type': 'item_comment', 'text': f'contains {token}'}) for token in query_tokens: - if token in issue.description: + if token in issue.name.lower(): value += 1 + matches.append({'type': 'ticket_name', 'text': f'contains {token}'}) + for comment in issue.comments.all(): + for token in query_tokens: + if token in comment.comment.lower(): + value += 1 + matches.append({'type': 'ticket_comment', 'text': f'contains {token}'}) + for email in issue.emails.all(): + for token in query_tokens: + if token in email.subject.lower(): + value += 1 + matches.append({'type': 'email_subject', 'text': f'contains {token}'}) + if token in email.body.lower(): + value += 1 + matches.append({'type': 'email_body', 'text': f'contains {token}'}) + if token in email.sender.lower(): + value += 1 + matches.append({'type': 'email_sender', 'text': f'contains {token}'}) if value > 0: - yield {'search_score': value, 'issue': issue} + yield {'search_score': value, 'issue': issue, 'search_matches': matches} @api_view(['GET']) @@ -160,7 +217,10 @@ def search_issues(request, event_slug, query): event = Event.objects.get(slug=event_slug) if not request.user.has_event_perm(event, 'view_issuethread'): return Response(status=403) - items = filter_issues(IssueThread.objects.filter(event=event), b64decode(query).decode('utf-8')) + serializer = IssueSerializer() + queryset = IssueThread.objects.filter(event=event) + items = filter_issues(queryset.prefetch_related(*serializer.Meta.prefetch_related_fields), + b64decode(query).decode('utf-8')) return Response(SearchResultSerializer(items, many=True).data) except Event.DoesNotExist: return Response(status=404) diff --git a/core/tickets/models.py b/core/tickets/models.py index f7ddb7b..794c8e4 100644 --- a/core/tickets/models.py +++ b/core/tickets/models.py @@ -16,6 +16,7 @@ STATE_CHOICES = ( ('pending_physical_confirmation', 'Needs to be confirmed physically'), ('pending_return', 'Needs to be returned'), ('pending_postponed', 'Postponed'), + ('pending_suspected_spam', 'Suspected Spam'), ('waiting_details', 'Waiting for details'), ('waiting_pre_shipping', 'Waiting for Address/Shipping Info'), ('closed_returned', 'Closed: Returned'), @@ -46,6 +47,11 @@ class IssueThread(SoftDeleteModel): event = models.ForeignKey(Event, null=True, on_delete=models.SET_NULL, related_name='issue_threads') manually_created = models.BooleanField(default=False) + def __init__(self, *args, **kwargs): + if 'initial_state' in kwargs: + self._initial_state = kwargs.pop('initial_state') + super().__init__(*args, **kwargs) + def short_uuid(self): return self.uuid[:8] @@ -110,8 +116,9 @@ def set_uuid(sender, instance, **kwargs): @receiver(post_save, sender=IssueThread) def create_issue_thread(sender, instance, created, **kwargs): - if created: - StateChange.objects.create(issue_thread=instance, state='pending_new') + if created and instance.state_changes.count() == 0: + initial_state = getattr(instance, '_initial_state', None) + StateChange.objects.create(issue_thread=instance, state=initial_state if initial_state else 'pending_new') class Comment(models.Model): diff --git a/core/tickets/serializers.py b/core/tickets/serializers.py index 50cdb72..ff695b1 100644 --- a/core/tickets/serializers.py +++ b/core/tickets/serializers.py @@ -139,10 +139,12 @@ class IssueSerializer(BasicIssueSerializer): class SearchResultSerializer(serializers.Serializer): search_score = serializers.IntegerField() - item = IssueSerializer() + search_matches = serializers.ListField(child=serializers.DictField()) + issue = IssueSerializer() def to_representation(self, instance): - return {**IssueSerializer(instance['item']).data, 'search_score': instance['search_score']} + return {**IssueSerializer(instance['issue']).data, 'search_score': instance['search_score'], + 'search_matches': instance['search_matches']} class Meta: model = IssueThread diff --git a/core/tickets/shared_serializers.py b/core/tickets/shared_serializers.py index ac16d81..3d46013 100644 --- a/core/tickets/shared_serializers.py +++ b/core/tickets/shared_serializers.py @@ -9,6 +9,7 @@ class RelationSerializer(serializers.ModelSerializer): class Meta: model = ItemRelation fields = ('id', 'status', 'timestamp', 'item', 'issue_thread') + read_only_fields = ('id', 'timestamp') class BasicIssueSerializer(serializers.ModelSerializer): diff --git a/core/tickets/tests/v2/test_tickets.py b/core/tickets/tests/v2/test_tickets.py index 9720625..d7bb346 100644 --- a/core/tickets/tests/v2/test_tickets.py +++ b/core/tickets/tests/v2/test_tickets.py @@ -4,6 +4,7 @@ from django.test import TestCase, Client from authentication.models import ExtendedUser from inventory.models import Event, Container, Item +from inventory.models import Comment as ItemComment from mail.models import Email, EmailAttachment from tickets.models import IssueThread, StateChange, Comment, ItemRelation, Assignment from django.contrib.auth.models import Permission @@ -383,15 +384,108 @@ class IssueSearchTest(TestCase): def setUp(self): super().setUp() - self.event = Event.objects.create(slug='EVENT', name='Event') self.user = ExtendedUser.objects.create_user('testuser', 'test', 'test') self.user.user_permissions.add(*Permission.objects.all()) self.user.save() + self.event = Event.objects.create(slug='EVENT', name='Event') + self.box = Container.objects.create(name='box1') + self.item = Item.objects.create(container=self.box, description="foo", event=self.event) self.token = AuthToken.objects.create(user=self.user) self.client = Client(headers={'Authorization': 'Token ' + self.token[1]}) - def test_search(self): + def test_search_empty_result(self): search_query = b64encode(b'abc').decode('utf-8') response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') self.assertEqual(200, response.status_code) self.assertEqual([], response.json()) + + def test_search(self): + now = datetime.now() + issue = IssueThread.objects.create( + name="test issue Abc", + event=self.event, + ) + mail1 = Email.objects.create( + subject='test', + body='test aBc', + sender='bar@test', + recipient='2@test', + issue_thread=issue, + timestamp=now, + ) + mail2 = Email.objects.create( + subject='Re: test', + body='test', + sender='2@test', + recipient='1@test', + issue_thread=issue, + in_reply_to=mail1.reference, + timestamp=now + timedelta(seconds=2), + ) + assignment = Assignment.objects.create( + issue_thread=issue, + assigned_to=self.user, + timestamp=now + timedelta(seconds=3), + ) + comment = Comment.objects.create( + issue_thread=issue, + comment="test deF", + timestamp=now + timedelta(seconds=4), + ) + match = ItemRelation.objects.create( + issue_thread=issue, + item=self.item, + timestamp=now + timedelta(seconds=5), + ) + item_comment = ItemComment.objects.create( + item=self.item, + comment="baz", + timestamp=now + timedelta(seconds=6), + ) + search_query = b64encode(b'abC').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) + score2 = response.json()[0]['search_score'] + + search_query = b64encode(b'dEf').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) + score1 = response.json()[0]['search_score'] + + search_query = b64encode(b'ghi').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(0, len(response.json())) + + search_query = b64encode(b'Abc def').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) + score3 = response.json()[0]['search_score'] + + self.assertGreater(score3, score2) + self.assertGreater(score2, score1) + self.assertGreater(score1, 0) + + search_query = b64encode(b'foo').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) + + search_query = b64encode(b'bar').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) + + search_query = b64encode(b'baz').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) diff --git a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml index 544b4e4..4005146 100644 --- a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml +++ b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml @@ -345,6 +345,13 @@ notify: - restart postfix + - name: configure rspamd dkim + template: + src: templates/rspamd-dkim.cf.j2 + dest: /etc/rspamd/local.d/dkim_signing.conf + notify: + - restart rspamd + - name: configure rspamd copy: content: | diff --git a/deploy/ansible/playbooks/templates/django.env.j2 b/deploy/ansible/playbooks/templates/django.env.j2 index 748ecf4..c9b1c83 100644 --- a/deploy/ansible/playbooks/templates/django.env.j2 +++ b/deploy/ansible/playbooks/templates/django.env.j2 @@ -13,5 +13,3 @@ STATIC_ROOT=/var/www/c3lf-sys3/staticfiles ACTIVE_SPAM_TRAINING=True DEBUG_MODE_ACTIVE={{ debug_mode_active }} DJANGO_SECRET_KEY={{ django_secret_key }} -TELEGRAM_GROUP_CHAT_ID={{ telegram_group_chat_id }} -TELEGRAM_BOT_TOKEN={{ telegram_bot_token }} \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/postfix.cf.j2 b/deploy/ansible/playbooks/templates/postfix.cf.j2 index f80d69b..f6e0b09 100644 --- a/deploy/ansible/playbooks/templates/postfix.cf.j2 +++ b/deploy/ansible/playbooks/templates/postfix.cf.j2 @@ -32,12 +32,11 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -myhostname = polaris.c3lf.de +myhostname = polaris.lab.or.it alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases myorigin = /etc/mailname mydestination = $myhostname, , localhost -relayhost = firefly.lab.or.it mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_size_limit = 0 recipient_delimiter = + diff --git a/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 b/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 new file mode 100644 index 0000000..9e21aa5 --- /dev/null +++ b/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 @@ -0,0 +1,79 @@ +# local.d/dkim_signing.conf + +enabled = true; + +# If false, messages with empty envelope from are not signed +allow_envfrom_empty = true; + +# If true, envelope/header domain mismatch is ignored +allow_hdrfrom_mismatch = false; + +# If true, multiple from headers are allowed (but only first is used) +allow_hdrfrom_multiple = false; + +# If true, username does not need to contain matching domain +allow_username_mismatch = false; + +# Default path to key, can include '$domain' and '$selector' variables +path = "/var/lib/rspamd/dkim/$domain.$selector.key"; + +# Default selector to use +selector = "dkim"; + +# If false, messages from authenticated users are not selected for signing +sign_authenticated = true; + +# If false, messages from local networks are not selected for signing +sign_local = true; + +# Map file of IP addresses/subnets to consider for signing +# sign_networks = "/some/file"; # or url + +# Symbol to add when message is signed +symbol = "DKIM_SIGNED"; + +# Whether to fallback to global config +try_fallback = true; + +# Domain to use for DKIM signing: can be "header" (MIME From), "envelope" (SMTP From), "recipient" (SMTP To), "auth" (SMTP username) or directly specified domain name +use_domain = "header"; + +# Domain to use for DKIM signing when sender is in sign_networks ("header"/"envelope"/"auth") +#use_domain_sign_networks = "header"; + +# Domain to use for DKIM signing when sender is a local IP ("header"/"envelope"/"auth") +#use_domain_sign_local = "header"; + +# Whether to normalise domains to eSLD +use_esld = true; + +# Whether to get keys from Redis +use_redis = false; + +# Hash for DKIM keys in Redis +key_prefix = "DKIM_KEYS"; + +# map of domains -> names of selectors (since rspamd 1.5.3) +#selector_map = "/etc/rspamd/dkim_selectors.map"; + +# map of domains -> paths to keys (since rspamd 1.5.3) +#path_map = "/etc/rspamd/dkim_paths.map"; + +# If `true` get pubkey from DNS record and check if it matches private key +check_pubkey = false; +# Set to `false` if you want to skip signing if public and private keys mismatch +allow_pubkey_mismatch = true; + +# Domain specific settings +domain { + # Domain name is used as key + c3lf.de { + + # Private key path + path = "/var/lib/rspamd/dkim/{{ mail_domain }}.key"; + + # Selector + selector = "{{ mail_domain }}"; + } +} + diff --git a/deploy/dev/docker-compose.yml b/deploy/dev/docker-compose.yml index dff5ab3..e44c276 100644 --- a/deploy/dev/docker-compose.yml +++ b/deploy/dev/docker-compose.yml @@ -3,20 +3,15 @@ services: build: context: ../../core dockerfile: ../deploy/dev/Dockerfile.backend - command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000' + command: bash -c 'python manage.py migrate && python testdata.py && python manage.py runserver 0.0.0.0:8000' environment: - HTTP_HOST=core - - DB_HOST=db - - DB_PORT=3306 - - DB_NAME=system3 - - DB_USER=system3 - - DB_PASSWORD=system3 + - DB_FILE=dev.db volumes: - ../../core:/code + - ../testdata.py:/code/testdata.py ports: - "8000:8000" - depends_on: - - db frontend: build: @@ -31,18 +26,3 @@ services: - "8080:8080" depends_on: - core - - db: - image: mariadb - environment: - MARIADB_RANDOM_ROOT_PASSWORD: true - MARIADB_DATABASE: system3 - MARIADB_USER: system3 - MARIADB_PASSWORD: system3 - volumes: - - mariadb_data:/var/lib/mysql - ports: - - "3306:3306" - -volumes: - mariadb_data: \ No newline at end of file diff --git a/deploy/testdata.py b/deploy/testdata.py new file mode 100644 index 0000000..dca385f --- /dev/null +++ b/deploy/testdata.py @@ -0,0 +1,88 @@ +import os + + +def setup(): + from authentication.models import ExtendedUser, EventPermission + from inventory.models import Event + from django.contrib.auth.models import Permission, Group + permissions = ['add_item', 'view_item', 'view_file', 'delete_item', 'change_item'] + if not ExtendedUser.objects.filter(username='admin').exists(): + admin = ExtendedUser.objects.create_superuser('admin', 'admin@example.com', 'admin') + admin.set_password('admin') + admin.user_permissions.add(*Permission.objects.all()) + admin.save() + + if not ExtendedUser.objects.filter(username='testuser').exists(): + testuser = ExtendedUser.objects.create_user('testuser', 'testuser@example.com', 'testuser') + testuser.set_password('testuser') + testuser.user_permissions.add(*Permission.objects.all()) + testuser.save() + + team = Group.objects.get(name='Team') + team.permissions.add( + *Permission.objects.all() + ) + + if not ExtendedUser.objects.filter(username='testuser2').exists(): + testuser2 = ExtendedUser.objects.create_user('testuser2', 'testuser2@example.com', 'testuser2') + testuser2.set_password('testuser2') + testuser2.groups.add(team) + testuser2.save() + + event1 = Event.objects.get_or_create(id=1, name='first test event', slug='TEST1', + start='2023-12-18 00:00:00.000000', end='2023-12-27 00:00:00.000000', + pre_start='2023-12-31 00:00:00.000000', post_end='2024-01-04 00:00:00.000000')[ + 0] + + event2 = Event.objects.get_or_create(id=2, name='second test event', slug='TEST2', + start='2024-12-18 00:00:00.000000', end='2024-12-27 00:00:00.000000', + pre_start='2024-12-31 00:00:00.000000', post_end='2025-01-04 00:00:00.000000')[ + 0] + + # for permission in permissions: + # EventPermission.objects.create(event=event_37c3, user=foo, + # permission=Permission.objects.get(codename=permission)) + + from tickets.models import IssueThread + + from mail.models import Email + + issue_thread = IssueThread.objects.get_or_create( + id=1, + name="test", + event=Event.objects.get(slug='TEST1') + )[0] + mail1 = Email.objects.get_or_create( + id=1, + subject='test subject', + body='test', + sender='test1@test', + recipient='test2@test', + issue_thread=issue_thread, + )[0] + mail1_reply = Email.objects.get_or_create( + id=2, + subject='Message received', + body='Thank you for your message.', + sender='test2@test', + recipient='test1@test', + in_reply_to=mail1.reference, + issue_thread=issue_thread, + )[0] + + +def main(): + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") + import django + + django.setup() + + from django.core.management import call_command + call_command('migrate') + + setup() + print('testdata initialised') + + +if __name__ == '__main__': + main() diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml index e93e901..b41dd63 100644 --- a/deploy/testing/docker-compose.yml +++ b/deploy/testing/docker-compose.yml @@ -20,7 +20,7 @@ services: build: context: ../../core dockerfile: ../deploy/testing/Dockerfile.backend - command: bash -c 'python manage.py migrate && python /code/server.py' + command: bash -c 'python manage.py migrate && python testdata.py && python /code/server.py' environment: - HTTP_HOST=core - REDIS_HOST=redis @@ -29,13 +29,16 @@ services: - DB_NAME=system3 - DB_USER=system3 - DB_PASSWORD=system3 + - MAIL_DOMAIN=mail:1025 volumes: - ../../core:/code + - ../testdata.py:/code/testdata.py ports: - "8000:8000" depends_on: - db - redis + - mail frontend: build: @@ -51,5 +54,19 @@ services: depends_on: - core + mail: + image: docker.io/axllent/mailpit + volumes: + - mailpit_data:/data + ports: + - 8025:8025 + - 1025:1025 + environment: + MP_MAX_MESSAGES: 5000 + MP_DATABASE: /data/mailpit.db + MP_SMTP_AUTH_ACCEPT_ANY: 1 + MP_SMTP_AUTH_ALLOW_INSECURE: 1 + volumes: - mariadb_data: \ No newline at end of file + mariadb_data: + mailpit_data: diff --git a/core/notifications/__init__.py b/web/node_modules/.forgit_fordocker similarity index 100% rename from core/notifications/__init__.py rename to web/node_modules/.forgit_fordocker diff --git a/web/src/components/AddItemModal.vue b/web/src/components/AddItemModal.vue index a3c23fd..24bd449 100644 --- a/web/src/components/AddItemModal.vue +++ b/web/src/components/AddItemModal.vue @@ -2,7 +2,29 @@
diff --git a/web/src/components/AddTicketModal.vue b/web/src/components/AddTicketModal.vue index 37d539c..b407670 100644 --- a/web/src/components/AddTicketModal.vue +++ b/web/src/components/AddTicketModal.vue @@ -19,11 +19,10 @@ diff --git a/web/src/components/Navbar.vue b/web/src/components/Navbar.vue index ccfb0f0..7f5e257 100644 --- a/web/src/components/Navbar.vue +++ b/web/src/components/Navbar.vue @@ -115,10 +115,10 @@ export default { this.$router.push(link); }, isItemView() { - return this.getActiveView === 'items' || this.getActiveView === 'item'; + return this.getActiveView === 'items' || this.getActiveView === 'item' || this.getActiveView === 'item_search'; }, isTicketView() { - return this.getActiveView === 'tickets' || this.getActiveView === 'ticket'; + return this.getActiveView === 'tickets' || this.getActiveView === 'ticket' || this.getActiveView === 'ticket_search'; }, setLayout(layout) { if (this.route.query.layout === layout) diff --git a/web/src/components/inputs/FormatedText.vue b/web/src/components/inputs/FormatedText.vue deleted file mode 100644 index 114fbf3..0000000 --- a/web/src/components/inputs/FormatedText.vue +++ /dev/null @@ -1,112 +0,0 @@ - - - \ No newline at end of file diff --git a/web/src/components/inputs/InputCombo.vue b/web/src/components/inputs/InputCombo.vue index fc64d42..2a291e0 100644 --- a/web/src/components/inputs/InputCombo.vue +++ b/web/src/components/inputs/InputCombo.vue @@ -43,11 +43,11 @@ export default { props: ['label', 'model', 'nameKey', 'uniqueKey', 'options', 'onOptionAdd'], data: ({options, model, nameKey, uniqueKey}) => ({ internalName: model[nameKey], - selectedOption: options.filter(e => e[uniqueKey] == model[uniqueKey])[0], + selectedOption: options.filter(e => e[uniqueKey] === model[uniqueKey])[0], addingOption: false }), computed: { - isValid: ({options, nameKey, internalName}) => options.some(e => e[nameKey] == internalName), + isValid: ({options, nameKey, internalName}) => options.some(e => e[nameKey] === internalName), sortedOptions: ({ options, nameKey @@ -56,7 +56,7 @@ export default { watch: { internalName(newValue) { if (this.isValid) { - if (!this.selectedOption || newValue != this.selectedOption[this.nameKey]) { + if (!this.selectedOption || newValue !== this.selectedOption[this.nameKey]) { this.selectedOption = this.options.filter(e => e[this.nameKey] === newValue)[0]; } this.model[this.nameKey] = this.selectedOption[this.nameKey]; diff --git a/web/src/components/inputs/SearchBox.vue b/web/src/components/inputs/SearchBox.vue index eb32b07..79fb798 100644 --- a/web/src/components/inputs/SearchBox.vue +++ b/web/src/components/inputs/SearchBox.vue @@ -12,6 +12,7 @@ + + \ No newline at end of file diff --git a/web/src/views/Items.vue b/web/src/views/Items.vue index 21b11d1..72b4079 100644 --- a/web/src/views/Items.vue +++ b/web/src/views/Items.vue @@ -67,11 +67,10 @@ + + \ No newline at end of file diff --git a/web/src/views/Tickets.vue b/web/src/views/Tickets.vue index 069ce0e..60d0d1d 100644 --- a/web/src/views/Tickets.vue +++ b/web/src/views/Tickets.vue @@ -26,7 +26,7 @@ :columns="['id', 'name', 'last_activity', 'assigned_to', ...(getEventSlug==='all'?['event']:[])]" :keyName="'state'" :sections="['pending_new', 'pending_open','pending_shipping', - 'pending_physical_confirmation','pending_return','pending_postponed'].map(stateInfo)"> + 'pending_physical_confirmation','pending_return','pending_postponed','pending_suspected_spam'].map(stateInfo)"> diff --git a/web/src/views/admin/Admin.vue b/web/src/views/admin/Admin.vue index b118ae5..5a71223 100644 --- a/web/src/views/admin/Admin.vue +++ b/web/src/views/admin/Admin.vue @@ -8,15 +8,9 @@
  • Dashboard
    • -
  • - Settings -
  • Events
    • -
  • - Notifications -
  • Shipping
    • diff --git a/web/src/views/admin/Events.vue b/web/src/views/admin/Events.vue index e2ff952..e156d56 100644 --- a/web/src/views/admin/Events.vue +++ b/web/src/views/admin/Events.vue @@ -1,6 +1,7 @@