diff --git a/core/mail/protocol.py b/core/mail/protocol.py index c5aaa4a..7fe6942 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -48,9 +48,15 @@ def unescape_and_decode_base64(s): return decoded -def unescape_simplified_quoted_printable(s): +def unescape_simplified_quoted_printable(s, encoding='utf-8'): import quopri - return quopri.decodestring(s).decode('utf-8') + return quopri.decodestring(s).decode(encoding) + + +def ascii_strip(s): + if not s: + return None + return ''.join([c for c in str(s) if 128 > ord(c) > 31]) def collect_references(issue_thread): @@ -116,6 +122,19 @@ def find_target_event(address): return None +def decode_email_segment(segment, charset, transfer_encoding): + decode_as = 'utf-8' + if charset == 'windows-1251': + decode_as = 'cp1251' + elif charset == 'iso-8859-1': + decode_as = 'latin1' + segment = unescape_and_decode_quoted_printable(segment) + segment = unescape_and_decode_base64(segment) + if transfer_encoding == 'quoted-printable': + segment = unescape_simplified_quoted_printable(segment, decode_as) + return segment + + def parse_email_body(raw, log=None): import email from hashlib import sha256 @@ -127,21 +146,24 @@ def parse_email_body(raw, log=None): if parsed.is_multipart(): for part in parsed.walk(): ctype = part.get_content_type() + charset = part.get_content_charset() cdispo = str(part.get('Content-Disposition')) + if ctype == 'multipart/mixed': + log.debug("Ignoring Multipart %s %s", ctype, cdispo) # skip any text/plain (txt) attachments - if ctype == 'text/plain' and 'attachment' not in cdispo: + elif ctype == 'text/plain' and 'attachment' not in cdispo: segment = part.get_payload() if not segment: continue - segment = unescape_and_decode_quoted_printable(segment) - segment = unescape_and_decode_base64(segment) - if part.get('Content-Transfer-Encoding') == 'quoted-printable': - segment = unescape_simplified_quoted_printable(segment) + segment = decode_email_segment(segment, charset, part.get('Content-Transfer-Encoding')) log.debug(segment) body = body + segment elif 'attachment' in cdispo or 'inline' in cdispo: - file = ContentFile(part.get_payload(decode=True)) + content = part.get_payload(decode=True) + if content is None: + continue + file = ContentFile(content) chash = sha256(file.read()).hexdigest() name = part.get_filename() if name is None: @@ -167,10 +189,7 @@ def parse_email_body(raw, log=None): else: log.warning("Unknown content type %s", parsed.get_content_type()) body = "Unknown content type" - body = unescape_and_decode_quoted_printable(body) - body = unescape_and_decode_base64(body) - if parsed.get('Content-Transfer-Encoding') == 'quoted-printable': - body = unescape_simplified_quoted_printable(body) + body = decode_email_segment(body, parsed.get_content_charset(), parsed.get('Content-Transfer-Encoding')) log.debug(body) return parsed, body, attachments @@ -182,8 +201,8 @@ def receive_email(envelope, log=None): header_from = parsed.get('From') header_to = parsed.get('To') - header_in_reply_to = parsed.get('In-Reply-To') - header_message_id = parsed.get('Message-ID') + header_in_reply_to = ascii_strip(parsed.get('In-Reply-To')) + header_message_id = ascii_strip(parsed.get('Message-ID')) if match(r'^([a-zA-Z ]*<)?MAILER-DAEMON@', header_from) and envelope.mail_from.strip("<>") == "": log.warning("Ignoring mailer daemon") @@ -195,7 +214,7 @@ def receive_email(envelope, log=None): recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower() sender = envelope.mail_from if envelope.mail_from else header_from - subject = parsed.get('Subject') + subject = ascii_strip(parsed.get('Subject')) if not subject: subject = "No subject" subject = unescape_and_decode_quoted_printable(subject) @@ -209,7 +228,8 @@ def receive_email(envelope, log=None): email = Email.objects.create( sender=sender, recipient=recipient, body=body, subject=subject, reference=header_message_id, - in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), event=target_event, + in_reply_to=header_in_reply_to, raw_file=ContentFile(envelope.content, name=random_filename), + event=target_event, issue_thread=active_issue_thread) for attachment in attachments: email.attachments.add(attachment) @@ -232,7 +252,7 @@ do not create a new request. Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid()) reply_email = Email.objects.create( - sender=recipient, recipient=sender, body=body, subject=subject, + sender=recipient, recipient=sender, body=body, subject=ascii_strip(subject), in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) reply = make_reply(reply_email, references, event=target_event.slug if target_event else None) else: diff --git a/core/mail/tests/v2/test_mails.py b/core/mail/tests/v2/test_mails.py index 3b358ca..455faf1 100644 --- a/core/mail/tests/v2/test_mails.py +++ b/core/mail/tests/v2/test_mails.py @@ -142,7 +142,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('test ä', Email.objects.all()[0].subject) self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body) - self.assertTrue( Email.objects.all()[0].raw_file.path) + self.assertTrue(Email.objects.all()[0].raw_file.path) def test_handle_quoted_printable_2(self): from aiosmtpd.smtp import Envelope @@ -163,7 +163,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('suche_Mütze', Email.objects.all()[0].subject) self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body) - self.assertTrue( Email.objects.all()[0].raw_file.path) + self.assertTrue(Email.objects.all()[0].raw_file.path) def test_handle_base64(self): from aiosmtpd.smtp import Envelope @@ -184,7 +184,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test aiosmtplib.send.assert_called_once() self.assertEqual('test', Email.objects.all()[0].subject) self.assertEqual('Text mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) - self.assertTrue( Email.objects.all()[0].raw_file.path) + self.assertTrue(Email.objects.all()[0].raw_file.path) def test_handle_client_reply(self): issue_thread = IssueThread.objects.create( @@ -232,7 +232,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual(IssueThread.objects.all()[0].name, 'test') self.assertEqual(IssueThread.objects.all()[0].state, 'pending_new') self.assertEqual(IssueThread.objects.all()[0].assigned_to, None) - self.assertTrue( Email.objects.all()[2].raw_file.path) + self.assertTrue(Email.objects.all()[2].raw_file.path) def test_handle_client_reply_2(self): issue_thread = IssueThread.objects.create( @@ -285,7 +285,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual(IssueThread.objects.all()[0].name, 'test') self.assertEqual(IssueThread.objects.all()[0].state, 'pending_open') self.assertEqual(IssueThread.objects.all()[0].assigned_to, None) - self.assertTrue( Email.objects.all()[2].raw_file.path) + self.assertTrue(Email.objects.all()[2].raw_file.path) def test_mail_reply(self): issue_thread = IssueThread.objects.create( @@ -887,6 +887,59 @@ hello \xe4\xf6\xfc''' self.assertEqual(1, len(states)) self.assertEqual('pending_new', states[0].state) + def test_mail_windows_1252(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + + aiosmtplib.send = make_mocked_coro() + + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + + envelope.content = b'''Subject: test +From: test1@test +To: test2@test +Message-ID: <1@test> +Content-Type: text/html; charset=windows-1252 +Content-Transfer-Encoding: quoted-printable + +=0D=0Ahello=''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual('250 Message accepted for delivery', result) + self.assertEqual(2, len(Email.objects.all())) + self.assertEqual(1, len(IssueThread.objects.all())) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('\r\nhello', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('test1@test', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) + def test_mail_quoted_printable_transfer_encoding(self): from aiosmtpd.smtp import Envelope from asgiref.sync import async_to_sync @@ -939,3 +992,146 @@ hello =C3=A4=C3=B6=C3=BC''' states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) self.assertEqual(1, len(states)) self.assertEqual('pending_new', states[0].state) + + def test_text_with_attachment2(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'''Subject: test +From: test1@test +To: test2@test +Message-ID: <1@test> +Content-Type: multipart/mixed; boundary="abc" +Content-Disposition: inline +Content-Transfer-Encoding: 8bit + +--abc +Content-Type: text/plain; charset=utf-8 +Content-Disposition: inline +Content-Transfer-Encoding: 8bit + +test1 + +--abc +Content-Type: image/jpeg; name="test.jpg" +Content-Disposition: attachment; filename="test.jpg" +Content-Transfer-Encoding: base64 +Content-ID: <1> +X-Attachment-Id: 1 + +dGVzdGltYWdl + +--abc--''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 2) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('test1\n', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('test1@test', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) + self.assertEqual(1, len(EmailAttachment.objects.all())) + self.assertEqual(1, EmailAttachment.objects.all()[0].id) + self.assertEqual('image/jpeg', EmailAttachment.objects.all()[0].mime_type) + self.assertEqual('test.jpg', EmailAttachment.objects.all()[0].name) + file_content = EmailAttachment.objects.all()[0].file.read() + self.assertEqual(b'testimage', file_content) + + + def test_text_non_utf8_in_multipart(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + + aiosmtplib.send = make_mocked_coro() + + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + + envelope.content = b'''Subject: test +From: test1@test +To: test2@test +Message-ID: <1@test> +Content-Type: multipart/alternative; boundary="abc" + +--abc +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit + +test1 + +--abc +Content-Type: text/plain; charset=iso-8859-1 +Content-Transfer-Encoding: quoted-printable + +hello =E4 + +--abc +Content-Type: text/plain; charset=windows-1252 +Content-Transfer-Encoding: quoted-printable + +=0D=0Ahello + +--abc--''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 2) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('test1@test', Email.objects.all()[0].sender) + self.assertEqual('test2@test', Email.objects.all()[0].recipient) + self.assertEqual('test1\nhello ä\n\r\nhello\n', Email.objects.all()[0].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[0].issue_thread) + self.assertEqual('<1@test>', Email.objects.all()[0].reference) + self.assertEqual(None, Email.objects.all()[0].in_reply_to) + self.assertEqual(expected_auto_reply_subject.format('test', IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].subject) + self.assertEqual('test2@test', Email.objects.all()[1].sender) + self.assertEqual('test1@test', Email.objects.all()[1].recipient) + self.assertEqual(expected_auto_reply.format(IssueThread.objects.all()[0].short_uuid()), + Email.objects.all()[1].body) + self.assertEqual(IssueThread.objects.all()[0], Email.objects.all()[1].issue_thread) + self.assertTrue(Email.objects.all()[1].reference.startswith("<")) + self.assertTrue(Email.objects.all()[1].reference.endswith("@localhost>")) + self.assertEqual("<1@test>", Email.objects.all()[1].in_reply_to) + self.assertEqual('test', IssueThread.objects.all()[0].name) + self.assertEqual('pending_new', IssueThread.objects.all()[0].state) + self.assertEqual(None, IssueThread.objects.all()[0].assigned_to) + states = StateChange.objects.filter(issue_thread=IssueThread.objects.all()[0]) + self.assertEqual(1, len(states)) + self.assertEqual('pending_new', states[0].state) diff --git a/core/tickets/api_v2.py b/core/tickets/api_v2.py index 99dc008..1020c71 100644 --- a/core/tickets/api_v2.py +++ b/core/tickets/api_v2.py @@ -143,12 +143,36 @@ def add_comment(request, pk): def filter_issues(issues, query): - query_tokens = query.split(' ') + query_tokens = query.lower().split(' ') for issue in issues: value = 0 + if issue.short_uuid() in query: + value += 10 + if "T#" + str(issue.id) in query: + value += 10 + elif "#" + str(issue.id) in query: + value += 9 + for item in issue.related_items: + if "I#" + str(item.id) in query: + value += 8 + elif "#" + str(item.id) in query: + value += 5 + for token in query_tokens: + if token in item.description.lower(): + value += 1 for token in query_tokens: - if token in issue.description: + if token in issue.name.lower(): value += 1 + for comment in issue.comments.all(): + for token in query_tokens: + if token in comment.comment.lower(): + value += 1 + for email in issue.emails.all(): + for token in query_tokens: + if token in email.subject.lower(): + value += 1 + if token in email.body.lower(): + value += 1 if value > 0: yield {'search_score': value, 'issue': issue} @@ -160,7 +184,10 @@ def search_issues(request, event_slug, query): event = Event.objects.get(slug=event_slug) if not request.user.has_event_perm(event, 'view_issuethread'): return Response(status=403) - items = filter_issues(IssueThread.objects.filter(event=event), b64decode(query).decode('utf-8')) + serializer = IssueSerializer() + queryset = IssueThread.objects.filter(event=event) + items = filter_issues(queryset.prefetch_related(*serializer.Meta.prefetch_related_fields), + b64decode(query).decode('utf-8')) return Response(SearchResultSerializer(items, many=True).data) except Event.DoesNotExist: return Response(status=404) diff --git a/core/tickets/serializers.py b/core/tickets/serializers.py index 50cdb72..a236d61 100644 --- a/core/tickets/serializers.py +++ b/core/tickets/serializers.py @@ -139,10 +139,10 @@ class IssueSerializer(BasicIssueSerializer): class SearchResultSerializer(serializers.Serializer): search_score = serializers.IntegerField() - item = IssueSerializer() + issue = IssueSerializer() def to_representation(self, instance): - return {**IssueSerializer(instance['item']).data, 'search_score': instance['search_score']} + return {**IssueSerializer(instance['issue']).data, 'search_score': instance['search_score']} class Meta: model = IssueThread diff --git a/core/tickets/tests/v2/test_tickets.py b/core/tickets/tests/v2/test_tickets.py index 9720625..4e3d83b 100644 --- a/core/tickets/tests/v2/test_tickets.py +++ b/core/tickets/tests/v2/test_tickets.py @@ -383,15 +383,85 @@ class IssueSearchTest(TestCase): def setUp(self): super().setUp() - self.event = Event.objects.create(slug='EVENT', name='Event') self.user = ExtendedUser.objects.create_user('testuser', 'test', 'test') self.user.user_permissions.add(*Permission.objects.all()) self.user.save() + self.event = Event.objects.create(slug='EVENT', name='Event') + self.box = Container.objects.create(name='box1') + self.item = Item.objects.create(container=self.box, description="foo", event=self.event) self.token = AuthToken.objects.create(user=self.user) self.client = Client(headers={'Authorization': 'Token ' + self.token[1]}) - def test_search(self): + def test_search_empty_result(self): search_query = b64encode(b'abc').decode('utf-8') response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') self.assertEqual(200, response.status_code) self.assertEqual([], response.json()) + + def test_search(self): + now = datetime.now() + issue = IssueThread.objects.create( + name="test issue Abc", + event=self.event, + ) + mail1 = Email.objects.create( + subject='test', + body='test aBc', + sender='test', + recipient='test', + issue_thread=issue, + timestamp=now, + ) + mail2 = Email.objects.create( + subject='test', + body='test', + sender='test', + recipient='test', + issue_thread=issue, + in_reply_to=mail1.reference, + timestamp=now + timedelta(seconds=2), + ) + assignment = Assignment.objects.create( + issue_thread=issue, + assigned_to=self.user, + timestamp=now + timedelta(seconds=3), + ) + comment = Comment.objects.create( + issue_thread=issue, + comment="test deF", + timestamp=now + timedelta(seconds=4), + ) + match = ItemRelation.objects.create( + issue_thread=issue, + item=self.item, + timestamp=now + timedelta(seconds=5), + ) + search_query = b64encode(b'abC').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) + score2 = response.json()[0]['search_score'] + + search_query = b64encode(b'dEf').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) + score1 = response.json()[0]['search_score'] + + search_query = b64encode(b'ghi').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(0, len(response.json())) + + search_query = b64encode(b'Abc def').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/tickets/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(issue.id, response.json()[0]['id']) + score3 = response.json()[0]['search_score'] + + self.assertGreater(score3, score2) + self.assertGreater(score2, score1) + self.assertGreater(score1, 0) diff --git a/web/src/components/Navbar.vue b/web/src/components/Navbar.vue index ccfb0f0..7f5e257 100644 --- a/web/src/components/Navbar.vue +++ b/web/src/components/Navbar.vue @@ -115,10 +115,10 @@ export default { this.$router.push(link); }, isItemView() { - return this.getActiveView === 'items' || this.getActiveView === 'item'; + return this.getActiveView === 'items' || this.getActiveView === 'item' || this.getActiveView === 'item_search'; }, isTicketView() { - return this.getActiveView === 'tickets' || this.getActiveView === 'ticket'; + return this.getActiveView === 'tickets' || this.getActiveView === 'ticket' || this.getActiveView === 'ticket_search'; }, setLayout(layout) { if (this.route.query.layout === layout) diff --git a/web/src/components/inputs/SearchBox.vue b/web/src/components/inputs/SearchBox.vue index eb32b07..79fb798 100644 --- a/web/src/components/inputs/SearchBox.vue +++ b/web/src/components/inputs/SearchBox.vue @@ -12,6 +12,7 @@ + + \ No newline at end of file diff --git a/web/src/views/TicketSearch.vue b/web/src/views/TicketSearch.vue new file mode 100644 index 0000000..35e7d07 --- /dev/null +++ b/web/src/views/TicketSearch.vue @@ -0,0 +1,102 @@ + + + + + \ No newline at end of file