From 0fa52645c2605be8afd3e63f92ccfb068e106131 Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 18 Jan 2025 22:15:04 +0100 Subject: [PATCH 01/21] handle plain base64 as transfer-encoding in incoming mails --- core/mail/protocol.py | 8 ++++++-- core/mail/tests/v2/test_mails.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 7fe6942..00872f0 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -128,10 +128,14 @@ def decode_email_segment(segment, charset, transfer_encoding): decode_as = 'cp1251' elif charset == 'iso-8859-1': decode_as = 'latin1' - segment = unescape_and_decode_quoted_printable(segment) - segment = unescape_and_decode_base64(segment) if transfer_encoding == 'quoted-printable': segment = unescape_simplified_quoted_printable(segment, decode_as) + elif transfer_encoding == 'base64': + import base64 + segment = base64.b64decode(segment).decode('utf-8') + else: + segment = unescape_and_decode_quoted_printable(segment) + segment = unescape_and_decode_base64(segment) return segment diff --git a/core/mail/tests/v2/test_mails.py b/core/mail/tests/v2/test_mails.py index 455faf1..d2f33fe 100644 --- a/core/mail/tests/v2/test_mails.py +++ b/core/mail/tests/v2/test_mails.py @@ -165,7 +165,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body) self.assertTrue(Email.objects.all()[0].raw_file.path) - def test_handle_base64(self): + def test_handle_base64_inline(self): from aiosmtpd.smtp import Envelope from asgiref.sync import async_to_sync import aiosmtplib @@ -186,6 +186,35 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual('Text mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) self.assertTrue(Email.objects.all()[0].raw_file.path) + def test_handle_base64_transfer_encoding(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'''Subject: test +From: test3@test +To: test4@test +Message-ID: <1@test> +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: base64 + +VGVzdCBtaXQgQmFzZTY0LUtvZGllcnVuZzogw6TDtsO8w58=''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 2) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('Test mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) + self.assertTrue(Email.objects.all()[0].raw_file.path) + def test_handle_client_reply(self): issue_thread = IssueThread.objects.create( name="test", From f133ae9e60450d3302a1c962c3e025c754ca52e7 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 17:39:13 +0100 Subject: [PATCH 02/21] allow searching while "all" event is selected --- web/src/store.js | 82 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/web/src/store.js b/web/src/store.js index b276086..e747ae7 100644 --- a/web/src/store.js +++ b/web/src/store.js @@ -77,10 +77,26 @@ const store = createStore({ getEventTickets: (state, getters) => getters.getEventSlug === 'all' ? getters.getAllTickets : getters.getAllTickets.filter(t => t.event === getters.getEventSlug || (t.event == null && getters.getEventSlug === 'none')), isItemsLoaded: (state, getters) => (getters.getEventSlug === 'all' || getters.getEventSlug === 'none') ? !!state.loadedItems : Object.keys(state.loadedItems).includes(getters.getEventSlug), isTicketsLoaded: (state, getters) => (getters.getEventSlug === 'all' || getters.getEventSlug === 'none') ? !!state.loadedTickets : Object.keys(state.loadedTickets).includes(getters.getEventSlug), - getItemsSearchResults: (state, getters) => state.loadedItemSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [], - getTicketsSearchResults: (state, getters) => state.loadedTicketSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [], - isItemsSearchLoaded: (state, getters) => Object.keys(state.loadedItemSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))), - isTicketsSearchLoaded: (state, getters) => Object.keys(state.loadedTicketSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))), + getItemsSearchResults: (state, getters) => { + if (getters.getEventSlug === 'all') { + return state.events.map(e => { + return state.loadedItemSearchResults[e.slug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + }).flat(); + } else { + return state.loadedItemSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + } + }, + getTicketsSearchResults: (state, getters) => { + if (getters.getEventSlug === 'all') { + return state.events.map(e => { + return state.loadedTicketSearchResults[e.slug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + }).flat(); + } else { + return state.loadedTicketSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + } + }, + isItemsSearchLoaded: (state, getters) => Object.keys(state.loadedItemSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))) || getters.getEventSlug === 'all', + isTicketsSearchLoaded: (state, getters) => Object.keys(state.loadedTicketSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))) || getters.getEventSlug === 'all', getActiveView: state => router.currentRoute.value.name || 'items', getFilters: state => router.currentRoute.value.query, getBoxes: state => state.loadedBoxes, @@ -379,26 +395,39 @@ const store = createStore({ }, async loadEventItems({commit, getters, state}) { if (!state.user.token) return; - if (state.fetchedData.items > Date.now() - 1000 * 60 * 60 * 24) return; - try { - const slug = getters.getEventSlug; - const {data, success} = await getters.session.get(`/2/${slug}/items/`); - if (data && success) { - commit('setItems', {slug, items: data}); + const load = async (slug) => { + try { + const {data, success} = await getters.session.get(`/2/${slug}/items/`); + if (data && success) { + commit('setItems', {slug, items: data}); + } + } catch (e) { + console.error("Error loading items"); } - } catch (e) { - console.error("Error loading items"); + } + const slug = getters.getEventSlug; + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); } }, async searchEventItems({commit, getters, state}, query) { const encoded_query = base64.encode(utf8.encode(query)); + const load = async (slug) => { + if (Object.keys(state.loadedItemSearchResults).includes(slug + '/' + encoded_query)) return; + const { + data, success + } = await getters.session.get(`/2/${slug}/items/${encoded_query}/`); + if (data && success) { + commit('setItemSearchResults', {slug, query: encoded_query, items: data}); + } + } const slug = getters.getEventSlug; - if (Object.keys(state.loadedItemSearchResults).includes(slug + '/' + encoded_query)) return; - const { - data, success - } = await getters.session.get(`/2/${slug}/items/${encoded_query}/`); - if (data && success) { - commit('setItemSearchResults', {slug, query: encoded_query, items: data}); + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); } }, async loadBoxes({commit, state, getters}) { @@ -446,12 +475,19 @@ const store = createStore({ }, async searchEventTickets({commit, getters, state}, query) { const encoded_query = base64.encode(utf8.encode(query)); + const load = async (slug) => { + if (Object.keys(state.loadedTicketSearchResults).includes(slug + '/' + encoded_query)) return; + const { + data, success + } = await getters.session.get(`/2/${slug}/tickets/${encoded_query}/`); + if (data && success) commit('setTicketSearchResults', {slug, query: encoded_query, items: data}); + } const slug = getters.getEventSlug; - if (Object.keys(state.loadedTicketSearchResults).includes(slug + '/' + encoded_query)) return; - const { - data, success - } = await getters.session.get(`/2/${slug}/tickets/${encoded_query}/`); - if (data && success) commit('setTicketSearchResults', {slug, query: encoded_query, items: data}); + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); + } }, async sendMail({commit, dispatch, state, getters}, {id, message}) { const {data, success} = await getters.session.post(`/2/tickets/${id}/reply/`, {message}, From 4ea74637a3d7f59accef14b4a2885c34edd3dfc5 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 18:30:42 +0100 Subject: [PATCH 03/21] finally get a grip on utf-8 --- core/mail/protocol.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 00872f0..3639989 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -53,6 +53,12 @@ def unescape_simplified_quoted_printable(s, encoding='utf-8'): return quopri.decodestring(s).decode(encoding) +def decode_inline_encodings(s): + s = unescape_and_decode_quoted_printable(s) + s = unescape_and_decode_base64(s) + return s + + def ascii_strip(s): if not s: return None @@ -134,8 +140,7 @@ def decode_email_segment(segment, charset, transfer_encoding): import base64 segment = base64.b64decode(segment).decode('utf-8') else: - segment = unescape_and_decode_quoted_printable(segment) - segment = unescape_and_decode_base64(segment) + segment = decode_inline_encodings(segment.decode('utf-8')) return segment @@ -160,7 +165,7 @@ def parse_email_body(raw, log=None): segment = part.get_payload() if not segment: continue - segment = decode_email_segment(segment, charset, part.get('Content-Transfer-Encoding')) + segment = decode_email_segment(segment.encode('utf-8'), charset, part.get('Content-Transfer-Encoding')) log.debug(segment) body = body + segment elif 'attachment' in cdispo or 'inline' in cdispo: @@ -193,7 +198,8 @@ def parse_email_body(raw, log=None): else: log.warning("Unknown content type %s", parsed.get_content_type()) body = "Unknown content type" - body = decode_email_segment(body, parsed.get_content_charset(), parsed.get('Content-Transfer-Encoding')) + body = decode_email_segment(body.encode('utf-8'), parsed.get_content_charset(), + parsed.get('Content-Transfer-Encoding')) log.debug(body) return parsed, body, attachments @@ -221,8 +227,9 @@ def receive_email(envelope, log=None): subject = ascii_strip(parsed.get('Subject')) if not subject: subject = "No subject" - subject = unescape_and_decode_quoted_printable(subject) - subject = unescape_and_decode_base64(subject) + subject = decode_inline_encodings(subject) + recipient = decode_inline_encodings(recipient) + sender = decode_inline_encodings(sender) target_event = find_target_event(recipient) active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event) @@ -256,7 +263,7 @@ do not create a new request. Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid()) reply_email = Email.objects.create( - sender=recipient, recipient=sender, body=body, subject=ascii_strip(subject), + sender=recipient, recipient=sender, body=body, subject=subject, in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) reply = make_reply(reply_email, references, event=target_event.slug if target_event else None) else: From fbbf8352cf9fdfe0af06c1b8df1ab92ae4832b50 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 19:43:01 +0100 Subject: [PATCH 04/21] don't report "Internal Server Error" if mail already exists --- core/mail/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 3639989..8fa2c3b 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -220,7 +220,7 @@ def receive_email(envelope, log=None): if Email.objects.filter(reference=header_message_id).exists(): # break before issue thread is created log.warning("Email already exists") - raise Exception("Email already exists") + raise SpecialMailException("Email already exists") recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower() sender = envelope.mail_from if envelope.mail_from else header_from From 2677f4b8b6c1d614e2bcb3ff3278eb5ea46e33c3 Mon Sep 17 00:00:00 2001 From: jedi Date: Sun, 26 Jan 2025 19:56:25 +0100 Subject: [PATCH 05/21] link item to ticket frontend --- web/src/store.js | 9 ++++++++- web/src/views/Ticket.vue | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/web/src/store.js b/web/src/store.js index e747ae7..34650e4 100644 --- a/web/src/store.js +++ b/web/src/store.js @@ -61,7 +61,6 @@ const store = createStore({ '2kg-de': '2kg Paket (DE)', '5kg-de': '5kg Paket (DE)', '10kg-de': '10kg Paket (DE)', - '2kg-eu': '2kg Paket (EU)', '5kg-eu': '5kg Paket (EU)', '10kg-eu': '10kg Paket (EU)', } @@ -564,6 +563,14 @@ const store = createStore({ state.fetchedData.tickets = 0; await Promise.all([dispatch('loadTickets'), dispatch('fetchShippingVouchers')]); } + }, + async linkTicketItem({dispatch, state, getters}, {ticket_id, item_id}) { + const {data, success} = await getters.session.post(`/2/matches/`, {issue_thread: ticket_id, item: item_id}); + if (data && success) { + state.fetchedData.tickets = 0; + state.fetchedData.items = 0; + await Promise.all([dispatch('loadTickets'), dispatch('loadEventItems')]); + } } }, plugins: [persistentStatePlugin({ // TODO change remember to some kind of enable field diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index dd0b413..220ec3f 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -81,6 +81,13 @@ Copy DHL contact to clipboard +
+ + +
- + Save Comment @@ -25,7 +25,7 @@