From 374abe885b58d2a222f6808505f8fda6da2b2f8c Mon Sep 17 00:00:00 2001 From: jedi Date: Wed, 13 Nov 2024 19:05:56 +0100 Subject: [PATCH 01/41] also deploy live branch automatically --- .forgejo/workflows/live.yml | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .forgejo/workflows/live.yml diff --git a/.forgejo/workflows/live.yml b/.forgejo/workflows/live.yml new file mode 100644 index 0000000..4d8a94a --- /dev/null +++ b/.forgejo/workflows/live.yml @@ -0,0 +1,60 @@ +on: + push: + branches: + - live + +jobs: + test: + runs-on: docker + container: + image: ghcr.io/catthehacker/ubuntu:act-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache-dependency-path: '**/requirements.dev.txt' + - name: Install dependencies + working-directory: core + run: pip3 install -r requirements.dev.txt + - name: Run django tests + working-directory: core + run: python3 manage.py test + + deploy: + needs: [test] + runs-on: docker + steps: + - uses: actions/checkout@v4 + - name: Install ansible + run: | + apt update -y + apt install python3-pip -y + python3 -m pip install ansible + python3 -m pip install ansible-lint + + - name: Populate relevant files + run: | + mkdir ~/.ssh + echo "${{ secrets.C3LF_SSH_LIVE }}" > ~/.ssh/id_ed25519 + chmod 0600 ~/.ssh/id_ed25519 + ls -lah ~/.ssh + command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client -y ) + eval $(ssh-agent -s) + ssh-add ~/.ssh/id_ed25519 + echo "polaris.lab.or.it ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIO//BP4rx4Aav0voFicFSoYEG4VPfrbcWwd2JFfItRZT" >> ~/.ssh/known_hosts + mkdir /etc/ansible + echo "${{ secrets.C3LF_INVENTORY_LIVE }}" > /etc/ansible/hosts + + - name: Check ansible version + run: | + ansible --version + + - name: List ansible hosts + run: | + ansible -m ping Polaris + + - name: Deploy testing + run: | + cd deploy/ansible + ansible-playbook playbooks/deploy-c3lf-sys3.yml From 9141752fdfcb39094293c255062c63be42200751 Mon Sep 17 00:00:00 2001 From: jedi Date: Tue, 17 Dec 2024 21:52:57 +0100 Subject: [PATCH 02/41] fix container selection in AddItemModal.vue --- web/src/components/AddItemModal.vue | 55 ++++++++++++++++++------ web/src/components/AddTicketModal.vue | 3 +- web/src/components/EditItem.vue | 50 --------------------- web/src/components/inputs/InputCombo.vue | 6 +-- web/src/views/Item.vue | 2 - web/src/views/Items.vue | 5 +-- 6 files changed, 48 insertions(+), 73 deletions(-) delete mode 100644 web/src/components/EditItem.vue diff --git a/web/src/components/AddItemModal.vue b/web/src/components/AddItemModal.vue index a3c23fd..24bd449 100644 --- a/web/src/components/AddItemModal.vue +++ b/web/src/components/AddItemModal.vue @@ -2,7 +2,29 @@
diff --git a/web/src/components/AddTicketModal.vue b/web/src/components/AddTicketModal.vue index 37d539c..b407670 100644 --- a/web/src/components/AddTicketModal.vue +++ b/web/src/components/AddTicketModal.vue @@ -19,11 +19,10 @@ diff --git a/web/src/components/inputs/InputCombo.vue b/web/src/components/inputs/InputCombo.vue index fc64d42..2a291e0 100644 --- a/web/src/components/inputs/InputCombo.vue +++ b/web/src/components/inputs/InputCombo.vue @@ -43,11 +43,11 @@ export default { props: ['label', 'model', 'nameKey', 'uniqueKey', 'options', 'onOptionAdd'], data: ({options, model, nameKey, uniqueKey}) => ({ internalName: model[nameKey], - selectedOption: options.filter(e => e[uniqueKey] == model[uniqueKey])[0], + selectedOption: options.filter(e => e[uniqueKey] === model[uniqueKey])[0], addingOption: false }), computed: { - isValid: ({options, nameKey, internalName}) => options.some(e => e[nameKey] == internalName), + isValid: ({options, nameKey, internalName}) => options.some(e => e[nameKey] === internalName), sortedOptions: ({ options, nameKey @@ -56,7 +56,7 @@ export default { watch: { internalName(newValue) { if (this.isValid) { - if (!this.selectedOption || newValue != this.selectedOption[this.nameKey]) { + if (!this.selectedOption || newValue !== this.selectedOption[this.nameKey]) { this.selectedOption = this.options.filter(e => e[this.nameKey] === newValue)[0]; } this.model[this.nameKey] = this.selectedOption[this.nameKey]; diff --git a/web/src/views/Item.vue b/web/src/views/Item.vue index d00b8b1..7b9f484 100644 --- a/web/src/views/Item.vue +++ b/web/src/views/Item.vue @@ -112,14 +112,12 @@ import InputString from "@/components/inputs/InputString.vue"; import AuthenticatedImage from "@/components/AuthenticatedImage.vue"; import InputPhoto from "@/components/inputs/InputPhoto.vue"; import Modal from "@/components/Modal.vue"; -import EditItem from "@/components/EditItem.vue"; import AsyncButton from "@/components/inputs/AsyncButton.vue"; export default { name: 'Item', components: { AsyncButton, - EditItem, Modal, InputPhoto, AuthenticatedImage, InputString, InputCombo, AsyncLoader, ClipboardButton, Timeline }, data() { diff --git a/web/src/views/Items.vue b/web/src/views/Items.vue index 21b11d1..30e553f 100644 --- a/web/src/views/Items.vue +++ b/web/src/views/Items.vue @@ -67,11 +67,10 @@ + + \ No newline at end of file diff --git a/web/src/views/TicketSearch.vue b/web/src/views/TicketSearch.vue new file mode 100644 index 0000000..35e7d07 --- /dev/null +++ b/web/src/views/TicketSearch.vue @@ -0,0 +1,102 @@ + + + + + \ No newline at end of file From 0fa52645c2605be8afd3e63f92ccfb068e106131 Mon Sep 17 00:00:00 2001 From: jedi Date: Sat, 18 Jan 2025 22:15:04 +0100 Subject: [PATCH 21/41] handle plain base64 as transfer-encoding in incoming mails --- core/mail/protocol.py | 8 ++++++-- core/mail/tests/v2/test_mails.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 7fe6942..00872f0 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -128,10 +128,14 @@ def decode_email_segment(segment, charset, transfer_encoding): decode_as = 'cp1251' elif charset == 'iso-8859-1': decode_as = 'latin1' - segment = unescape_and_decode_quoted_printable(segment) - segment = unescape_and_decode_base64(segment) if transfer_encoding == 'quoted-printable': segment = unescape_simplified_quoted_printable(segment, decode_as) + elif transfer_encoding == 'base64': + import base64 + segment = base64.b64decode(segment).decode('utf-8') + else: + segment = unescape_and_decode_quoted_printable(segment) + segment = unescape_and_decode_base64(segment) return segment diff --git a/core/mail/tests/v2/test_mails.py b/core/mail/tests/v2/test_mails.py index 455faf1..d2f33fe 100644 --- a/core/mail/tests/v2/test_mails.py +++ b/core/mail/tests/v2/test_mails.py @@ -165,7 +165,7 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual('Text mit Quoted-Printable-Kodierung: äöüß', Email.objects.all()[0].body) self.assertTrue(Email.objects.all()[0].raw_file.path) - def test_handle_base64(self): + def test_handle_base64_inline(self): from aiosmtpd.smtp import Envelope from asgiref.sync import async_to_sync import aiosmtplib @@ -186,6 +186,35 @@ class LMTPHandlerTestCase(TestCase): # TODO replace with less hacky test self.assertEqual('Text mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) self.assertTrue(Email.objects.all()[0].raw_file.path) + def test_handle_base64_transfer_encoding(self): + from aiosmtpd.smtp import Envelope + from asgiref.sync import async_to_sync + import aiosmtplib + aiosmtplib.send = make_mocked_coro() + handler = LMTPHandler() + server = mock.Mock() + session = mock.Mock() + envelope = Envelope() + envelope.mail_from = 'test1@test' + envelope.rcpt_tos = ['test2@test'] + envelope.content = b'''Subject: test +From: test3@test +To: test4@test +Message-ID: <1@test> +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: base64 + +VGVzdCBtaXQgQmFzZTY0LUtvZGllcnVuZzogw6TDtsO8w58=''' + + result = async_to_sync(handler.handle_DATA)(server, session, envelope) + self.assertEqual(result, '250 Message accepted for delivery') + self.assertEqual(len(Email.objects.all()), 2) + self.assertEqual(len(IssueThread.objects.all()), 1) + aiosmtplib.send.assert_called_once() + self.assertEqual('test', Email.objects.all()[0].subject) + self.assertEqual('Test mit Base64-Kodierung: äöüß', Email.objects.all()[0].body) + self.assertTrue(Email.objects.all()[0].raw_file.path) + def test_handle_client_reply(self): issue_thread = IssueThread.objects.create( name="test", From f133ae9e60450d3302a1c962c3e025c754ca52e7 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 17:39:13 +0100 Subject: [PATCH 22/41] allow searching while "all" event is selected --- web/src/store.js | 82 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/web/src/store.js b/web/src/store.js index b276086..e747ae7 100644 --- a/web/src/store.js +++ b/web/src/store.js @@ -77,10 +77,26 @@ const store = createStore({ getEventTickets: (state, getters) => getters.getEventSlug === 'all' ? getters.getAllTickets : getters.getAllTickets.filter(t => t.event === getters.getEventSlug || (t.event == null && getters.getEventSlug === 'none')), isItemsLoaded: (state, getters) => (getters.getEventSlug === 'all' || getters.getEventSlug === 'none') ? !!state.loadedItems : Object.keys(state.loadedItems).includes(getters.getEventSlug), isTicketsLoaded: (state, getters) => (getters.getEventSlug === 'all' || getters.getEventSlug === 'none') ? !!state.loadedTickets : Object.keys(state.loadedTickets).includes(getters.getEventSlug), - getItemsSearchResults: (state, getters) => state.loadedItemSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [], - getTicketsSearchResults: (state, getters) => state.loadedTicketSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [], - isItemsSearchLoaded: (state, getters) => Object.keys(state.loadedItemSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))), - isTicketsSearchLoaded: (state, getters) => Object.keys(state.loadedTicketSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))), + getItemsSearchResults: (state, getters) => { + if (getters.getEventSlug === 'all') { + return state.events.map(e => { + return state.loadedItemSearchResults[e.slug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + }).flat(); + } else { + return state.loadedItemSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + } + }, + getTicketsSearchResults: (state, getters) => { + if (getters.getEventSlug === 'all') { + return state.events.map(e => { + return state.loadedTicketSearchResults[e.slug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + }).flat(); + } else { + return state.loadedTicketSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [] + } + }, + isItemsSearchLoaded: (state, getters) => Object.keys(state.loadedItemSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))) || getters.getEventSlug === 'all', + isTicketsSearchLoaded: (state, getters) => Object.keys(state.loadedTicketSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))) || getters.getEventSlug === 'all', getActiveView: state => router.currentRoute.value.name || 'items', getFilters: state => router.currentRoute.value.query, getBoxes: state => state.loadedBoxes, @@ -379,26 +395,39 @@ const store = createStore({ }, async loadEventItems({commit, getters, state}) { if (!state.user.token) return; - if (state.fetchedData.items > Date.now() - 1000 * 60 * 60 * 24) return; - try { - const slug = getters.getEventSlug; - const {data, success} = await getters.session.get(`/2/${slug}/items/`); - if (data && success) { - commit('setItems', {slug, items: data}); + const load = async (slug) => { + try { + const {data, success} = await getters.session.get(`/2/${slug}/items/`); + if (data && success) { + commit('setItems', {slug, items: data}); + } + } catch (e) { + console.error("Error loading items"); } - } catch (e) { - console.error("Error loading items"); + } + const slug = getters.getEventSlug; + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); } }, async searchEventItems({commit, getters, state}, query) { const encoded_query = base64.encode(utf8.encode(query)); + const load = async (slug) => { + if (Object.keys(state.loadedItemSearchResults).includes(slug + '/' + encoded_query)) return; + const { + data, success + } = await getters.session.get(`/2/${slug}/items/${encoded_query}/`); + if (data && success) { + commit('setItemSearchResults', {slug, query: encoded_query, items: data}); + } + } const slug = getters.getEventSlug; - if (Object.keys(state.loadedItemSearchResults).includes(slug + '/' + encoded_query)) return; - const { - data, success - } = await getters.session.get(`/2/${slug}/items/${encoded_query}/`); - if (data && success) { - commit('setItemSearchResults', {slug, query: encoded_query, items: data}); + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); } }, async loadBoxes({commit, state, getters}) { @@ -446,12 +475,19 @@ const store = createStore({ }, async searchEventTickets({commit, getters, state}, query) { const encoded_query = base64.encode(utf8.encode(query)); + const load = async (slug) => { + if (Object.keys(state.loadedTicketSearchResults).includes(slug + '/' + encoded_query)) return; + const { + data, success + } = await getters.session.get(`/2/${slug}/tickets/${encoded_query}/`); + if (data && success) commit('setTicketSearchResults', {slug, query: encoded_query, items: data}); + } const slug = getters.getEventSlug; - if (Object.keys(state.loadedTicketSearchResults).includes(slug + '/' + encoded_query)) return; - const { - data, success - } = await getters.session.get(`/2/${slug}/tickets/${encoded_query}/`); - if (data && success) commit('setTicketSearchResults', {slug, query: encoded_query, items: data}); + if (slug === 'all') { + await Promise.all(state.events.map(e => load(e.slug))); + } else { + await load(slug); + } }, async sendMail({commit, dispatch, state, getters}, {id, message}) { const {data, success} = await getters.session.post(`/2/tickets/${id}/reply/`, {message}, From 4ea74637a3d7f59accef14b4a2885c34edd3dfc5 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 18:30:42 +0100 Subject: [PATCH 23/41] finally get a grip on utf-8 --- core/mail/protocol.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 00872f0..3639989 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -53,6 +53,12 @@ def unescape_simplified_quoted_printable(s, encoding='utf-8'): return quopri.decodestring(s).decode(encoding) +def decode_inline_encodings(s): + s = unescape_and_decode_quoted_printable(s) + s = unescape_and_decode_base64(s) + return s + + def ascii_strip(s): if not s: return None @@ -134,8 +140,7 @@ def decode_email_segment(segment, charset, transfer_encoding): import base64 segment = base64.b64decode(segment).decode('utf-8') else: - segment = unescape_and_decode_quoted_printable(segment) - segment = unescape_and_decode_base64(segment) + segment = decode_inline_encodings(segment.decode('utf-8')) return segment @@ -160,7 +165,7 @@ def parse_email_body(raw, log=None): segment = part.get_payload() if not segment: continue - segment = decode_email_segment(segment, charset, part.get('Content-Transfer-Encoding')) + segment = decode_email_segment(segment.encode('utf-8'), charset, part.get('Content-Transfer-Encoding')) log.debug(segment) body = body + segment elif 'attachment' in cdispo or 'inline' in cdispo: @@ -193,7 +198,8 @@ def parse_email_body(raw, log=None): else: log.warning("Unknown content type %s", parsed.get_content_type()) body = "Unknown content type" - body = decode_email_segment(body, parsed.get_content_charset(), parsed.get('Content-Transfer-Encoding')) + body = decode_email_segment(body.encode('utf-8'), parsed.get_content_charset(), + parsed.get('Content-Transfer-Encoding')) log.debug(body) return parsed, body, attachments @@ -221,8 +227,9 @@ def receive_email(envelope, log=None): subject = ascii_strip(parsed.get('Subject')) if not subject: subject = "No subject" - subject = unescape_and_decode_quoted_printable(subject) - subject = unescape_and_decode_base64(subject) + subject = decode_inline_encodings(subject) + recipient = decode_inline_encodings(recipient) + sender = decode_inline_encodings(sender) target_event = find_target_event(recipient) active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event) @@ -256,7 +263,7 @@ do not create a new request. Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid()) reply_email = Email.objects.create( - sender=recipient, recipient=sender, body=body, subject=ascii_strip(subject), + sender=recipient, recipient=sender, body=body, subject=subject, in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread) reply = make_reply(reply_email, references, event=target_event.slug if target_event else None) else: From fbbf8352cf9fdfe0af06c1b8df1ab92ae4832b50 Mon Sep 17 00:00:00 2001 From: jedi Date: Mon, 20 Jan 2025 19:43:01 +0100 Subject: [PATCH 24/41] don't report "Internal Server Error" if mail already exists --- core/mail/protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/mail/protocol.py b/core/mail/protocol.py index 3639989..8fa2c3b 100644 --- a/core/mail/protocol.py +++ b/core/mail/protocol.py @@ -220,7 +220,7 @@ def receive_email(envelope, log=None): if Email.objects.filter(reference=header_message_id).exists(): # break before issue thread is created log.warning("Email already exists") - raise Exception("Email already exists") + raise SpecialMailException("Email already exists") recipient = envelope.rcpt_tos[0].lower() if envelope.rcpt_tos else header_to.lower() sender = envelope.mail_from if envelope.mail_from else header_from From 2677f4b8b6c1d614e2bcb3ff3278eb5ea46e33c3 Mon Sep 17 00:00:00 2001 From: jedi Date: Sun, 26 Jan 2025 19:56:25 +0100 Subject: [PATCH 25/41] link item to ticket frontend --- web/src/store.js | 9 ++++++++- web/src/views/Ticket.vue | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/web/src/store.js b/web/src/store.js index e747ae7..34650e4 100644 --- a/web/src/store.js +++ b/web/src/store.js @@ -61,7 +61,6 @@ const store = createStore({ '2kg-de': '2kg Paket (DE)', '5kg-de': '5kg Paket (DE)', '10kg-de': '10kg Paket (DE)', - '2kg-eu': '2kg Paket (EU)', '5kg-eu': '5kg Paket (EU)', '10kg-eu': '10kg Paket (EU)', } @@ -564,6 +563,14 @@ const store = createStore({ state.fetchedData.tickets = 0; await Promise.all([dispatch('loadTickets'), dispatch('fetchShippingVouchers')]); } + }, + async linkTicketItem({dispatch, state, getters}, {ticket_id, item_id}) { + const {data, success} = await getters.session.post(`/2/matches/`, {issue_thread: ticket_id, item: item_id}); + if (data && success) { + state.fetchedData.tickets = 0; + state.fetchedData.items = 0; + await Promise.all([dispatch('loadTickets'), dispatch('loadEventItems')]); + } } }, plugins: [persistentStatePlugin({ // TODO change remember to some kind of enable field diff --git a/web/src/views/Ticket.vue b/web/src/views/Ticket.vue index dd0b413..220ec3f 100644 --- a/web/src/views/Ticket.vue +++ b/web/src/views/Ticket.vue @@ -81,6 +81,13 @@ Copy DHL contact to clipboard +
+ + +
- + Save Comment @@ -25,7 +25,7 @@