finally get a grip on utf-8

core/mail/protocol.py

@@ -53,6 +53,12 @@ def unescape_simplified_quoted_printable(s, encoding='utf-8'):
     return quopri.decodestring(s).decode(encoding)
 
 
+def decode_inline_encodings(s):
+    s = unescape_and_decode_quoted_printable(s)
+    s = unescape_and_decode_base64(s)
+    return s
+
+
 def ascii_strip(s):
     if not s:
         return None
@@ -134,8 +140,7 @@ def decode_email_segment(segment, charset, transfer_encoding):
         import base64
         segment = base64.b64decode(segment).decode('utf-8')
     else:
-        segment = unescape_and_decode_quoted_printable(segment)
-        segment = unescape_and_decode_base64(segment)
+        segment = decode_inline_encodings(segment.decode('utf-8'))
     return segment
 
 
@@ -160,7 +165,7 @@ def parse_email_body(raw, log=None):
                 segment = part.get_payload()
                 if not segment:
                     continue
-                segment = decode_email_segment(segment, charset, part.get('Content-Transfer-Encoding'))
+                segment = decode_email_segment(segment.encode('utf-8'), charset, part.get('Content-Transfer-Encoding'))
                 log.debug(segment)
                 body = body + segment
             elif 'attachment' in cdispo or 'inline' in cdispo:
@@ -193,7 +198,8 @@ def parse_email_body(raw, log=None):
         else:
             log.warning("Unknown content type %s", parsed.get_content_type())
             body = "Unknown content type"
-        body = decode_email_segment(body, parsed.get_content_charset(), parsed.get('Content-Transfer-Encoding'))
+        body = decode_email_segment(body.encode('utf-8'), parsed.get_content_charset(),
+                                    parsed.get('Content-Transfer-Encoding'))
         log.debug(body)
 
     return parsed, body, attachments
@@ -221,8 +227,9 @@ def receive_email(envelope, log=None):
     subject = ascii_strip(parsed.get('Subject'))
     if not subject:
         subject = "No subject"
-    subject = unescape_and_decode_quoted_printable(subject)
-    subject = unescape_and_decode_base64(subject)
+    subject = decode_inline_encodings(subject)
+    recipient = decode_inline_encodings(recipient)
+    sender = decode_inline_encodings(sender)
     target_event = find_target_event(recipient)
 
     active_issue_thread, new = find_active_issue_thread(header_in_reply_to, recipient, subject, target_event)
@@ -256,7 +263,7 @@ do not create a new request.
 
 Your c3lf (Cloakroom + Lost&Found) Team'''.format(active_issue_thread.short_uuid())
             reply_email = Email.objects.create(
-                sender=recipient, recipient=sender, body=body, subject=ascii_strip(subject),
+                sender=recipient, recipient=sender, body=body, subject=subject,
                 in_reply_to=header_message_id, event=target_event, issue_thread=active_issue_thread)
             reply = make_reply(reply_email, references, event=target_event.slug if target_event else None)
     else:

web/src/store.js

@@ -77,10 +77,26 @@ const store = createStore({
         getEventTickets: (state, getters) => getters.getEventSlug === 'all' ? getters.getAllTickets : getters.getAllTickets.filter(t => t.event === getters.getEventSlug || (t.event == null && getters.getEventSlug === 'none')),
         isItemsLoaded: (state, getters) => (getters.getEventSlug === 'all' || getters.getEventSlug === 'none') ? !!state.loadedItems : Object.keys(state.loadedItems).includes(getters.getEventSlug),
         isTicketsLoaded: (state, getters) => (getters.getEventSlug === 'all' || getters.getEventSlug === 'none') ? !!state.loadedTickets : Object.keys(state.loadedTickets).includes(getters.getEventSlug),
-        getItemsSearchResults: (state, getters) => state.loadedItemSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [],
-        getTicketsSearchResults: (state, getters) => state.loadedTicketSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || [],
-        isItemsSearchLoaded: (state, getters) => Object.keys(state.loadedItemSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))),
-        isTicketsSearchLoaded: (state, getters) => Object.keys(state.loadedTicketSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))),
+        getItemsSearchResults: (state, getters) => {
+            if (getters.getEventSlug === 'all') {
+                return state.events.map(e => {
+                    return state.loadedItemSearchResults[e.slug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || []
+                }).flat();
+            } else {
+                return state.loadedItemSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || []
+            }
+        },
+        getTicketsSearchResults: (state, getters) => {
+            if (getters.getEventSlug === 'all') {
+                return state.events.map(e => {
+                    return state.loadedTicketSearchResults[e.slug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || []
+                }).flat();
+            } else {
+                return state.loadedTicketSearchResults[getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))] || []
+            }
+        },
+        isItemsSearchLoaded: (state, getters) => Object.keys(state.loadedItemSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))) || getters.getEventSlug === 'all',
+        isTicketsSearchLoaded: (state, getters) => Object.keys(state.loadedTicketSearchResults).includes(getters.getEventSlug + '/' + base64.encode(utf8.encode(getters.searchQuery))) || getters.getEventSlug === 'all',
         getActiveView: state => router.currentRoute.value.name || 'items',
         getFilters: state => router.currentRoute.value.query,
         getBoxes: state => state.loadedBoxes,
@@ -379,26 +395,39 @@ const store = createStore({
         },
         async loadEventItems({commit, getters, state}) {
             if (!state.user.token) return;
-            if (state.fetchedData.items > Date.now() - 1000 * 60 * 60 * 24) return;
-            try {
-                const slug = getters.getEventSlug;
-                const {data, success} = await getters.session.get(`/2/${slug}/items/`);
-                if (data && success) {
-                    commit('setItems', {slug, items: data});
+            const load = async (slug) => {
+                try {
+                    const {data, success} = await getters.session.get(`/2/${slug}/items/`);
+                    if (data && success) {
+                        commit('setItems', {slug, items: data});
+                    }
+                } catch (e) {
+                    console.error("Error loading items");
                 }
-            } catch (e) {
-                console.error("Error loading items");
+            }
+            const slug = getters.getEventSlug;
+            if (slug === 'all') {
+                await Promise.all(state.events.map(e => load(e.slug)));
+            } else {
+                await load(slug);
             }
         },
         async searchEventItems({commit, getters, state}, query) {
             const encoded_query = base64.encode(utf8.encode(query));
+            const load = async (slug) => {
+                if (Object.keys(state.loadedItemSearchResults).includes(slug + '/' + encoded_query)) return;
+                const {
+                    data, success
+                } = await getters.session.get(`/2/${slug}/items/${encoded_query}/`);
+                if (data && success) {
+                    commit('setItemSearchResults', {slug, query: encoded_query, items: data});
+                }
+            }
             const slug = getters.getEventSlug;
-            if (Object.keys(state.loadedItemSearchResults).includes(slug + '/' + encoded_query)) return;
-            const {
-                data, success
-            } = await getters.session.get(`/2/${slug}/items/${encoded_query}/`);
-            if (data && success) {
-                commit('setItemSearchResults', {slug, query: encoded_query, items: data});
+            if (slug === 'all') {
+                await Promise.all(state.events.map(e => load(e.slug)));
+            } else {
+                await load(slug);
             }
         },
         async loadBoxes({commit, state, getters}) {
@@ -446,12 +475,19 @@ const store = createStore({
         },
         async searchEventTickets({commit, getters, state}, query) {
             const encoded_query = base64.encode(utf8.encode(query));
+            const load = async (slug) => {
+                if (Object.keys(state.loadedTicketSearchResults).includes(slug + '/' + encoded_query)) return;
+                const {
+                    data, success
+                } = await getters.session.get(`/2/${slug}/tickets/${encoded_query}/`);
+                if (data && success) commit('setTicketSearchResults', {slug, query: encoded_query, items: data});
+            }
             const slug = getters.getEventSlug;
-            if (Object.keys(state.loadedTicketSearchResults).includes(slug + '/' + encoded_query)) return;
-            const {
-                data, success
-            } = await getters.session.get(`/2/${slug}/tickets/${encoded_query}/`);
-            if (data && success) commit('setTicketSearchResults', {slug, query: encoded_query, items: data});
+            if (slug === 'all') {
+                await Promise.all(state.events.map(e => load(e.slug)));
+            } else {
+                await load(slug);
+            }
         },
         async sendMail({commit, dispatch, state, getters}, {id, message}) {
             const {data, success} = await getters.session.post(`/2/tickets/${id}/reply/`, {message},