diff --git a/.forgejo/workflows/deploy_staging.yml b/.forgejo/workflows/deploy_staging.yml index 3b44d24..4c5fcd0 100644 --- a/.forgejo/workflows/deploy_staging.yml +++ b/.forgejo/workflows/deploy_staging.yml @@ -35,7 +35,7 @@ jobs: - name: Populate relevant files run: | - mkdir ~/.ssh + mkdir -p ~/.ssh echo "${{ secrets.C3LF_SSH_TESTING }}" > ~/.ssh/id_ed25519 chmod 0600 ~/.ssh/id_ed25519 ls -lah ~/.ssh @@ -43,7 +43,7 @@ jobs: eval $(ssh-agent -s) ssh-add ~/.ssh/id_ed25519 echo "andromeda.lab.or.it ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDXPoO0PE+B9PYwbGaLo98zhbmjAkp6eBtVeZe43v/+T" >> ~/.ssh/known_hosts - mkdir /etc/ansible + mkdir -p /etc/ansible echo "${{ secrets.C3LF_INVENTORY_TESTING }}" > /etc/ansible/hosts - name: Check ansible version diff --git a/core/core/metrics.py b/core/core/metrics.py new file mode 100644 index 0000000..083bda2 --- /dev/null +++ b/core/core/metrics.py @@ -0,0 +1,35 @@ +from django.apps import apps +from prometheus_client.core import CounterMetricFamily, REGISTRY +from django.db.models import Case, Value, When, BooleanField, Count +from inventory.models import Item + +class ItemCountCollector(object): + + def collect(self): + counter = CounterMetricFamily("item_count", "Current number of items", labels=['event', 'returned_state']) + + yield counter + + if not apps.models_ready or not apps.apps_ready: + return + + queryset = ( + Item.all_objects + .annotate( + returned=Case( + When(returned_at__isnull=False, then=Value(False)), + default=Value(True), + output_field=BooleanField() + ) + ) + .values('event__slug', 'returned', 'event_id') + .annotate(amount=Count('id')) + .order_by('event__slug', 'returned') # Optional: order by slug and returned + ) + + for e in queryset: + counter.add_metric([e["event__slug"].lower(), str(e["returned"])], e["amount"]) + + yield counter + +REGISTRY.register(ItemCountCollector()) \ No newline at end of file diff --git a/core/core/settings.py b/core/core/settings.py index 5a8f20f..805a27b 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -124,19 +124,12 @@ TEMPLATES = [ }, ] -WSGI_APPLICATION = 'core.wsgi.application' +ASGI_APPLICATION = 'core.asgi.application' # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases -if 'test' in sys.argv: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', - } - } -else: +if os.getenv('DB_HOST') is not None: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', @@ -149,6 +142,20 @@ else: 'charset': 'utf8mb4', 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'" } + }, + } +elif os.getenv('DB_FILE') is not None: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.getenv('DB_FILE', 'local.db'), + } + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', } } diff --git a/core/core/urls.py b/core/core/urls.py index 1c5f158..2386891 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -19,6 +19,8 @@ from django.urls import path, include from .version import get_info +from .metrics import * + urlpatterns = [ path('djangoadmin/', admin.site.urls), path('api/2/', include('inventory.api_v2')), diff --git a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml index 544b4e4..4005146 100644 --- a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml +++ b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml @@ -345,6 +345,13 @@ notify: - restart postfix + - name: configure rspamd dkim + template: + src: templates/rspamd-dkim.cf.j2 + dest: /etc/rspamd/local.d/dkim_signing.conf + notify: + - restart rspamd + - name: configure rspamd copy: content: | diff --git a/deploy/ansible/playbooks/templates/postfix.cf.j2 b/deploy/ansible/playbooks/templates/postfix.cf.j2 index f80d69b..f6e0b09 100644 --- a/deploy/ansible/playbooks/templates/postfix.cf.j2 +++ b/deploy/ansible/playbooks/templates/postfix.cf.j2 @@ -32,12 +32,11 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -myhostname = polaris.c3lf.de +myhostname = polaris.lab.or.it alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases myorigin = /etc/mailname mydestination = $myhostname, , localhost -relayhost = firefly.lab.or.it mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_size_limit = 0 recipient_delimiter = + diff --git a/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 b/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 new file mode 100644 index 0000000..9e21aa5 --- /dev/null +++ b/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 @@ -0,0 +1,79 @@ +# local.d/dkim_signing.conf + +enabled = true; + +# If false, messages with empty envelope from are not signed +allow_envfrom_empty = true; + +# If true, envelope/header domain mismatch is ignored +allow_hdrfrom_mismatch = false; + +# If true, multiple from headers are allowed (but only first is used) +allow_hdrfrom_multiple = false; + +# If true, username does not need to contain matching domain +allow_username_mismatch = false; + +# Default path to key, can include '$domain' and '$selector' variables +path = "/var/lib/rspamd/dkim/$domain.$selector.key"; + +# Default selector to use +selector = "dkim"; + +# If false, messages from authenticated users are not selected for signing +sign_authenticated = true; + +# If false, messages from local networks are not selected for signing +sign_local = true; + +# Map file of IP addresses/subnets to consider for signing +# sign_networks = "/some/file"; # or url + +# Symbol to add when message is signed +symbol = "DKIM_SIGNED"; + +# Whether to fallback to global config +try_fallback = true; + +# Domain to use for DKIM signing: can be "header" (MIME From), "envelope" (SMTP From), "recipient" (SMTP To), "auth" (SMTP username) or directly specified domain name +use_domain = "header"; + +# Domain to use for DKIM signing when sender is in sign_networks ("header"/"envelope"/"auth") +#use_domain_sign_networks = "header"; + +# Domain to use for DKIM signing when sender is a local IP ("header"/"envelope"/"auth") +#use_domain_sign_local = "header"; + +# Whether to normalise domains to eSLD +use_esld = true; + +# Whether to get keys from Redis +use_redis = false; + +# Hash for DKIM keys in Redis +key_prefix = "DKIM_KEYS"; + +# map of domains -> names of selectors (since rspamd 1.5.3) +#selector_map = "/etc/rspamd/dkim_selectors.map"; + +# map of domains -> paths to keys (since rspamd 1.5.3) +#path_map = "/etc/rspamd/dkim_paths.map"; + +# If `true` get pubkey from DNS record and check if it matches private key +check_pubkey = false; +# Set to `false` if you want to skip signing if public and private keys mismatch +allow_pubkey_mismatch = true; + +# Domain specific settings +domain { + # Domain name is used as key + c3lf.de { + + # Private key path + path = "/var/lib/rspamd/dkim/{{ mail_domain }}.key"; + + # Selector + selector = "{{ mail_domain }}"; + } +} + diff --git a/deploy/dev/docker-compose.yml b/deploy/dev/docker-compose.yml index dff5ab3..e44c276 100644 --- a/deploy/dev/docker-compose.yml +++ b/deploy/dev/docker-compose.yml @@ -3,20 +3,15 @@ services: build: context: ../../core dockerfile: ../deploy/dev/Dockerfile.backend - command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000' + command: bash -c 'python manage.py migrate && python testdata.py && python manage.py runserver 0.0.0.0:8000' environment: - HTTP_HOST=core - - DB_HOST=db - - DB_PORT=3306 - - DB_NAME=system3 - - DB_USER=system3 - - DB_PASSWORD=system3 + - DB_FILE=dev.db volumes: - ../../core:/code + - ../testdata.py:/code/testdata.py ports: - "8000:8000" - depends_on: - - db frontend: build: @@ -31,18 +26,3 @@ services: - "8080:8080" depends_on: - core - - db: - image: mariadb - environment: - MARIADB_RANDOM_ROOT_PASSWORD: true - MARIADB_DATABASE: system3 - MARIADB_USER: system3 - MARIADB_PASSWORD: system3 - volumes: - - mariadb_data:/var/lib/mysql - ports: - - "3306:3306" - -volumes: - mariadb_data: \ No newline at end of file diff --git a/deploy/testdata.py b/deploy/testdata.py new file mode 100644 index 0000000..dca385f --- /dev/null +++ b/deploy/testdata.py @@ -0,0 +1,88 @@ +import os + + +def setup(): + from authentication.models import ExtendedUser, EventPermission + from inventory.models import Event + from django.contrib.auth.models import Permission, Group + permissions = ['add_item', 'view_item', 'view_file', 'delete_item', 'change_item'] + if not ExtendedUser.objects.filter(username='admin').exists(): + admin = ExtendedUser.objects.create_superuser('admin', 'admin@example.com', 'admin') + admin.set_password('admin') + admin.user_permissions.add(*Permission.objects.all()) + admin.save() + + if not ExtendedUser.objects.filter(username='testuser').exists(): + testuser = ExtendedUser.objects.create_user('testuser', 'testuser@example.com', 'testuser') + testuser.set_password('testuser') + testuser.user_permissions.add(*Permission.objects.all()) + testuser.save() + + team = Group.objects.get(name='Team') + team.permissions.add( + *Permission.objects.all() + ) + + if not ExtendedUser.objects.filter(username='testuser2').exists(): + testuser2 = ExtendedUser.objects.create_user('testuser2', 'testuser2@example.com', 'testuser2') + testuser2.set_password('testuser2') + testuser2.groups.add(team) + testuser2.save() + + event1 = Event.objects.get_or_create(id=1, name='first test event', slug='TEST1', + start='2023-12-18 00:00:00.000000', end='2023-12-27 00:00:00.000000', + pre_start='2023-12-31 00:00:00.000000', post_end='2024-01-04 00:00:00.000000')[ + 0] + + event2 = Event.objects.get_or_create(id=2, name='second test event', slug='TEST2', + start='2024-12-18 00:00:00.000000', end='2024-12-27 00:00:00.000000', + pre_start='2024-12-31 00:00:00.000000', post_end='2025-01-04 00:00:00.000000')[ + 0] + + # for permission in permissions: + # EventPermission.objects.create(event=event_37c3, user=foo, + # permission=Permission.objects.get(codename=permission)) + + from tickets.models import IssueThread + + from mail.models import Email + + issue_thread = IssueThread.objects.get_or_create( + id=1, + name="test", + event=Event.objects.get(slug='TEST1') + )[0] + mail1 = Email.objects.get_or_create( + id=1, + subject='test subject', + body='test', + sender='test1@test', + recipient='test2@test', + issue_thread=issue_thread, + )[0] + mail1_reply = Email.objects.get_or_create( + id=2, + subject='Message received', + body='Thank you for your message.', + sender='test2@test', + recipient='test1@test', + in_reply_to=mail1.reference, + issue_thread=issue_thread, + )[0] + + +def main(): + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") + import django + + django.setup() + + from django.core.management import call_command + call_command('migrate') + + setup() + print('testdata initialised') + + +if __name__ == '__main__': + main() diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml index e93e901..b41dd63 100644 --- a/deploy/testing/docker-compose.yml +++ b/deploy/testing/docker-compose.yml @@ -20,7 +20,7 @@ services: build: context: ../../core dockerfile: ../deploy/testing/Dockerfile.backend - command: bash -c 'python manage.py migrate && python /code/server.py' + command: bash -c 'python manage.py migrate && python testdata.py && python /code/server.py' environment: - HTTP_HOST=core - REDIS_HOST=redis @@ -29,13 +29,16 @@ services: - DB_NAME=system3 - DB_USER=system3 - DB_PASSWORD=system3 + - MAIL_DOMAIN=mail:1025 volumes: - ../../core:/code + - ../testdata.py:/code/testdata.py ports: - "8000:8000" depends_on: - db - redis + - mail frontend: build: @@ -51,5 +54,19 @@ services: depends_on: - core + mail: + image: docker.io/axllent/mailpit + volumes: + - mailpit_data:/data + ports: + - 8025:8025 + - 1025:1025 + environment: + MP_MAX_MESSAGES: 5000 + MP_DATABASE: /data/mailpit.db + MP_SMTP_AUTH_ACCEPT_ANY: 1 + MP_SMTP_AUTH_ALLOW_INSECURE: 1 + volumes: - mariadb_data: \ No newline at end of file + mariadb_data: + mailpit_data: diff --git a/web/node_modules/.forgit_fordocker b/web/node_modules/.forgit_fordocker new file mode 100644 index 0000000..e69de29 diff --git a/web/src/components/AddItemModal.vue b/web/src/components/AddItemModal.vue index a3c23fd..24bd449 100644 --- a/web/src/components/AddItemModal.vue +++ b/web/src/components/AddItemModal.vue @@ -2,7 +2,29 @@
diff --git a/web/src/components/AddTicketModal.vue b/web/src/components/AddTicketModal.vue index 37d539c..b407670 100644 --- a/web/src/components/AddTicketModal.vue +++ b/web/src/components/AddTicketModal.vue @@ -19,11 +19,10 @@ diff --git a/web/src/components/inputs/InputCombo.vue b/web/src/components/inputs/InputCombo.vue index fc64d42..2a291e0 100644 --- a/web/src/components/inputs/InputCombo.vue +++ b/web/src/components/inputs/InputCombo.vue @@ -43,11 +43,11 @@ export default { props: ['label', 'model', 'nameKey', 'uniqueKey', 'options', 'onOptionAdd'], data: ({options, model, nameKey, uniqueKey}) => ({ internalName: model[nameKey], - selectedOption: options.filter(e => e[uniqueKey] == model[uniqueKey])[0], + selectedOption: options.filter(e => e[uniqueKey] === model[uniqueKey])[0], addingOption: false }), computed: { - isValid: ({options, nameKey, internalName}) => options.some(e => e[nameKey] == internalName), + isValid: ({options, nameKey, internalName}) => options.some(e => e[nameKey] === internalName), sortedOptions: ({ options, nameKey @@ -56,7 +56,7 @@ export default { watch: { internalName(newValue) { if (this.isValid) { - if (!this.selectedOption || newValue != this.selectedOption[this.nameKey]) { + if (!this.selectedOption || newValue !== this.selectedOption[this.nameKey]) { this.selectedOption = this.options.filter(e => e[this.nameKey] === newValue)[0]; } this.model[this.nameKey] = this.selectedOption[this.nameKey]; diff --git a/web/src/store.js b/web/src/store.js index 9aa3dd3..aa4820d 100644 --- a/web/src/store.js +++ b/web/src/store.js @@ -339,7 +339,7 @@ const store = createStore({ commit('replaceEvents', [...state.events.filter(e => e.id !== event_id)]) } }, - async updateEvent({commit, dispatch, state}, {id, partial_event}){ + async updateEvent({commit, dispatch, state}, {id, partial_event}) { const {data, success} = await http.patch(`/2/events/${id}/`, partial_event, state.user.token); if (success) { commit('replaceEvents', [...state.events.filter(e => e.id !== id), data]) @@ -443,7 +443,8 @@ const store = createStore({ } }, async postManualTicket({commit, dispatch, state, getters}, {sender, message, title,}) { - const {data, success} = await getters.session.post(`/2/tickets/manual/`, { + const slug = getters.getEventSlug; + const {data, success} = await getters.session.post(`/2/${slug !== 'all' ? slug : 'none'}/tickets/manual/`, { name: title, sender, body: message, recipient: 'mail@c3lf.de' }); await dispatch('loadTickets'); @@ -456,7 +457,10 @@ const store = createStore({ } }, async postItemComment({commit, dispatch, state, getters}, {id, message}) { - const {data, success} = await getters.session.post(`/2/${getters.getEventSlug}/item/${id}/comment/`, {comment: message}); + const { + data, + success + } = await getters.session.post(`/2/${getters.getEventSlug}/item/${id}/comment/`, {comment: message}); if (data && success) { state.fetchedData.items = 0; await dispatch('loadEventItems'); diff --git a/web/src/views/Item.vue b/web/src/views/Item.vue index d00b8b1..7b9f484 100644 --- a/web/src/views/Item.vue +++ b/web/src/views/Item.vue @@ -112,14 +112,12 @@ import InputString from "@/components/inputs/InputString.vue"; import AuthenticatedImage from "@/components/AuthenticatedImage.vue"; import InputPhoto from "@/components/inputs/InputPhoto.vue"; import Modal from "@/components/Modal.vue"; -import EditItem from "@/components/EditItem.vue"; import AsyncButton from "@/components/inputs/AsyncButton.vue"; export default { name: 'Item', components: { AsyncButton, - EditItem, Modal, InputPhoto, AuthenticatedImage, InputString, InputCombo, AsyncLoader, ClipboardButton, Timeline }, data() { diff --git a/web/src/views/Items.vue b/web/src/views/Items.vue index 21b11d1..72b4079 100644 --- a/web/src/views/Items.vue +++ b/web/src/views/Items.vue @@ -67,11 +67,10 @@