diff --git a/.forgejo/workflows/deploy_staging.yml b/.forgejo/workflows/deploy_staging.yml index 4c5fcd0..0dccd66 100644 --- a/.forgejo/workflows/deploy_staging.yml +++ b/.forgejo/workflows/deploy_staging.yml @@ -20,9 +20,12 @@ jobs: - name: Run django tests working-directory: core run: python3 manage.py test + - name: Run django coverage + working-directory: core + run: coverage manage.py test deploy: - needs: [test] + needs: [ test ] runs-on: docker steps: - uses: actions/checkout@v4 @@ -35,7 +38,7 @@ jobs: - name: Populate relevant files run: | - mkdir -p ~/.ssh + mkdir ~/.ssh echo "${{ secrets.C3LF_SSH_TESTING }}" > ~/.ssh/id_ed25519 chmod 0600 ~/.ssh/id_ed25519 ls -lah ~/.ssh @@ -43,7 +46,7 @@ jobs: eval $(ssh-agent -s) ssh-add ~/.ssh/id_ed25519 echo "andromeda.lab.or.it ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDXPoO0PE+B9PYwbGaLo98zhbmjAkp6eBtVeZe43v/+T" >> ~/.ssh/known_hosts - mkdir -p /etc/ansible + mkdir /etc/ansible echo "${{ secrets.C3LF_INVENTORY_TESTING }}" > /etc/ansible/hosts - name: Check ansible version diff --git a/core/core/metrics.py b/core/core/metrics.py deleted file mode 100644 index 149829c..0000000 --- a/core/core/metrics.py +++ /dev/null @@ -1,35 +0,0 @@ -from django.apps import apps -from prometheus_client.core import CounterMetricFamily, REGISTRY -from django.db.models import Case, Value, When, BooleanField, Count -from inventory.models import Item - -class ItemCountCollector(object): - - def collect(self): - counter = CounterMetricFamily("item_count", "Current number of items", labels=['event', 'returned_state']) - - yield counter - - if not apps.models_ready or not apps.apps_ready: - return - - queryset = ( - Item.all_objects - .annotate( - returned=Case( - When(returned_at__isnull=True, then=Value(False)), - default=Value(True), - output_field=BooleanField() - ) - ) - .values('event__slug', 'returned', 'event_id') - .annotate(amount=Count('id')) - .order_by('event__slug', 'returned') # Optional: order by slug and returned - ) - - for e in queryset: - counter.add_metric([e["event__slug"].lower(), str(e["returned"])], e["amount"]) - - yield counter - -REGISTRY.register(ItemCountCollector()) \ No newline at end of file diff --git a/core/core/settings.py b/core/core/settings.py index 805a27b..5a8f20f 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -124,12 +124,19 @@ TEMPLATES = [ }, ] -ASGI_APPLICATION = 'core.asgi.application' +WSGI_APPLICATION = 'core.wsgi.application' # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases -if os.getenv('DB_HOST') is not None: +if 'test' in sys.argv: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + } + } +else: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', @@ -142,20 +149,6 @@ if os.getenv('DB_HOST') is not None: 'charset': 'utf8mb4', 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'" } - }, - } -elif os.getenv('DB_FILE') is not None: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.getenv('DB_FILE', 'local.db'), - } - } -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', } } diff --git a/core/core/urls.py b/core/core/urls.py index 2386891..1c5f158 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -19,8 +19,6 @@ from django.urls import path, include from .version import get_info -from .metrics import * - urlpatterns = [ path('djangoadmin/', admin.site.urls), path('api/2/', include('inventory.api_v2')), diff --git a/core/inventory/tests/v2/test_items.py b/core/inventory/tests/v2/test_items.py index 0c85eb4..9b25384 100644 --- a/core/inventory/tests/v2/test_items.py +++ b/core/inventory/tests/v2/test_items.py @@ -13,6 +13,8 @@ from base64 import b64encode from tickets.models import IssueThread, ItemRelation +from base64 import b64encode + class ItemTestCase(TestCase): @@ -343,3 +345,75 @@ class ItemSearchTestCase(TestCase): self.assertEqual('BOX1', response.json()[1]['box']) self.assertEqual(self.box1.id, response.json()[1]['cid']) self.assertEqual(1, response.json()[1]['search_score']) + + + +class ItemSearchTestCase(TestCase): + + def setUp(self): + super().setUp() + self.event = Event.objects.create(slug='EVENT', name='Event') + self.box = Container.objects.create(name='BOX') + self.user = ExtendedUser.objects.create_user('testuser', 'test', 'test') + self.user.user_permissions.add(*Permission.objects.all()) + self.token = AuthToken.objects.create(user=self.user) + self.client = Client(headers={'Authorization': 'Token ' + self.token[1]}) + self.item1 = Item.objects.create(container=self.box, event=self.event, description='abc def') + self.item2 = Item.objects.create(container=self.box, event=self.event, description='def ghi') + self.item3 = Item.objects.create(container=self.box, event=self.event, description='jkl mno pqr') + self.item4 = Item.objects.create(container=self.box, event=self.event, description='stu vwx') + + def test_search(self): + search_query = b64encode(b'abc').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/items/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(self.item1.id, response.json()[0]['id']) + self.assertEqual('abc def', response.json()[0]['description']) + self.assertEqual('BOX', response.json()[0]['box']) + self.assertEqual(self.box.id, response.json()[0]['cid']) + self.assertEqual(1, response.json()[0]['search_score']) + + def test_search2(self): + search_query = b64encode(b'def').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/items/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(2, len(response.json())) + self.assertEqual(self.item1.id, response.json()[0]['id']) + self.assertEqual('abc def', response.json()[0]['description']) + self.assertEqual('BOX', response.json()[0]['box']) + self.assertEqual(self.box.id, response.json()[0]['cid']) + self.assertEqual(1, response.json()[0]['search_score']) + self.assertEqual(self.item2.id, response.json()[1]['id']) + self.assertEqual('def ghi', response.json()[1]['description']) + self.assertEqual('BOX', response.json()[1]['box']) + self.assertEqual(self.box.id, response.json()[1]['cid']) + self.assertEqual(1, response.json()[0]['search_score']) + + def test_search3(self): + search_query = b64encode(b'jkl').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/items/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json())) + self.assertEqual(self.item3.id, response.json()[0]['id']) + self.assertEqual('jkl mno pqr', response.json()[0]['description']) + self.assertEqual('BOX', response.json()[0]['box']) + self.assertEqual(self.box.id, response.json()[0]['cid']) + self.assertEqual(1, response.json()[0]['search_score']) + + def test_search4(self): + search_query = b64encode(b'abc def').decode('utf-8') + response = self.client.get(f'/api/2/{self.event.slug}/items/{search_query}/') + self.assertEqual(200, response.status_code) + self.assertEqual(2, len(response.json())) + self.assertEqual(self.item1.id, response.json()[0]['id']) + self.assertEqual('abc def', response.json()[0]['description']) + self.assertEqual('BOX', response.json()[0]['box']) + self.assertEqual(self.box.id, response.json()[0]['cid']) + self.assertEqual(2, response.json()[0]['search_score']) + self.assertEqual(self.item2.id, response.json()[1]['id']) + self.assertEqual('def ghi', response.json()[1]['description']) + self.assertEqual('BOX', response.json()[1]['box']) + self.assertEqual(self.box.id, response.json()[1]['cid']) + self.assertEqual(1, response.json()[1]['search_score']) + diff --git a/core/tickets/serializers.py b/core/tickets/serializers.py index 50cdb72..6ce73be 100644 --- a/core/tickets/serializers.py +++ b/core/tickets/serializers.py @@ -146,3 +146,14 @@ class SearchResultSerializer(serializers.Serializer): class Meta: model = IssueThread + + +class SearchResultSerializer(serializers.Serializer): + search_score = serializers.IntegerField() + item = IssueSerializer() + + def to_representation(self, instance): + return {**IssueSerializer(instance['item']).data, 'search_score': instance['search_score']} + + class Meta: + model = IssueThread diff --git a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml index 4005146..544b4e4 100644 --- a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml +++ b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml @@ -345,13 +345,6 @@ notify: - restart postfix - - name: configure rspamd dkim - template: - src: templates/rspamd-dkim.cf.j2 - dest: /etc/rspamd/local.d/dkim_signing.conf - notify: - - restart rspamd - - name: configure rspamd copy: content: | diff --git a/deploy/ansible/playbooks/templates/postfix.cf.j2 b/deploy/ansible/playbooks/templates/postfix.cf.j2 index f6e0b09..f80d69b 100644 --- a/deploy/ansible/playbooks/templates/postfix.cf.j2 +++ b/deploy/ansible/playbooks/templates/postfix.cf.j2 @@ -32,11 +32,12 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -myhostname = polaris.lab.or.it +myhostname = polaris.c3lf.de alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases myorigin = /etc/mailname mydestination = $myhostname, , localhost +relayhost = firefly.lab.or.it mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_size_limit = 0 recipient_delimiter = + diff --git a/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 b/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 deleted file mode 100644 index 9e21aa5..0000000 --- a/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 +++ /dev/null @@ -1,79 +0,0 @@ -# local.d/dkim_signing.conf - -enabled = true; - -# If false, messages with empty envelope from are not signed -allow_envfrom_empty = true; - -# If true, envelope/header domain mismatch is ignored -allow_hdrfrom_mismatch = false; - -# If true, multiple from headers are allowed (but only first is used) -allow_hdrfrom_multiple = false; - -# If true, username does not need to contain matching domain -allow_username_mismatch = false; - -# Default path to key, can include '$domain' and '$selector' variables -path = "/var/lib/rspamd/dkim/$domain.$selector.key"; - -# Default selector to use -selector = "dkim"; - -# If false, messages from authenticated users are not selected for signing -sign_authenticated = true; - -# If false, messages from local networks are not selected for signing -sign_local = true; - -# Map file of IP addresses/subnets to consider for signing -# sign_networks = "/some/file"; # or url - -# Symbol to add when message is signed -symbol = "DKIM_SIGNED"; - -# Whether to fallback to global config -try_fallback = true; - -# Domain to use for DKIM signing: can be "header" (MIME From), "envelope" (SMTP From), "recipient" (SMTP To), "auth" (SMTP username) or directly specified domain name -use_domain = "header"; - -# Domain to use for DKIM signing when sender is in sign_networks ("header"/"envelope"/"auth") -#use_domain_sign_networks = "header"; - -# Domain to use for DKIM signing when sender is a local IP ("header"/"envelope"/"auth") -#use_domain_sign_local = "header"; - -# Whether to normalise domains to eSLD -use_esld = true; - -# Whether to get keys from Redis -use_redis = false; - -# Hash for DKIM keys in Redis -key_prefix = "DKIM_KEYS"; - -# map of domains -> names of selectors (since rspamd 1.5.3) -#selector_map = "/etc/rspamd/dkim_selectors.map"; - -# map of domains -> paths to keys (since rspamd 1.5.3) -#path_map = "/etc/rspamd/dkim_paths.map"; - -# If `true` get pubkey from DNS record and check if it matches private key -check_pubkey = false; -# Set to `false` if you want to skip signing if public and private keys mismatch -allow_pubkey_mismatch = true; - -# Domain specific settings -domain { - # Domain name is used as key - c3lf.de { - - # Private key path - path = "/var/lib/rspamd/dkim/{{ mail_domain }}.key"; - - # Selector - selector = "{{ mail_domain }}"; - } -} - diff --git a/deploy/dev/docker-compose.yml b/deploy/dev/docker-compose.yml index e44c276..dff5ab3 100644 --- a/deploy/dev/docker-compose.yml +++ b/deploy/dev/docker-compose.yml @@ -3,15 +3,20 @@ services: build: context: ../../core dockerfile: ../deploy/dev/Dockerfile.backend - command: bash -c 'python manage.py migrate && python testdata.py && python manage.py runserver 0.0.0.0:8000' + command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000' environment: - HTTP_HOST=core - - DB_FILE=dev.db + - DB_HOST=db + - DB_PORT=3306 + - DB_NAME=system3 + - DB_USER=system3 + - DB_PASSWORD=system3 volumes: - ../../core:/code - - ../testdata.py:/code/testdata.py ports: - "8000:8000" + depends_on: + - db frontend: build: @@ -26,3 +31,18 @@ services: - "8080:8080" depends_on: - core + + db: + image: mariadb + environment: + MARIADB_RANDOM_ROOT_PASSWORD: true + MARIADB_DATABASE: system3 + MARIADB_USER: system3 + MARIADB_PASSWORD: system3 + volumes: + - mariadb_data:/var/lib/mysql + ports: + - "3306:3306" + +volumes: + mariadb_data: \ No newline at end of file diff --git a/deploy/testdata.py b/deploy/testdata.py deleted file mode 100644 index dca385f..0000000 --- a/deploy/testdata.py +++ /dev/null @@ -1,88 +0,0 @@ -import os - - -def setup(): - from authentication.models import ExtendedUser, EventPermission - from inventory.models import Event - from django.contrib.auth.models import Permission, Group - permissions = ['add_item', 'view_item', 'view_file', 'delete_item', 'change_item'] - if not ExtendedUser.objects.filter(username='admin').exists(): - admin = ExtendedUser.objects.create_superuser('admin', 'admin@example.com', 'admin') - admin.set_password('admin') - admin.user_permissions.add(*Permission.objects.all()) - admin.save() - - if not ExtendedUser.objects.filter(username='testuser').exists(): - testuser = ExtendedUser.objects.create_user('testuser', 'testuser@example.com', 'testuser') - testuser.set_password('testuser') - testuser.user_permissions.add(*Permission.objects.all()) - testuser.save() - - team = Group.objects.get(name='Team') - team.permissions.add( - *Permission.objects.all() - ) - - if not ExtendedUser.objects.filter(username='testuser2').exists(): - testuser2 = ExtendedUser.objects.create_user('testuser2', 'testuser2@example.com', 'testuser2') - testuser2.set_password('testuser2') - testuser2.groups.add(team) - testuser2.save() - - event1 = Event.objects.get_or_create(id=1, name='first test event', slug='TEST1', - start='2023-12-18 00:00:00.000000', end='2023-12-27 00:00:00.000000', - pre_start='2023-12-31 00:00:00.000000', post_end='2024-01-04 00:00:00.000000')[ - 0] - - event2 = Event.objects.get_or_create(id=2, name='second test event', slug='TEST2', - start='2024-12-18 00:00:00.000000', end='2024-12-27 00:00:00.000000', - pre_start='2024-12-31 00:00:00.000000', post_end='2025-01-04 00:00:00.000000')[ - 0] - - # for permission in permissions: - # EventPermission.objects.create(event=event_37c3, user=foo, - # permission=Permission.objects.get(codename=permission)) - - from tickets.models import IssueThread - - from mail.models import Email - - issue_thread = IssueThread.objects.get_or_create( - id=1, - name="test", - event=Event.objects.get(slug='TEST1') - )[0] - mail1 = Email.objects.get_or_create( - id=1, - subject='test subject', - body='test', - sender='test1@test', - recipient='test2@test', - issue_thread=issue_thread, - )[0] - mail1_reply = Email.objects.get_or_create( - id=2, - subject='Message received', - body='Thank you for your message.', - sender='test2@test', - recipient='test1@test', - in_reply_to=mail1.reference, - issue_thread=issue_thread, - )[0] - - -def main(): - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") - import django - - django.setup() - - from django.core.management import call_command - call_command('migrate') - - setup() - print('testdata initialised') - - -if __name__ == '__main__': - main() diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml index b41dd63..e93e901 100644 --- a/deploy/testing/docker-compose.yml +++ b/deploy/testing/docker-compose.yml @@ -20,7 +20,7 @@ services: build: context: ../../core dockerfile: ../deploy/testing/Dockerfile.backend - command: bash -c 'python manage.py migrate && python testdata.py && python /code/server.py' + command: bash -c 'python manage.py migrate && python /code/server.py' environment: - HTTP_HOST=core - REDIS_HOST=redis @@ -29,16 +29,13 @@ services: - DB_NAME=system3 - DB_USER=system3 - DB_PASSWORD=system3 - - MAIL_DOMAIN=mail:1025 volumes: - ../../core:/code - - ../testdata.py:/code/testdata.py ports: - "8000:8000" depends_on: - db - redis - - mail frontend: build: @@ -54,19 +51,5 @@ services: depends_on: - core - mail: - image: docker.io/axllent/mailpit - volumes: - - mailpit_data:/data - ports: - - 8025:8025 - - 1025:1025 - environment: - MP_MAX_MESSAGES: 5000 - MP_DATABASE: /data/mailpit.db - MP_SMTP_AUTH_ACCEPT_ANY: 1 - MP_SMTP_AUTH_ALLOW_INSECURE: 1 - volumes: - mariadb_data: - mailpit_data: + mariadb_data: \ No newline at end of file diff --git a/web/node_modules/.forgit_fordocker b/web/node_modules/.forgit_fordocker deleted file mode 100644 index e69de29..0000000 diff --git a/web/src/components/AddItemModal.vue b/web/src/components/AddItemModal.vue index 24bd449..a3c23fd 100644 --- a/web/src/components/AddItemModal.vue +++ b/web/src/components/AddItemModal.vue @@ -2,29 +2,7 @@