diff --git a/web/node_modules/.forgit_fordocker b/core/core/management/__init__.py similarity index 100% rename from web/node_modules/.forgit_fordocker rename to core/core/management/__init__.py diff --git a/core/core/management/commands/__init__.py b/core/core/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/core/management/commands/update_metrics.py b/core/core/management/commands/update_metrics.py new file mode 100644 index 0000000..a51cfea --- /dev/null +++ b/core/core/management/commands/update_metrics.py @@ -0,0 +1,8 @@ +from django.core.management.base import BaseCommand, CommandError +from core.metrics import update_item_count + +class Command(BaseCommand): + help = "Refresh the metrics" + + def handle(self, *args, **options): + update_item_count() \ No newline at end of file diff --git a/core/core/metrics.py b/core/core/metrics.py new file mode 100644 index 0000000..4f79754 --- /dev/null +++ b/core/core/metrics.py @@ -0,0 +1,11 @@ +from prometheus_client import Gauge +from django_prometheus import exports +from inventory.models import * + +g_items_total = Gauge('c3lf_items_total', 'Current Total items') + +def update_item_count(): + # Get the count of MyModel objects + count = Item.objects.count() + # Set the gauge to the current count + g_items_total.set(count) \ No newline at end of file diff --git a/core/core/settings.py b/core/core/settings.py index 805a27b..31a024f 100644 --- a/core/core/settings.py +++ b/core/core/settings.py @@ -70,6 +70,7 @@ INSTALLED_APPS = [ 'inventory', 'mail', 'notify_sessions', + 'core' ] REST_FRAMEWORK = { @@ -124,12 +125,19 @@ TEMPLATES = [ }, ] -ASGI_APPLICATION = 'core.asgi.application' +WSGI_APPLICATION = 'core.wsgi.application' # Database # https://docs.djangoproject.com/en/4.2/ref/settings/#databases -if os.getenv('DB_HOST') is not None: +if 'test' in sys.argv: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + } + } +else: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', @@ -142,20 +150,6 @@ if os.getenv('DB_HOST') is not None: 'charset': 'utf8mb4', 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'" } - }, - } -elif os.getenv('DB_FILE') is not None: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.getenv('DB_FILE', 'local.db'), - } - } -else: - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', } } diff --git a/core/core/urls.py b/core/core/urls.py index 1c5f158..2386891 100644 --- a/core/core/urls.py +++ b/core/core/urls.py @@ -19,6 +19,8 @@ from django.urls import path, include from .version import get_info +from .metrics import * + urlpatterns = [ path('djangoadmin/', admin.site.urls), path('api/2/', include('inventory.api_v2')), diff --git a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml index 544b4e4..af5e9e8 100644 --- a/deploy/ansible/playbooks/deploy-c3lf-sys3.yml +++ b/deploy/ansible/playbooks/deploy-c3lf-sys3.yml @@ -311,6 +311,16 @@ notify: - restart c3lf-sys3 + - name: add c3lf-sys3-cron service + template: + src: templates/c3lf-sys3-cron.service.j2 + dest: /etc/systemd/system/c3lf-sys3-cron.service + + - name: add c3lf-sys3-cron timer + template: + src: templates/c3lf-sys3-cron.timer.j2 + dest: /etc/systemd/system/c3lf-sys3-cron.timer + - name: reload systemd systemd: daemon_reload: yes @@ -321,6 +331,12 @@ state: started enabled: yes + - name: start c3lf-sys3 timer + service: + name: c3lf-sys3-cron.timer + state: started + enabled: yes + - name: add postfix to www-data group user: name: postfix @@ -345,6 +361,13 @@ notify: - restart postfix + - name: configure rspamd dkim + template: + src: templates/rspamd-dkim.cf.j2 + dest: /etc/rspamd/local.d/dkim_signing.conf + notify: + - restart rspamd + - name: configure rspamd copy: content: | diff --git a/deploy/ansible/playbooks/templates/c3lf-sys3-cron.service.j2 b/deploy/ansible/playbooks/templates/c3lf-sys3-cron.service.j2 new file mode 100644 index 0000000..0df787e --- /dev/null +++ b/deploy/ansible/playbooks/templates/c3lf-sys3-cron.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=c3lf sys3 background service +After=network.target + +[Service] +User=www-data +Group=www-data +WorkingDirectory=/var/www/c3lf-sys3 +ExecStart=/var/www/c3lf-sys3/venv/bin/python3 /var/www/c3lf-sys3/repo/core/manage.py update_metrics +Environment=DJANGO_SETTINGS_MODULE=core.settings + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/c3lf-sys3-cron.timer.j2 b/deploy/ansible/playbooks/templates/c3lf-sys3-cron.timer.j2 new file mode 100644 index 0000000..6d888b4 --- /dev/null +++ b/deploy/ansible/playbooks/templates/c3lf-sys3-cron.timer.j2 @@ -0,0 +1,9 @@ +[Unit] +Description=Run Nextcloud cron.php every 5 minutes + +[Timer] +OnCalendar=minutely +Unit=c3lf-sys3-cron.service + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/deploy/ansible/playbooks/templates/postfix.cf.j2 b/deploy/ansible/playbooks/templates/postfix.cf.j2 index f80d69b..ee073d6 100644 --- a/deploy/ansible/playbooks/templates/postfix.cf.j2 +++ b/deploy/ansible/playbooks/templates/postfix.cf.j2 @@ -37,7 +37,6 @@ alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases myorigin = /etc/mailname mydestination = $myhostname, , localhost -relayhost = firefly.lab.or.it mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 mailbox_size_limit = 0 recipient_delimiter = + diff --git a/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 b/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 new file mode 100644 index 0000000..0429a02 --- /dev/null +++ b/deploy/ansible/playbooks/templates/rspamd-dkim.cf.j2 @@ -0,0 +1,79 @@ +# local.d/dkim_signing.conf + +enabled = true; + +# If false, messages with empty envelope from are not signed +allow_envfrom_empty = true; + +# If true, envelope/header domain mismatch is ignored +allow_hdrfrom_mismatch = false; + +# If true, multiple from headers are allowed (but only first is used) +allow_hdrfrom_multiple = false; + +# If true, username does not need to contain matching domain +allow_username_mismatch = false; + +# Default path to key, can include '$domain' and '$selector' variables +path = "/var/lib/rspamd/dkim/$domain.$selector.key"; + +# Default selector to use +selector = "dkim"; + +# If false, messages from authenticated users are not selected for signing +sign_authenticated = true; + +# If false, messages from local networks are not selected for signing +sign_local = true; + +# Map file of IP addresses/subnets to consider for signing +# sign_networks = "/some/file"; # or url + +# Symbol to add when message is signed +symbol = "DKIM_SIGNED"; + +# Whether to fallback to global config +try_fallback = true; + +# Domain to use for DKIM signing: can be "header" (MIME From), "envelope" (SMTP From), "recipient" (SMTP To), "auth" (SMTP username) or directly specified domain name +use_domain = "header"; + +# Domain to use for DKIM signing when sender is in sign_networks ("header"/"envelope"/"auth") +#use_domain_sign_networks = "header"; + +# Domain to use for DKIM signing when sender is a local IP ("header"/"envelope"/"auth") +#use_domain_sign_local = "header"; + +# Whether to normalise domains to eSLD +use_esld = true; + +# Whether to get keys from Redis +use_redis = false; + +# Hash for DKIM keys in Redis +key_prefix = "DKIM_KEYS"; + +# map of domains -> names of selectors (since rspamd 1.5.3) +#selector_map = "/etc/rspamd/dkim_selectors.map"; + +# map of domains -> paths to keys (since rspamd 1.5.3) +#path_map = "/etc/rspamd/dkim_paths.map"; + +# If `true` get pubkey from DNS record and check if it matches private key +check_pubkey = false; +# Set to `false` if you want to skip signing if public and private keys mismatch +allow_pubkey_mismatch = true; + +# Domain specific settings +domain { + # Domain name is used as key + {{ mail_domain }} { + + # Private key path + path = "/var/lib/rspamd/dkim/{{ mail_domain }}.key"; + + # Selector + selector = "ds"; + } +} + diff --git a/deploy/dev/docker-compose.yml b/deploy/dev/docker-compose.yml index e44c276..dff5ab3 100644 --- a/deploy/dev/docker-compose.yml +++ b/deploy/dev/docker-compose.yml @@ -3,15 +3,20 @@ services: build: context: ../../core dockerfile: ../deploy/dev/Dockerfile.backend - command: bash -c 'python manage.py migrate && python testdata.py && python manage.py runserver 0.0.0.0:8000' + command: bash -c 'python manage.py migrate && python manage.py runserver 0.0.0.0:8000' environment: - HTTP_HOST=core - - DB_FILE=dev.db + - DB_HOST=db + - DB_PORT=3306 + - DB_NAME=system3 + - DB_USER=system3 + - DB_PASSWORD=system3 volumes: - ../../core:/code - - ../testdata.py:/code/testdata.py ports: - "8000:8000" + depends_on: + - db frontend: build: @@ -26,3 +31,18 @@ services: - "8080:8080" depends_on: - core + + db: + image: mariadb + environment: + MARIADB_RANDOM_ROOT_PASSWORD: true + MARIADB_DATABASE: system3 + MARIADB_USER: system3 + MARIADB_PASSWORD: system3 + volumes: + - mariadb_data:/var/lib/mysql + ports: + - "3306:3306" + +volumes: + mariadb_data: \ No newline at end of file diff --git a/deploy/testdata.py b/deploy/testdata.py deleted file mode 100644 index dca385f..0000000 --- a/deploy/testdata.py +++ /dev/null @@ -1,88 +0,0 @@ -import os - - -def setup(): - from authentication.models import ExtendedUser, EventPermission - from inventory.models import Event - from django.contrib.auth.models import Permission, Group - permissions = ['add_item', 'view_item', 'view_file', 'delete_item', 'change_item'] - if not ExtendedUser.objects.filter(username='admin').exists(): - admin = ExtendedUser.objects.create_superuser('admin', 'admin@example.com', 'admin') - admin.set_password('admin') - admin.user_permissions.add(*Permission.objects.all()) - admin.save() - - if not ExtendedUser.objects.filter(username='testuser').exists(): - testuser = ExtendedUser.objects.create_user('testuser', 'testuser@example.com', 'testuser') - testuser.set_password('testuser') - testuser.user_permissions.add(*Permission.objects.all()) - testuser.save() - - team = Group.objects.get(name='Team') - team.permissions.add( - *Permission.objects.all() - ) - - if not ExtendedUser.objects.filter(username='testuser2').exists(): - testuser2 = ExtendedUser.objects.create_user('testuser2', 'testuser2@example.com', 'testuser2') - testuser2.set_password('testuser2') - testuser2.groups.add(team) - testuser2.save() - - event1 = Event.objects.get_or_create(id=1, name='first test event', slug='TEST1', - start='2023-12-18 00:00:00.000000', end='2023-12-27 00:00:00.000000', - pre_start='2023-12-31 00:00:00.000000', post_end='2024-01-04 00:00:00.000000')[ - 0] - - event2 = Event.objects.get_or_create(id=2, name='second test event', slug='TEST2', - start='2024-12-18 00:00:00.000000', end='2024-12-27 00:00:00.000000', - pre_start='2024-12-31 00:00:00.000000', post_end='2025-01-04 00:00:00.000000')[ - 0] - - # for permission in permissions: - # EventPermission.objects.create(event=event_37c3, user=foo, - # permission=Permission.objects.get(codename=permission)) - - from tickets.models import IssueThread - - from mail.models import Email - - issue_thread = IssueThread.objects.get_or_create( - id=1, - name="test", - event=Event.objects.get(slug='TEST1') - )[0] - mail1 = Email.objects.get_or_create( - id=1, - subject='test subject', - body='test', - sender='test1@test', - recipient='test2@test', - issue_thread=issue_thread, - )[0] - mail1_reply = Email.objects.get_or_create( - id=2, - subject='Message received', - body='Thank you for your message.', - sender='test2@test', - recipient='test1@test', - in_reply_to=mail1.reference, - issue_thread=issue_thread, - )[0] - - -def main(): - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "core.settings") - import django - - django.setup() - - from django.core.management import call_command - call_command('migrate') - - setup() - print('testdata initialised') - - -if __name__ == '__main__': - main() diff --git a/deploy/testing/docker-compose.yml b/deploy/testing/docker-compose.yml index b41dd63..e93e901 100644 --- a/deploy/testing/docker-compose.yml +++ b/deploy/testing/docker-compose.yml @@ -20,7 +20,7 @@ services: build: context: ../../core dockerfile: ../deploy/testing/Dockerfile.backend - command: bash -c 'python manage.py migrate && python testdata.py && python /code/server.py' + command: bash -c 'python manage.py migrate && python /code/server.py' environment: - HTTP_HOST=core - REDIS_HOST=redis @@ -29,16 +29,13 @@ services: - DB_NAME=system3 - DB_USER=system3 - DB_PASSWORD=system3 - - MAIL_DOMAIN=mail:1025 volumes: - ../../core:/code - - ../testdata.py:/code/testdata.py ports: - "8000:8000" depends_on: - db - redis - - mail frontend: build: @@ -54,19 +51,5 @@ services: depends_on: - core - mail: - image: docker.io/axllent/mailpit - volumes: - - mailpit_data:/data - ports: - - 8025:8025 - - 1025:1025 - environment: - MP_MAX_MESSAGES: 5000 - MP_DATABASE: /data/mailpit.db - MP_SMTP_AUTH_ACCEPT_ANY: 1 - MP_SMTP_AUTH_ALLOW_INSECURE: 1 - volumes: - mariadb_data: - mailpit_data: + mariadb_data: \ No newline at end of file