initial commit

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

roles/caddy/defaults/main.yml

@@ -0,0 +1,14 @@
+---
+# Caddy metrics port (scraped by Prometheus)
+caddy_metrics_port: 9000
+
+# Static sites served by Caddy — override in config.yml
+# Each entry gets /srv/caddy/sites/<domain>/ and www → apex redirect
+caddy_sites:
+  - "{{ domain }}"
+
+# Service domain defaults — override individually in config.yml or overrides.yml
+webmail_domain: "webmail.{{ domain }}"
+rspamd_domain: "rspamd.{{ domain }}"
+goaccess_domain: "stats.{{ domain }}"
+grafana_domain: "watch.{{ domain }}"

roles/caddy/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Restart Caddy
+  # full restart (not caddy reload) avoids stale bind-mount inodes when Caddyfile changes on host
+  community.docker.docker_compose_v2:
+    project_src: /srv/caddy
+    state: present
+    build: never
+

roles/caddy/tasks/main.yml

@@ -0,0 +1,104 @@
+- name: Allow HTTP traffic
+  ufw:
+    rule: allow
+    port: 80
+    proto: tcp
+
+- name: Allow HTTPS traffic
+  ufw:
+    rule: allow
+    port: 443
+    proto: tcp
+
+- name: Allow HTTPS/QUIC (HTTP/3) traffic
+  ufw:
+    rule: allow
+    port: 443
+    proto: udp
+
+- name: Create Caddy directories
+  file:
+    path: "/srv/caddy/{{ item }}"
+    state: directory
+    owner: root
+    group: docker
+    mode: "0755"
+  loop:
+    - ""
+    - data
+    - config
+    - sites
+
+- name: Create site roots
+  file:
+    path: "/srv/caddy/sites/{{ item }}"
+    state: directory
+    owner: root
+    group: docker
+    mode: "0775" # also allow members of the docker group to write
+  loop: "{{ caddy_sites }}"
+
+- name: Install Caddyfile
+  template:
+    src: Caddyfile.j2
+    dest: /srv/caddy/Caddyfile
+    owner: root
+    group: docker
+    mode: "0644"
+  notify: Restart Caddy
+  tags: config
+
+- name: Check for cached goaccess hash
+  ansible.builtin.stat:
+    path: /srv/caddy/.goaccess_hash
+  register: _goaccess_hash_stat
+  when: enable_goaccess | default(true)
+
+- name: Read goaccess hash from cache
+  ansible.builtin.slurp:
+    src: /srv/caddy/.goaccess_hash
+  register: _goaccess_hash_file
+  when: enable_goaccess | default(true) and _goaccess_hash_stat.stat.exists
+
+- name: Set goaccess hash fact from cache
+  ansible.builtin.set_fact:
+    caddy_goaccess_hash_stdout: "{{ _goaccess_hash_file.content | b64decode | trim }}"
+  when: enable_goaccess | default(true) and _goaccess_hash_stat.stat.exists
+
+- name: Generate goaccess password hash
+  ansible.builtin.command:
+    argv:
+      - docker
+      - run
+      - --rm
+      - "caddy:{{ caddy_version }}"
+      - caddy
+      - hash-password
+      - --plaintext
+      - "{{ goaccess_password }}"
+  register: _goaccess_hash_result
+  changed_when: false
+  no_log: true
+  when: enable_goaccess | default(true) and not _goaccess_hash_stat.stat.exists
+
+- name: Cache goaccess hash
+  ansible.builtin.copy:
+    content: "{{ _goaccess_hash_result.stdout }}"
+    dest: /srv/caddy/.goaccess_hash
+    mode: "0600"
+  when: enable_goaccess | default(true) and not _goaccess_hash_stat.stat.exists
+
+- name: Set goaccess hash fact from generation
+  ansible.builtin.set_fact:
+    caddy_goaccess_hash_stdout: "{{ _goaccess_hash_result.stdout }}"
+  when: enable_goaccess | default(true) and not _goaccess_hash_stat.stat.exists
+
+- name: Deploy Caddy compose.yml
+  template:
+    src: compose.yml.j2
+    dest: /srv/caddy/compose.yml
+    owner: root
+    group: docker
+    mode: "0644"
+  notify: Restart Caddy
+  tags: config

roles/caddy/templates/Caddyfile.j2

@@ -0,0 +1,124 @@
+{
+	email {{ admin_user }}@{{ domain }}
+	log {
+		output stdout
+	}
+	metrics {
+		per_host
+	}
+}
+
+(access_log) {
+	log
+}
+
+:{{ caddy_metrics_port }} {
+	metrics
+}
+
+{% for site in caddy_sites %}
+# Redirect www → apex
+www.{{ site }} {
+	import access_log
+	redir https://{{ site }}{uri} permanent
+}
+
+{{ site }} {
+	import access_log
+	root * /srv/sites/{{ site }}
+	encode zstd gzip
+	file_server
+
+	header {
+		Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+		X-Content-Type-Options "nosniff"
+		X-Frame-Options "DENY"
+		Referrer-Policy "strict-origin-when-cross-origin"
+	}
+{% if site == domain and enable_tuwunel | default(false) %}
+
+	handle /.well-known/matrix/server {
+		header Content-Type application/json
+		respond `{"m.server": "{{ tuwunel_domain }}:443"}`
+	}
+
+	handle /.well-known/matrix/client {
+		header Content-Type application/json
+		header Access-Control-Allow-Origin *
+		respond `{"m.homeserver": {"base_url": "https://{{ tuwunel_domain }}"}}`
+	}
+{% endif %}
+}
+
+{% endfor %}
+{% if enable_mail | default(false) %}
+{{ webmail_domain }} {
+	import access_log
+	reverse_proxy rainloop:{{ rainloop_port }}
+}
+
+{{ rspamd_domain }} {
+	import access_log
+	reverse_proxy mailserver:{{ rspamd_port }}
+}
+
+{% endif %}
+{% if enable_forgejo | default(false) %}
+{{ forgejo_domain }} {
+	import access_log
+	reverse_proxy forgejo:{{ forgejo_port }}
+
+	header {
+		Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+		X-Content-Type-Options "nosniff"
+		X-Frame-Options "SAMEORIGIN"
+		Referrer-Policy "strict-origin-when-cross-origin"
+	}
+}
+
+{% endif %}
+{% if enable_monitoring | default(false) %}
+{{ grafana_domain }} {
+	import access_log
+	reverse_proxy grafana:{{ grafana_port }} {
+		header_up Host {host}
+		header_up X-Real-IP {remote_host}
+	}
+}
+
+{% endif %}
+{% if enable_tuwunel | default(false) %}
+{{ tuwunel_domain }} {
+	import access_log
+	reverse_proxy tuwunel:{{ tuwunel_port }}
+
+	header {
+		Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+		X-Content-Type-Options "nosniff"
+		X-Frame-Options "DENY"
+		Referrer-Policy "strict-origin-when-cross-origin"
+	}
+}
+
+{% endif %}
+{% if enable_radicale | default(false) %}
+{{ radicale_domain }} {
+	import access_log
+
+	redir /.well-known/caldav / permanent
+	redir /.well-known/carddav / permanent
+
+	reverse_proxy radicale:{{ radicale_port }}
+}
+
+{% endif %}
+{% if enable_goaccess | default(false) %}
+{{ goaccess_domain }} {
+	import access_log
+	root * /srv/goaccess/reports
+	file_server browse
+	basic_auth {
+		{$GOACCESS_USER} {$GOACCESS_HASH}
+	}
+}
+{% endif %}

roles/caddy/templates/compose.yml.j2

@@ -0,0 +1,67 @@
+services:
+  caddy:
+    image: caddy:{{ caddy_version }}
+    container_name: caddy
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+      - "443:443/udp"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q -O /dev/null http://localhost:{{ caddy_metrics_port }}/metrics || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    volumes:
+      - /srv/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
+      - /srv/caddy/data:/data
+      - /srv/caddy/config:/config
+      - /srv/caddy/sites:/srv/sites:ro
+      - /srv/goaccess/reports:/srv/goaccess/reports:ro
+    environment:
+{% if enable_goaccess | default(true) %}
+      GOACCESS_USER: "{{ goaccess_user }}"
+      GOACCESS_HASH: "{{ caddy_goaccess_hash_stdout | replace('$', '$$') }}"
+{% endif %}
+    networks:
+      - caddy
+{% if enable_mail | default(true) %}
+      - webmail
+{% endif %}
+{% if enable_forgejo | default(true) %}
+      - git
+{% endif %}
+{% if enable_monitoring | default(true) %}
+      - monitoring
+{% endif %}
+{% if enable_tuwunel | default(true) %}
+      - tuwunel
+{% endif %}
+{% if enable_radicale | default(false) %}
+      - radicale
+{% endif %}
+
+networks:
+  caddy:
+    external: true
+{% if enable_mail | default(true) %}
+  webmail:
+    external: true
+{% endif %}
+{% if enable_forgejo | default(true) %}
+  git:
+    external: true
+{% endif %}
+{% if enable_monitoring | default(true) %}
+  monitoring:
+    external: true
+{% endif %}
+{% if enable_tuwunel | default(true) %}
+  tuwunel:
+    external: true
+{% endif %}
+{% if enable_radicale | default(false) %}
+  radicale:
+    external: true
+{% endif %}

roles/common/tasks/main.yml

@@ -0,0 +1,11 @@
+- name: Ensure base packages are installed
+  apt:
+    name:
+      - ca-certificates
+      - curl
+      - gnupg
+      - lsb-release
+      - rsync
+    state: present
+    update_cache: true
+

roles/diun/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Check schedule (cron format)
+diun_schedule: "0 */6 * * *"
+
+# Notification toggles
+diun_notify_matrix: false

roles/diun/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+- name: restart diun
+  community.docker.docker_compose_v2:
+    project_src: /srv/diun
+    state: present
+    recreate: always
+    build: never

roles/diun/tasks/main.yml

@@ -0,0 +1,29 @@
+---
+- name: Create diun directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - /srv/diun
+    - /srv/diun/data
+
+- name: Create diun configuration
+  template:
+    src: diun.yml.j2
+    dest: /srv/diun/data/diun.yml
+    mode: '0644'
+  notify: restart diun
+
+- name: Create compose file
+  template:
+    src: compose.yml.j2
+    dest: /srv/diun/compose.yml
+    mode: '0644'
+  notify: restart diun
+
+- name: Deploy diun
+  community.docker.docker_compose_v2:
+    project_src: /srv/diun
+    state: present
+    build: never

roles/diun/templates/compose.yml.j2

@@ -0,0 +1,21 @@
+services:
+  diun:
+    image: crazymax/diun:{{ diun_version }}
+    container_name: diun
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /srv/diun/data:/data
+    environment:
+      CONFIG: /data/diun.yml
+      TZ: {{ timezone | default('UTC') }}
+      LOG_LEVEL: info
+      LOG_JSON: "true"
+    labels:
+      diun.enable: "false"
+    networks:
+      - monitoring
+
+networks:
+  monitoring:
+    external: true

roles/diun/templates/diun.yml.j2

@@ -0,0 +1,30 @@
+watch:
+  schedule: "{{ diun_schedule | default('0 */6 * * *') }}"
+  firstCheckNotif: false
+
+providers:
+  docker:
+    watchByDefault: true
+    watchStopped: false
+
+{% if diun_notify_email | default(false) or diun_notify_matrix | default(false) %}
+notif:
+{% if diun_notify_email | default(false) %}
+  mail:
+    host: {{ mail_hostname }}
+    port: 587
+    ssl: false
+    insecureSkipVerify: false
+    username: {{ diun_email_user }}
+    password: {{ diun_email_password }}
+    from: {{ diun_email_user }}
+    to: {{ diun_email_to }}
+{% endif %}
+{% if diun_notify_matrix | default(false) %}
+  matrix:
+    homeserverURL: https://{{ tuwunel_domain }}
+    user: {{ diun_matrix_user }}
+    password: {{ diun_matrix_password }}
+    roomID: {{ diun_matrix_room_id }}
+{% endif %}
+{% endif %}

roles/dns/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+dns_zones: []

roles/dns/tasks/main.yml

@@ -0,0 +1,25 @@
+---
+- name: Ensure DNS zone exists
+  hetzner.hcloud.zone:
+    name: "{{ item.zone }}"
+    mode: primary
+    api_token: "{{ hcloud_token }}"
+    state: present
+  loop: "{{ dns_zones }}"
+  loop_control:
+    label: "{{ item.zone }}"
+  tags: dns
+
+- name: Manage DNS records
+  hetzner.hcloud.zone_rrset:
+    zone: "{{ item.0.zone }}"
+    name: "{{ item.1.name }}"
+    type: "{{ item.1.type }}"
+    ttl: "{{ item.1.ttl | default(300) }}"
+    records: "{{ item.1.records }}"
+    api_token: "{{ hcloud_token }}"
+    state: present
+  loop: "{{ dns_zones | subelements('records') }}"
+  loop_control:
+    label: "{{ item.0.zone }} {{ item.1.name }} {{ item.1.type }}"
+  tags: dns

roles/docker/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: restart docker
+  systemd:
+    name: docker
+    state: restarted
+

roles/docker/tasks/main.yml

@@ -0,0 +1,76 @@
+---
+- name: Install prerequisite packages
+  apt:
+    name:
+      - ca-certificates
+      - curl
+      - gnupg
+      - lsb-release
+    state: present
+    update_cache: true
+
+- name: Add Docker GPG key
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    state: present
+
+- name: Add Docker apt repository
+  apt_repository:
+    repo: >
+      deb [arch=amd64]
+      https://download.docker.com/linux/ubuntu
+      {{ ansible_facts['distribution_release'] }} stable
+    state: present
+    filename: docker
+
+- name: Install Docker engine and compose plugin
+  apt:
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+    state: present
+    update_cache: true
+
+- name: Enable and start Docker
+  systemd:
+    name: docker
+    enabled: true
+    state: started
+
+- name: Ensure Docker config directory
+  file:
+    path: /etc/docker
+    state: directory
+    mode: '0755'
+
+- name: Configure Docker to use journald
+  copy:
+    dest: /etc/docker/daemon.json
+    content: |
+      {
+        "log-driver": "journald"
+      }
+    mode: '0644'
+  notify: restart docker
+
+- name: Add admin user to docker group
+  user:
+    name: "{{ admin_user }}"
+    groups: docker
+    append: true
+
+# ---- Filesystem layout ----
+
+- name: Create base directory
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - /srv
+

roles/docker_network/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+# caddy is always deployed
+- name: Ensure caddy Docker network exists
+  community.docker.docker_network:
+    name: caddy
+    state: present
+
+- name: Ensure mail Docker networks exist
+  community.docker.docker_network:
+    name: "{{ item }}"
+    state: present
+  loop:
+    - mail
+    - webmail
+  when: enable_mail | default(true)
+
+- name: Ensure monitoring Docker network exists
+  community.docker.docker_network:
+    name: monitoring
+    state: present
+  when: enable_monitoring | default(true)
+
+- name: Ensure git Docker network exists
+  community.docker.docker_network:
+    name: git
+    state: present
+  when: enable_forgejo | default(true)
+
+- name: Ensure tuwunel Docker network exists
+  community.docker.docker_network:
+    name: tuwunel
+    state: present
+  when: enable_tuwunel | default(true)
+
+- name: Ensure radicale Docker network exists
+  community.docker.docker_network:
+    name: radicale
+    state: present
+  when: enable_radicale | default(false)

roles/fail2ban/files/filter.d/caddy-auth.conf

@@ -0,0 +1,3 @@
+[Definition]
+# matches 401 responses using remote_ip extracted from X-Real-IP by Caddy in JSON access logs
+failregex = .*"remote_ip":"<HOST>".*"status":401.*

roles/fail2ban/files/filter.d/caddy-scanners.conf

@@ -0,0 +1,3 @@
+[Definition]
+# admin.php is intentionally specific; broader /admin would match legitimate paths (e.g. /admin-api)
+failregex = .*"remote_ip":"<HOST>".*"uri":"\/(wp-admin|wp-login|phpmyadmin|xmlrpc|\.env|\.git|cgi-bin|admin\.php|setup\.php|eval-stdin).*".*

roles/fail2ban/files/filter.d/docker-mailserver.conf

@@ -0,0 +1,8 @@
+[Definition]
+# Postfix SASL auth failures: warning: unknown[IP]: SASL ... authentication failed
+failregex = ^.*warning: .*\[<HOST>\]: SASL .* authentication failed.*$
+# Dovecot auth failures: auth failed ... rip=IP
+            ^.*dovecot: (?:imap|pop3)-login: .*\(auth failed.*rip=<HOST>,.*$
+
+ignoreregex =
+

roles/fail2ban/files/filter.d/forgejo-auth.conf

@@ -0,0 +1,4 @@
+[Definition]
+# Matches both web login failures and SSH auth failures
+failregex = ^.*Failed authentication attempt from <HOST>(:\d+)?$
+            ^.*Failed login for user '[^']*' from <HOST>$

roles/fail2ban/files/jail.d/caddy.conf

@@ -0,0 +1,16 @@
+[caddy-scanners]
+enabled = true
+journalmatch = CONTAINER_NAME=caddy
+filter = caddy-scanners
+maxretry = 3
+findtime = 10m
+bantime = 24h
+
+# high maxretry/short bantime: Grafana auth can be slow; strict limits cause false positives
+[caddy-auth]
+enabled = true
+journalmatch = CONTAINER_NAME=caddy
+filter = caddy-auth
+maxretry = 40
+findtime = 10m
+bantime = 1h

roles/fail2ban/files/jail.d/forgejo.conf

@@ -0,0 +1,8 @@
+[forgejo]
+enabled = true
+backend = systemd
+journalmatch = CONTAINER_NAME=forgejo
+filter = forgejo-auth
+maxretry = 5
+findtime = 10m
+bantime = 24h

roles/fail2ban/files/jail.d/mailserver.conf

@@ -0,0 +1,9 @@
+[mailserver]
+enabled = true
+backend = systemd
+journalmatch = CONTAINER_NAME=mailserver
+filter = docker-mailserver
+maxretry = 5
+findtime = 10m
+bantime = 24h
+

roles/fail2ban/handlers/main.yml

@@ -0,0 +1,5 @@
+- name: Reload fail2ban
+  service:
+    name: fail2ban
+    state: reloaded
+

roles/fail2ban/tasks/main.yml

@@ -0,0 +1,36 @@
+- name: Ensure fail2ban directories exist
+  file:
+    path: "/etc/fail2ban/{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - ""
+    - jail.d
+    - filter.d
+
+- name: Remove obsolete grafana fail2ban configs
+  file:
+    path: "/etc/fail2ban/{{ item }}"
+    state: absent
+  loop:
+    - jail.d/grafana.conf
+    - filter.d/grafana-auth.conf
+  notify: Reload fail2ban
+
+- name: Deploy fail2ban jail.local
+  template:
+    src: jail.local.j2
+    dest: /etc/fail2ban/jail.local
+    mode: '0644'
+  notify: Reload fail2ban
+
+- name: Copy fail2ban jail and filter configs
+  copy:
+    src: "{{ item }}"
+    dest: "/etc/fail2ban/{{ item | regex_replace('^.*/files/', '') }}"
+    mode: '0644'
+  with_fileglob:
+    - "{{ role_path }}/files/jail.d/*"
+    - "{{ role_path }}/files/filter.d/*"
+  notify: Reload fail2ban
+

roles/fail2ban/templates/jail.local.j2

@@ -0,0 +1,8 @@
+[DEFAULT]
+backend = systemd
+bantime = 24h
+findtime = 10m
+maxretry = 5
+{% if fail2ban_ignoreip | default([]) | length > 0 %}
+ignoreip = 127.0.0.1/8 ::1 {{ fail2ban_ignoreip | default([]) | join(' ') }}
+{% endif %}

roles/forgejo/defaults/main.yml

@@ -0,0 +1,26 @@
+---
+# Display name shown in the UI, emails, and page title
+forgejo_app_name: "Forgejo"
+
+# Ports (internal to docker network)
+forgejo_port: 3000
+forgejo_ssh_port: 2222
+
+# Registration and access
+forgejo_disable_registration: true
+forgejo_require_signin: false
+
+# Timezone for the Forgejo UI — defaults to the system timezone
+forgejo_timezone: "{{ timezone | default('UTC') }}"
+
+# Email notifications (set to true and configure smtp vars to enable)
+forgejo_mailer_enabled: false
+# forgejo_smtp_host: mail.example.com
+# forgejo_smtp_port: 587
+# forgejo_smtp_user: notifications@example.com
+# forgejo_mailer_from: "Forgejo <notifications@example.com>"
+# forgejo_smtp_password: defined in vault.yml
+
+# Actions runner
+forgejo_runner_name: default-runner
+forgejo_runner_labels: "docker:docker://node:20-bookworm,ubuntu-latest:docker://ubuntu:latest,ubuntu-22.04:docker://ubuntu:22.04"

roles/forgejo/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart forgejo
+  community.docker.docker_compose_v2:
+    project_src: /srv/forgejo
+    state: restarted
+    build: never

roles/forgejo/tasks/main.yml

@@ -0,0 +1,128 @@
+---
+- name: Allow Forgejo SSH traffic
+  ufw:
+    rule: allow
+    port: "{{ forgejo_ssh_port }}"
+    proto: tcp
+
+- name: Create Forgejo directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - /srv/forgejo
+
+- name: Create Forgejo data directory
+  ansible.builtin.file:
+    path: /srv/forgejo/data
+    state: directory
+    mode: '0755'
+
+# stat+chown: avoids UID/GID lookup warnings for container-internal UIDs not present on host
+- name: Stat Forgejo data directory
+  ansible.builtin.stat:
+    path: /srv/forgejo/data
+  register: forgejo_data_stat
+
+- name: Set Forgejo data directory ownership
+  ansible.builtin.command: chown 1000:1000 /srv/forgejo/data
+  when: forgejo_data_stat.stat.uid != 1000 or forgejo_data_stat.stat.gid != 1000
+
+- name: Create runner data directory
+  ansible.builtin.file:
+    path: /srv/forgejo/runner
+    state: directory
+    mode: '0755'
+  when: enable_forgejo_runner | default(true)
+
+# stat+chown: avoids UID/GID lookup warnings for container-internal UIDs not present on host
+- name: Stat runner data directory
+  ansible.builtin.stat:
+    path: /srv/forgejo/runner
+  register: forgejo_runner_stat
+  when: enable_forgejo_runner | default(true)
+
+- name: Set runner data directory ownership
+  ansible.builtin.command: chown 1000:1000 /srv/forgejo/runner
+  when: (enable_forgejo_runner | default(true)) and (forgejo_runner_stat.stat.uid != 1000 or forgejo_runner_stat.stat.gid != 1000)
+
+- name: Deploy Forgejo docker-compose file
+  ansible.builtin.template:
+    src: compose.yml.j2
+    dest: /srv/forgejo/compose.yml
+    mode: '0644'
+  notify: Restart forgejo
+
+- name: Deploy Forgejo app.ini configuration
+  ansible.builtin.template:
+    src: app.ini.j2
+    dest: /srv/forgejo/data/gitea/conf/app.ini
+    mode: '0644'
+  notify: Restart forgejo
+
+- name: Start Forgejo server
+  community.docker.docker_compose_v2:
+    project_src: /srv/forgejo
+    services:
+      - forgejo
+    state: present
+    build: never
+  register: forgejo_output
+
+- name: Wait for Forgejo to be ready
+  ansible.builtin.uri:
+    url: "http://localhost:{{ forgejo_port }}"
+    status_code: 200
+  retries: 30
+  delay: 2
+  when: forgejo_output.changed
+
+# Runner registration (one-time)
+- name: Check if runner is already registered
+  ansible.builtin.stat:
+    path: /srv/forgejo/runner/.runner
+  register: runner_file
+  when: enable_forgejo_runner | default(true)
+
+- name: Generate runner registration token
+  community.docker.docker_container_exec:
+    container: forgejo
+    command: forgejo forgejo-cli actions generate-runner-token
+    user: git
+  register: runner_token
+  when:
+    - enable_forgejo_runner | default(true)
+    - not runner_file.stat.exists
+
+- name: Deploy runner config
+  ansible.builtin.template:
+    src: runner-config.yml.j2
+    dest: /srv/forgejo/runner/config.yml
+    mode: '0644'
+  when: enable_forgejo_runner | default(true)
+  notify: Restart forgejo
+
+- name: Register Forgejo runner
+  ansible.builtin.command:
+    cmd: >-
+      docker run --rm
+      --network git
+      -v /srv/forgejo/runner:/data
+      code.forgejo.org/forgejo/runner:{{ forgejo_runner_version }}
+      forgejo-runner register --no-interactive
+      --instance http://forgejo:3000
+      --token {{ runner_token.stdout | trim }}
+      --name {{ forgejo_runner_name }}
+      --labels {{ forgejo_runner_labels }}
+  when:
+    - enable_forgejo_runner | default(true)
+    - not runner_file.stat.exists
+  notify: Restart forgejo
+
+- name: Start all Forgejo services
+  community.docker.docker_compose_v2:
+    project_src: /srv/forgejo
+    state: present
+    build: never
+  when: enable_forgejo_runner | default(true)

roles/forgejo/templates/app.ini.j2

@@ -0,0 +1,71 @@
+APP_NAME = {{ forgejo_app_name }}
+RUN_MODE = prod
+WORK_PATH = /data/gitea
+
+[server]
+DOMAIN = {{ forgejo_domain }}
+ROOT_URL = https://{{ forgejo_domain }}/
+HTTP_PORT = 3000
+SSH_DOMAIN = {{ forgejo_domain }}
+SSH_PORT = {{ forgejo_ssh_port }}
+START_SSH_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+PATH = /data/gitea/gitea.db
+
+[repository]
+ROOT = /data/git/repositories
+
+[log]
+MODE = console
+LEVEL = Info
+
+[security]
+INSTALL_LOCK = true
+SECRET_KEY = {{ forgejo_secret_key }}
+INTERNAL_TOKEN = {{ forgejo_internal_token }}
+
+[service]
+DISABLE_REGISTRATION = {{ forgejo_disable_registration }}
+REQUIRE_SIGNIN_VIEW = {{ forgejo_require_signin }}
+DEFAULT_KEEP_EMAIL_PRIVATE = true
+
+[mailer]
+ENABLED = {{ forgejo_mailer_enabled }}
+{% if forgejo_mailer_enabled %}
+FROM = {{ forgejo_mailer_from }}
+PROTOCOL = smtp
+SMTP_ADDR = {{ forgejo_smtp_host }}
+SMTP_PORT = {{ forgejo_smtp_port }}
+USER = {{ forgejo_smtp_user }}
+PASSWD = {{ forgejo_smtp_password }}
+{% endif %}
+
+[session]
+PROVIDER = file
+
+[picture]
+DISABLE_GRAVATAR = false
+ENABLE_FEDERATED_AVATAR = true
+
+[openid]
+ENABLE_OPENID_SIGNIN = false
+ENABLE_OPENID_SIGNUP = false
+
+[oauth2]
+JWT_SECRET = {{ forgejo_jwt_secret }}
+
+[attachment]
+ENABLED = true
+MAX_SIZE = 50
+
+[time]
+DEFAULT_UI_LOCATION = {{ forgejo_timezone }}
+
+[metrics]
+ENABLED = true
+
+[actions]
+ENABLED = true
+DEFAULT_ACTIONS_URL = https://code.forgejo.org

roles/forgejo/templates/compose.yml.j2

@@ -0,0 +1,49 @@
+services:
+  forgejo:
+    image: codeberg.org/forgejo/forgejo:{{ forgejo_version }}
+    container_name: forgejo
+    restart: unless-stopped
+    environment:
+      - USER_UID=1000
+      - USER_GID=1000
+    volumes:
+      - /srv/forgejo/data:/data
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+    ports:
+      - "{{ forgejo_port }}:3000"
+      - "{{ forgejo_ssh_port }}:2222"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q -O /dev/null http://localhost:3000/api/v1/version || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 60s
+    networks:
+      - git
+      - monitoring
+
+{% if enable_forgejo_runner | default(true) %}
+  runner:
+    image: code.forgejo.org/forgejo/runner:{{ forgejo_runner_version }}
+    container_name: forgejo-runner
+    restart: unless-stopped
+    user: "0:0"
+    depends_on:
+      forgejo:
+        condition: service_healthy
+    volumes:
+      - /srv/forgejo/runner:/data
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      DOCKER_HOST: unix:///var/run/docker.sock
+    command: forgejo-runner daemon --config /data/config.yml
+    networks:
+      - git
+{% endif %}
+
+networks:
+  git:
+    external: true
+  monitoring:
+    external: true

roles/forgejo/templates/runner-config.yml.j2

@@ -0,0 +1,14 @@
+log:
+  level: info
+
+runner:
+  file: .runner
+  capacity: 1
+  timeout: 3h
+
+container:
+  # job containers must be on this network to resolve the forgejo hostname for git operations
+  network: "git"
+  privileged: false
+  valid_volumes:
+    - '**'

roles/goaccess/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# Time to sync access logs and regenerate reports (daily)
+goaccess_sync_time: "05:00:00"

roles/goaccess/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

roles/goaccess/tasks/main.yml

@@ -0,0 +1,91 @@
+---
+- name: Install GoAccess and jq
+  ansible.builtin.apt:
+    name:
+      - goaccess
+      - jq
+    state: present
+
+- name: Create GoAccess directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - /srv/goaccess
+    - /srv/goaccess/data
+    - /srv/goaccess/reports
+
+- name: Deploy GoAccess config
+  ansible.builtin.template:
+    src: goaccess.conf.j2
+    dest: /srv/goaccess/goaccess.conf
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Deploy report generation script
+  ansible.builtin.template:
+    src: goaccess-report.sh.j2
+    dest: /usr/local/bin/goaccess-report
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Deploy report generation systemd service
+  ansible.builtin.template:
+    src: goaccess-report.service.j2
+    dest: /etc/systemd/system/goaccess-report.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Deploy report generation systemd timer
+  ansible.builtin.template:
+    src: goaccess-report.timer.j2
+    dest: /etc/systemd/system/goaccess-report.timer
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Deploy sync script
+  ansible.builtin.template:
+    src: goaccess-sync.sh.j2
+    dest: /usr/local/bin/goaccess-sync
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Deploy sync systemd service
+  ansible.builtin.template:
+    src: goaccess-sync.service.j2
+    dest: /etc/systemd/system/goaccess-sync.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Deploy sync systemd timer
+  ansible.builtin.template:
+    src: goaccess-sync.timer.j2
+    dest: /etc/systemd/system/goaccess-sync.timer
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Flush handlers to reload systemd
+  ansible.builtin.meta: flush_handlers
+
+- name: Enable and start GoAccess timers
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+  loop:
+    - goaccess-report.timer
+    - goaccess-sync.timer

roles/goaccess/templates/goaccess-report.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=GoAccess Report Generation
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/goaccess-report

roles/goaccess/templates/goaccess-report.sh.j2

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPORTS_DIR="/srv/goaccess/reports"
+DATA_DIR="/srv/goaccess/data"
+CONF="/srv/goaccess/goaccess.conf"
+
+SITES=(
+{% for site in goaccess_sites %}
+  "{{ site }}"
+{% endfor %}
+)
+
+# Fetch logs once from journald
+LOGS=$(journalctl CONTAINER_NAME=caddy --since "2 hours ago" --output=cat 2>/dev/null || true)
+
+# Skip if no logs
+if [[ -z "$LOGS" ]]; then
+  echo "No Caddy logs found, skipping."
+  exit 0
+fi
+
+# Generate per-site reports
+for site in "${SITES[@]}"; do
+  db_path="${DATA_DIR}/${site}"
+  mkdir -p "$db_path"
+
+  echo "$LOGS" \
+    | jq -c "select(.request.host == \"${site}\")" 2>/dev/null \
+    | goaccess \
+        --log-format=CADDY \
+        --persist \
+        --restore \
+        --db-path="$db_path" \
+        -o "${REPORTS_DIR}/${site}.html" \
+        - || echo "Warning: GoAccess failed for ${site}"
+done
+
+# Generate combined "all sites" report
+all_db="${DATA_DIR}/all"
+mkdir -p "$all_db"
+
+echo "$LOGS" \
+  | goaccess \
+      --log-format=CADDY \
+      --persist \
+      --restore \
+      --db-path="$all_db" \
+      -o "${REPORTS_DIR}/index.html" \
+      - || echo "Warning: GoAccess failed for combined report"
+
+echo "Reports generated at $(date -Iseconds)"

roles/goaccess/templates/goaccess-report.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=GoAccess Report Generation
+
+[Timer]
+OnCalendar=*-*-* *:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/goaccess/templates/goaccess-sync.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=GoAccess Report Sync to Storage Box
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/goaccess-sync

roles/goaccess/templates/goaccess-sync.sh.j2

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+rsync -az --delete \
+  -e "ssh -i {{ restic_ssh_key }} -p {{ restic_ssh_port }} -o StrictHostKeyChecking=no -o BatchMode=yes" \
+  /srv/goaccess/reports/ \
+  {{ restic_user }}@{{ restic_host }}:analytics/

roles/goaccess/templates/goaccess-sync.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=GoAccess Report Sync to Storage Box
+
+[Timer]
+OnCalendar=*-*-* {{ goaccess_sync_time }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target

roles/goaccess/templates/goaccess.conf.j2

@@ -0,0 +1,9 @@
+log-format CADDY
+date-format %s
+time-format %s
+
+# Persist parsed data to disk
+db-path /srv/goaccess/data
+
+# HTML report settings
+html-report-title GoAccess Analytics

roles/mail/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Rainloop webmail port (internal to docker network)
+rainloop_port: 8888
+
+# Rspamd web UI port (internal to docker network)
+rspamd_port: 11334

roles/mail/handlers/main.yml

@@ -0,0 +1,21 @@
+- name: Restart mailserver
+  community.docker.docker_compose_v2:
+    project_src: /srv/mail
+    services:
+      - mailserver
+    state: restarted
+    build: never
+
+- name: Restart Rainloop
+  community.docker.docker_compose_v2:
+    project_src: /srv/mail
+    services:
+      - rainloop
+    state: restarted
+    build: never
+
+- name: Restart mail stack
+  community.docker.docker_compose_v2:
+    project_src: /srv/mail
+    state: restarted
+    build: never

roles/mail/tasks/aliases.yml

@@ -0,0 +1,19 @@
+# read-only docker exec always reports changed; changed_when: false suppresses spurious output
+- name: List existing mail aliases
+  command: docker exec mailserver setup alias list
+  register: mail_alias_list
+  changed_when: false
+  tags:
+    - users
+
+- name: Create mail aliases if missing
+  command: >
+    docker exec mailserver
+    setup alias add {{ item.from }}
+    {{ item.to if item.to is string else item.to | join(',') }}
+  loop: "{{ mail_aliases }}"
+  when: item.from not in mail_alias_list.stdout
+  tags:
+    - users
+
+

roles/mail/tasks/main.yml

@@ -0,0 +1,139 @@
+- name: Allow SMTP traffic
+  ufw:
+    rule: allow
+    port: 25
+    proto: tcp
+
+- name: Allow mail submission traffic
+  ufw:
+    rule: allow
+    port: 587
+    proto: tcp
+
+- name: Allow IMAP over TLS traffic
+  ufw:
+    rule: allow
+    port: 993
+    proto: tcp
+
+- name: Create docker-mailserver directory
+  file:
+    path: "/srv/mail"
+    state: directory
+    owner: root
+    group: docker # to allow access to the compose file
+    mode: '0755'
+
+- name: Create docker-mailserver directories
+  file:
+    path: "/srv/mail/{{ item }}"
+    state: directory
+    owner: root
+    group: docker
+    mode: '0750'
+  loop:
+    - env
+    - config/rspamd/override.d
+
+- name: Create maillogs directory
+  file:
+    path: /srv/mail/maillogs
+    state: directory
+    mode: '0755'  # container startup script needs to traverse and chown subdirs
+
+# stat+chown: avoids UID/GID lookup warnings for container-internal UIDs not present on host
+- name: Stat maillogs directory
+  stat:
+    path: /srv/mail/maillogs
+  register: maillogs_stat
+
+- name: Set maillogs directory ownership
+  command: chown 113:0 /srv/mail/maillogs
+  when: maillogs_stat.stat.uid != 113 or maillogs_stat.stat.gid != 0
+
+- name: Create mailstate directory
+  file:
+    path: /srv/mail/mailstate
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'  # container startup script needs to traverse and chown subdirs
+
+- name: Create maildata directory
+  file:
+    path: /srv/mail/maildata
+    state: directory
+    mode: '0751'  # container startup script needs to traverse and chown subdirs
+
+- name: Create config directory
+  file:
+    path: /srv/mail/config
+    state: directory
+    mode: '0751'  # container startup script needs to traverse and chown subdirs
+
+- name: Create rainloop data directory
+  file:
+    path: /srv/mail/rainloop/data
+    state: directory
+    mode: '0755'
+
+# stat+chown: avoids UID/GID lookup warnings for container-internal UIDs not present on host
+- name: Stat rainloop data directory
+  stat:
+    path: /srv/mail/rainloop/data
+  register: rainloop_data_stat
+
+- name: Set rainloop data directory ownership
+  command: chown 991:991 /srv/mail/rainloop/data
+  when: rainloop_data_stat.stat.uid != 991 or rainloop_data_stat.stat.gid != 991
+
+- name: Ensure certbot is installed
+  apt:
+    name: certbot
+    state: present
+
+- name: Obtain a Let's Encrypt certificate for {{ mail_hostname }}
+  command: >
+    certbot certonly --standalone
+    -d {{ mail_hostname }}
+    --non-interactive --agree-tos -m postmaster@{{ domain }}
+  args:
+    creates: /etc/letsencrypt/live/{{ mail_hostname }}/fullchain.pem
+  tags: config
+
+- name: Deploy mail compose file
+  template:
+    src: compose.yml.j2
+    dest: /srv/mail/compose.yml
+  notify: Restart mail stack
+  tags: config
+
+- name: Deploy mailserver environment file
+  template:
+    src: mailserver.env.j2
+    dest: /srv/mail/env/mailserver.env
+    mode: '0640'
+    owner: root
+    group: docker
+  notify: Restart mailserver
+  tags: config
+
+- name: Deploy rspamd web UI config
+  template:
+    src: worker-controller.inc.j2
+    dest: /srv/mail/config/rspamd/override.d/worker-controller.inc
+    mode: '0644'
+  notify: Restart mailserver
+  tags: config
+
+- name: Start mailserver
+  community.docker.docker_compose_v2:
+    project_src: /srv/mail
+    state: present
+    build: never
+  tags: config
+
+- import_tasks: users.yml
+- import_tasks: aliases.yml
+# webmail interface
+- import_tasks: rainloop.yml

roles/mail/tasks/rainloop.yml

@@ -0,0 +1,21 @@
+- name: Ensure Rainloop allowed domains are set
+  ini_file:
+    path: /srv/mail/rainloop/data/_data_/_default_/configs/application.ini
+    section: security
+    option: AllowedDomains
+    value: "{{ mail_domains | join(',') }}"
+    backup: yes
+  notify:
+    - Restart Rainloop
+
+- name: Set proper mode of Rainloop data directory
+  file:
+    path: /srv/mail/rainloop
+    state: directory
+    recurse: yes
+    mode: u+rwX,g+rX
+
+# chown -R always exits 0; changed_when: false suppresses spurious "changed" in playbook output
+- name: Set proper ownership of Rainloop data directory
+  command: chown -R 991:991 /srv/mail/rainloop
+  changed_when: false

roles/mail/tasks/users.yml

@@ -0,0 +1,26 @@
+# read-only docker exec always reports changed; changed_when: false suppresses spurious output
+- name: Check if mail user exists
+  command: docker exec mailserver setup email list
+  register: mail_user_list
+  changed_when: false
+  tags:
+    - users
+
+- name: Create mail users if missing
+  ansible.builtin.command:
+    argv:
+      - docker
+      - exec
+      - mailserver
+      - setup
+      - email
+      - add
+      - "{{ item.address }}"
+      - "{{ item.password }}"
+  loop: "{{ mail_users }}"
+  when: item.address not in mail_user_list.stdout
+  no_log: true
+  ignore_errors: yes
+  tags:
+    - users
+

roles/mail/templates/compose.yml.j2

@@ -0,0 +1,60 @@
+services:
+
+  mailserver:
+    image: docker.io/mailserver/docker-mailserver:{{ mailserver_version }}
+    container_name: mailserver
+    hostname: {{ mail_hostname.split('.')[0] }}
+    domainname: {{ domain }}
+    env_file: env/mailserver.env
+    ports:
+      - "25:25"
+      - "587:587"
+      - "993:993"
+    healthcheck:
+      test: ["CMD-SHELL", "supervisorctl status | grep -E 'postfix|dovecot' | grep -q RUNNING"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 120s
+    volumes:
+      - /srv/mail/config:/tmp/docker-mailserver
+      - /srv/mail/maildata:/var/mail
+      - /srv/mail/mailstate:/var/mail-state
+      - /srv/mail/maillogs:/var/log/mail
+      - /etc/localtime:/etc/localtime:ro
+      - /etc/letsencrypt:/etc/letsencrypt:ro
+    restart: unless-stopped
+    cap_add:
+      - NET_ADMIN
+    networks:
+      - mail
+      - webmail
+
+  rainloop:
+    image: hardware/rainloop:{{ rainloop_version }}
+    container_name: rainloop
+    restart: unless-stopped
+    depends_on:
+      mailserver:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q --spider http://localhost:{{ rainloop_port }}/ || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    environment:
+      - RAINLOOP_ADMIN=admin
+      - RAINLOOP_ADMIN_PASSWORD={{ rainloop_admin_password }}
+    volumes:
+      - /srv/mail/rainloop/data:/rainloop/data
+    ports: []  # no host ports, only accessible via Docker network
+    networks:
+      - mail
+      - webmail
+
+networks:
+  mail:
+    external: true
+  webmail:
+    external: true

roles/mail/templates/mailserver.env.j2

@@ -0,0 +1,33 @@
+DMS_DEBUG=0
+# PERMIT_DOCKER=network
+
+# Hostname + primary domain (split from mail_hostname variable)
+HOSTNAME={{ mail_hostname.split('.')[0] }}
+DOMAINNAME={{ domain }}
+
+# Let's Encrypt
+ENABLE_TLS=1
+SSL_TYPE=letsencrypt
+LETSENCRYPT_DOMAIN={{ mail_hostname }}
+LETSENCRYPT_EMAIL={{ admin_user }}@{{ domain }}
+
+# Override hostname (FQDN) and domains handled by this server
+OVERRIDE_HOSTNAME={{ mail_hostname }}
+OVERRIDE_DOMAIN={{ mail_domains | join(',') }}
+
+# Other docker-mailserver options
+POSTMASTER_ADDRESS=postmaster@{{ domain }}
+ONE_DIR=1
+ENABLE_OPENDKIM=0
+# rspamd handles DMARC and SPF natively; enabling these would duplicate validation
+ENABLE_OPENDMARC=0
+ENABLE_POLICYD_SPF=0
+ENABLE_AMAVIS=0
+ENABLE_CLAMAV=0
+ENABLE_FAIL2BAN=0
+ENABLE_SPAMASSASSIN=0
+ENABLE_RSPAMD=1
+
+RSPAMD_LEARN=1
+
+POSTFIX_MESSAGE_SIZE_LIMIT=26214400

roles/mail/templates/worker-controller.inc.j2

@@ -0,0 +1,2 @@
+# Rspamd web UI password
+password = "{{ rspamd_web_password }}";

roles/monitoring/defaults/main.yml

@@ -0,0 +1,21 @@
+---
+# Data storage path on host
+monitoring_data_path: /srv/monitoring
+
+# Grafana
+grafana_port: 3000          # internal container port
+grafana_expose_port: []     # host port to expose (empty = not exposed outside docker)
+
+# Prometheus
+prometheus_port: 9090
+prometheus_expose_port: false
+prometheus_retention_days: 15
+
+# Loki (log aggregation)
+loki_port: 3100
+loki_expose_port: false
+loki_retention_days: 7
+
+# Alloy (metrics/logs collector)
+alloy_port: 12345
+alloy_expose_port: false

roles/monitoring/files/caddy-dashboard.json

@@ -0,0 +1,667 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (host) (rate(caddy_http_requests_total[5m]))",
+          "refId": "A",
+          "legendFormat": "{{host}}"
+        }
+      ],
+      "title": "Request Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum by (le, host) (rate(caddy_http_request_duration_seconds_bucket[5m])))",
+          "refId": "A",
+          "legendFormat": "{{host}} - p95"
+        },
+        {
+          "expr": "histogram_quantile(0.50, sum by (le, host) (rate(caddy_http_request_duration_seconds_bucket[5m])))",
+          "refId": "B",
+          "legendFormat": "{{host}} - p50"
+        }
+      ],
+      "title": "Request Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "2xx"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "green",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "4xx"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "yellow",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "5xx"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "red",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (host) (rate(caddy_http_response_duration_seconds_count{code=~\"2..\"}[5m]))",
+          "refId": "A",
+          "legendFormat": "{{host}} - 2xx"
+        },
+        {
+          "expr": "sum by (host) (rate(caddy_http_response_duration_seconds_count{code=~\"4..\"}[5m]))",
+          "refId": "B",
+          "legendFormat": "{{host}} - 4xx"
+        },
+        {
+          "expr": "sum by (host) (rate(caddy_http_response_duration_seconds_count{code=~\"5..\"}[5m]))",
+          "refId": "C",
+          "legendFormat": "{{host}} - 5xx"
+        }
+      ],
+      "title": "Response Status Codes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "binBps"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": ".*Sent.*"
+            },
+            "properties": [
+              {
+                "id": "custom.transform",
+                "value": "negative-Y"
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (host) (rate(caddy_http_response_size_bytes_sum[5m]))",
+          "refId": "A",
+          "legendFormat": "{{host}} - Sent"
+        },
+        {
+          "expr": "sum by (host) (rate(caddy_http_request_size_bytes_sum[5m]))",
+          "refId": "B",
+          "legendFormat": "{{host}} - Received"
+        }
+      ],
+      "title": "Bandwidth",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(caddy_http_requests_total[5m]))",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Requests/sec",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.5
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum(rate(caddy_http_response_duration_seconds_count{code=~\"5..\"}[5m])) / sum(rate(caddy_http_requests_total[5m])) OR on() vector(0)",
+          "refId": "A"
+        }
+      ],
+      "title": "Error Rate (5xx)",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 2,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 5
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(caddy_http_request_duration_seconds_bucket[5m])) by (le))",
+          "refId": "A"
+        }
+      ],
+      "title": "p95 Latency (Overall)",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum(caddy_http_requests_in_flight)",
+          "refId": "A"
+        }
+      ],
+      "title": "Requests In Flight",
+      "type": "stat"
+    },
+    {
+      "datasource": "Loki",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 9,
+      "options": {
+        "showTime": true,
+        "showLabels": false,
+        "showCommonLabels": false,
+        "wrapLogMessage": false,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "sortOrder": "Descending"
+      },
+      "targets": [
+        {
+          "expr": "{container=\"caddy\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "Caddy Logs",
+      "type": "logs"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["caddy", "web"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Caddy Web Server",
+  "uid": "caddy-metrics",
+  "version": 1
+}

roles/monitoring/files/forgejo-dashboard.json

@@ -0,0 +1,703 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "gitea_repositories",
+          "refId": "A",
+          "legendFormat": "Repositories"
+        },
+        {
+          "expr": "gitea_users",
+          "refId": "B",
+          "legendFormat": "Users"
+        }
+      ],
+      "title": "Repositories & Users Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "gitea_issues_open",
+          "refId": "A",
+          "legendFormat": "Open Issues"
+        },
+        {
+          "expr": "gitea_issues_closed",
+          "refId": "B",
+          "legendFormat": "Closed Issues"
+        }
+      ],
+      "title": "Issues Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "gitea_stars",
+          "refId": "A",
+          "legendFormat": "Stars"
+        },
+        {
+          "expr": "gitea_watches",
+          "refId": "B",
+          "legendFormat": "Watchers"
+        },
+        {
+          "expr": "gitea_follows",
+          "refId": "C",
+          "legendFormat": "Follows"
+        }
+      ],
+      "title": "Engagement Metrics",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "gitea_releases",
+          "refId": "A",
+          "legendFormat": "Releases"
+        },
+        {
+          "expr": "gitea_milestones",
+          "refId": "B",
+          "legendFormat": "Milestones"
+        },
+        {
+          "expr": "gitea_projects",
+          "refId": "C",
+          "legendFormat": "Projects"
+        }
+      ],
+      "title": "Project Metrics",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 0,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "gitea_repositories",
+          "refId": "A"
+        }
+      ],
+      "title": "Repositories",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 4,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "gitea_users",
+          "refId": "A"
+        }
+      ],
+      "title": "Users",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 8,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "gitea_issues",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Issues",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 5
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 12,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "gitea_issues_open",
+          "refId": "A"
+        }
+      ],
+      "title": "Open Issues",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 16,
+        "y": 16
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "gitea_organizations",
+          "refId": "A"
+        }
+      ],
+      "title": "Organizations",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 20,
+        "y": 16
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "gitea_publickeys",
+          "refId": "A"
+        }
+      ],
+      "title": "SSH Keys",
+      "type": "stat"
+    },
+    {
+      "datasource": "Loki",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 11,
+      "options": {
+        "showTime": true,
+        "showLabels": false,
+        "showCommonLabels": false,
+        "wrapLogMessage": false,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "sortOrder": "Descending"
+      },
+      "targets": [
+        {
+          "expr": "{container=\"forgejo\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "Forgejo Logs",
+      "type": "logs"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["forgejo", "git"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Forgejo",
+  "uid": "forgejo-metrics",
+  "version": 1
+}

roles/monitoring/files/mailserver-dashboard.json

@@ -0,0 +1,642 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"status=sent\" [5m])) or vector(0)",
+          "refId": "A",
+          "legendFormat": "Sent"
+        },
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"status=deferred\" [5m])) or vector(0)",
+          "refId": "B",
+          "legendFormat": "Deferred"
+        },
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"status=bounced\" [5m])) or vector(0)",
+          "refId": "C",
+          "legendFormat": "Bounced"
+        }
+      ],
+      "title": "Mail Delivery Status (5m)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"dovecot.*IMAP.*Login\" [5m])) or vector(0)",
+          "refId": "A",
+          "legendFormat": "IMAP Logins"
+        },
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"dovecot.*auth.*unknown user\" [5m])) or vector(0)",
+          "refId": "B",
+          "legendFormat": "Failed Auth"
+        },
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"postfix/submission.*auth=1\" [5m])) or vector(0)",
+          "refId": "C",
+          "legendFormat": "SMTP Auth Success"
+        }
+      ],
+      "title": "Authentication Activity (5m)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"postfix/smtpd.*connect from\" [5m])) or vector(0)",
+          "refId": "A",
+          "legendFormat": "Incoming Connections"
+        },
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"postscreen.*PREGREET\" [5m])) or vector(0)",
+          "refId": "B",
+          "legendFormat": "Blocked (PREGREET)"
+        },
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"postfix.*reject\" [5m])) or vector(0)",
+          "refId": "C",
+          "legendFormat": "Rejected"
+        }
+      ],
+      "title": "SMTP Connections & Filtering (5m)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"Connection timed out\" [5m])) or vector(0)",
+          "refId": "A",
+          "legendFormat": "Connection Timeouts"
+        },
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"Network is unreachable\" [5m])) or vector(0)",
+          "refId": "B",
+          "legendFormat": "Network Unreachable"
+        }
+      ],
+      "title": "Delivery Errors (5m)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["sum"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"status=sent\" [$__range])) or vector(0)",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "title": "Total Sent (Current Range)",
+      "type": "stat"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 10
+              },
+              {
+                "color": "red",
+                "value": 50
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["sum"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"status=deferred\" [$__range])) or vector(0)",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "title": "Total Deferred (Current Range)",
+      "type": "stat"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 5
+              },
+              {
+                "color": "red",
+                "value": 20
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["sum"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"unknown user\" [$__range])) or vector(0)",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "title": "Failed Auth Attempts (Current Range)",
+      "type": "stat"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 10
+              },
+              {
+                "color": "red",
+                "value": 50
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["sum"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({container=\"mailserver\"} |~ \"PREGREET\" [$__range])) or vector(0)",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "title": "Spam Blocked (Current Range)",
+      "type": "stat"
+    },
+    {
+      "datasource": "Loki",
+      "gridPos": {
+        "h": 12,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 9,
+      "options": {
+        "showTime": true,
+        "showLabels": false,
+        "showCommonLabels": false,
+        "wrapLogMessage": false,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "sortOrder": "Descending"
+      },
+      "targets": [
+        {
+          "expr": "{container=\"mailserver\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "Mailserver Logs",
+      "type": "logs"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["mail", "postfix", "dovecot"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Mail Server",
+  "uid": "mailserver-logs",
+  "version": 1
+}

roles/monitoring/files/node-exporter-dashboard.json

@@ -0,0 +1,364 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "refId": "A",
+          "legendFormat": "CPU Usage"
+        }
+      ],
+      "title": "CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 * (1 - ((node_memory_MemAvailable_bytes or node_memory_Buffers_bytes + node_memory_Cached_bytes + node_memory_MemFree_bytes) / node_memory_MemTotal_bytes))",
+          "refId": "A",
+          "legendFormat": "Memory Usage"
+        }
+      ],
+      "title": "Memory Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 - ((node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"rootfs\"})",
+          "refId": "A",
+          "legendFormat": "Disk Usage /"
+        }
+      ],
+      "title": "Disk Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "Bps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "irate(node_network_receive_bytes_total{device!=\"lo\"}[5m])",
+          "refId": "A",
+          "legendFormat": "{{device}} - Receive"
+        },
+        {
+          "expr": "irate(node_network_transmit_bytes_total{device!=\"lo\"}[5m])",
+          "refId": "B",
+          "legendFormat": "{{device}} - Transmit"
+        }
+      ],
+      "title": "Network Traffic",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["node-exporter"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Node Exporter System Metrics",
+  "uid": "node-exporter",
+  "version": 1
+}

roles/monitoring/files/service-overview-dashboard.json

@@ -0,0 +1,922 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 17,
+      "panels": [],
+      "title": "Logs (use Level and Search filters above)",
+      "type": "row"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineWidth": 1,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            }
+          },
+          "mappings": [],
+          "noValue": "0",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null}
+            ]
+          }
+        },
+        "overrides": [
+          {"matcher": {"id": "byName", "options": "fatal"}, "properties": [{"id": "color", "value": {"fixedColor": "dark-red", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "panic"}, "properties": [{"id": "color", "value": {"fixedColor": "dark-red", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "critical"}, "properties": [{"id": "color", "value": {"fixedColor": "red", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "crit"}, "properties": [{"id": "color", "value": {"fixedColor": "red", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "error"}, "properties": [{"id": "color", "value": {"fixedColor": "orange", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "err"}, "properties": [{"id": "color", "value": {"fixedColor": "orange", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "warn"}, "properties": [{"id": "color", "value": {"fixedColor": "yellow", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "warning"}, "properties": [{"id": "color", "value": {"fixedColor": "yellow", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "notice"}, "properties": [{"id": "color", "value": {"fixedColor": "blue", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "info"}, "properties": [{"id": "color", "value": {"fixedColor": "green", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "debug"}, "properties": [{"id": "color", "value": {"fixedColor": "purple", "mode": "fixed"}}]}
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 1
+      },
+      "id": 16,
+      "options": {
+        "legend": {
+          "calcs": ["sum"],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (level) (count_over_time({level=~\"${level:regex}\"} |~ \"(?i)$search\" [$__interval]))",
+          "legendFormat": "{{level}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Log Volume by Level",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Loki",
+      "gridPos": {
+        "h": 8,
+        "w": 16,
+        "x": 8,
+        "y": 1
+      },
+      "id": 10,
+      "options": {
+        "showTime": true,
+        "showLabels": true,
+        "showCommonLabels": false,
+        "wrapLogMessage": false,
+        "prettifyLogMessage": false,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "sortOrder": "Descending",
+        "noDataMessage": " "
+      },
+      "targets": [
+        {
+          "expr": "{level=~\"${level:regex}\"} |~ \"(?i)$search\"",
+          "refId": "A"
+        }
+      ],
+      "title": "Logs",
+      "type": "logs"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {"color": "red", "text": "DOWN"},
+                "1": {"color": "green", "text": "UP"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red", "value": null},
+              {"color": "green", "value": 1}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 0, "y": 9},
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "up{job=\"caddy\"}", "refId": "A"}],
+      "title": "Caddy",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {"color": "red", "text": "DOWN"},
+                "1": {"color": "green", "text": "UP"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red", "value": null},
+              {"color": "green", "value": 1}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 4, "y": 9},
+      "id": 2,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "up{job=\"forgejo\"}", "refId": "A"}],
+      "title": "Forgejo",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "1": {"color": "green", "text": "UP"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null}
+            ]
+          }
+        }
+      },
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {"color": "red", "text": "DOWN"},
+                "1": {"color": "green", "text": "UP"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red", "value": null},
+              {"color": "green", "value": 1}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 8, "y": 9},
+      "id": 20,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "count(container_last_seen{name=\"tuwunel\"})", "refId": "A"}],
+      "title": "Tuwunel",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "1": {"color": "green", "text": "UP"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 3, "x": 12, "y": 9},
+      "id": 3,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "1", "refId": "A"}],
+      "title": "Mailserver",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {"color": "red", "text": "DOWN"},
+                "1": {"color": "green", "text": "UP"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red", "value": null},
+              {"color": "green", "value": 1}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 3, "x": 15, "y": 9},
+      "id": 4,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "up{job=\"prometheus\"}", "refId": "A"}],
+      "title": "Prometheus",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {"color": "red", "text": "DOWN"},
+                "1": {"color": "green", "text": "UP"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red", "value": null},
+              {"color": "green", "value": 1}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 3, "x": 18, "y": 9},
+      "id": 5,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "up{job=\"alloy\"}", "refId": "A"}],
+      "title": "Alloy",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "1": {"color": "green", "text": "UP"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 3, "x": 21, "y": 9},
+      "id": 6,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "1", "refId": "A"}],
+      "title": "Loki",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {"color": "red", "text": "FAILED"},
+                "1": {"color": "green", "text": "OK"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red", "value": null},
+              {"color": "yellow", "value": 0.5},
+              {"color": "green", "value": 1}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 0, "y": 13},
+      "id": 11,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "text": {"valueSize": 32},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "restic_backup_success", "refId": "A"}],
+      "title": "Last Backup",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 86400},
+              {"color": "red", "value": 172800}
+            ]
+          },
+          "unit": "dtdurations"
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 4, "y": 13},
+      "id": 12,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "text": {"valueSize": 24},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "time() - restic_backup_timestamp_seconds", "refId": "A"}],
+      "title": "Time Since Last Backup",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {"color": "red", "text": "FAILED"},
+                "1": {"color": "green", "text": "OK"}
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red", "value": null},
+              {"color": "yellow", "value": 0.5},
+              {"color": "green", "value": 1}
+            ]
+          }
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 8, "y": 13},
+      "id": 13,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "text": {"valueSize": 32},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "restic_prune_success", "refId": "A"}],
+      "title": "Last Prune",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 604800},
+              {"color": "red", "value": 1209600}
+            ]
+          },
+          "unit": "dtdurations"
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 12, "y": 13},
+      "id": 14,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "text": {"valueSize": 24},
+        "textMode": "value"
+      },
+      "targets": [{"expr": "time() - restic_prune_timestamp_seconds", "refId": "A"}],
+      "title": "Time Since Last Prune",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null}
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 16, "y": 13},
+      "id": 15,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["lastNotNull"], "fields": ""},
+        "text": {"valueSize": 32},
+        "textMode": "value"
+      },
+      "targets": [
+        {
+          "expr": "count(container_last_seen{name=~\".+\"})",
+          "refId": "A"
+        }
+      ],
+      "title": "Active Containers",
+      "type": "stat"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 50},
+              {"color": "red", "value": 200}
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": {"h": 4, "w": 4, "x": 20, "y": 13},
+      "id": 18,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {"values": false, "calcs": ["sum"], "fields": ""},
+        "text": {"valueSize": 32},
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({job=\"rspamd\"} |~ \"\\\\(reject\\\\)|\\\\(greylist\\\\)\" [$__range]))",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "title": "Spam Blocked",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {"tooltip": false, "viz": false, "legend": false},
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 60},
+              {"color": "red", "value": 80}
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 8, "x": 0, "y": 17},
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {"mode": "multi", "sort": "none"}
+      },
+      "targets": [
+        {
+          "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "refId": "A",
+          "legendFormat": "CPU Usage"
+        }
+      ],
+      "title": "System CPU",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {"tooltip": false, "viz": false, "legend": false},
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 70},
+              {"color": "red", "value": 85}
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 8, "x": 8, "y": 17},
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {"mode": "multi", "sort": "none"}
+      },
+      "targets": [
+        {
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
+          "refId": "A",
+          "legendFormat": "Memory Usage"
+        }
+      ],
+      "title": "System Memory",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {"tooltip": false, "viz": false, "legend": false},
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {"type": "linear"},
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {"group": "A", "mode": "none"},
+            "thresholdsStyle": {"mode": "off"}
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "yellow", "value": 70},
+              {"color": "red", "value": 85}
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {"h": 8, "w": 8, "x": 16, "y": 17},
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {"mode": "multi", "sort": "none"}
+      },
+      "targets": [
+        {
+          "expr": "100 - ((node_filesystem_avail_bytes{mountpoint=\"/\"} * 100) / node_filesystem_size_bytes{mountpoint=\"/\"})",
+          "refId": "A",
+          "legendFormat": "Disk Usage"
+        }
+      ],
+      "title": "System Disk",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Loki",
+      "fieldConfig": {
+        "defaults": {
+          "color": {"mode": "palette-classic"},
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {"legend": false, "tooltip": false, "viz": false},
+            "lineWidth": 1,
+            "scaleDistribution": {"type": "linear"},
+            "stacking": {"group": "A", "mode": "normal"}
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{"color": "green", "value": null}]
+          }
+        },
+        "overrides": [
+          {"matcher": {"id": "byName", "options": "Rejected"}, "properties": [{"id": "color", "value": {"fixedColor": "red", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "Greylisted"}, "properties": [{"id": "color", "value": {"fixedColor": "orange", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "Spam Header"}, "properties": [{"id": "color", "value": {"fixedColor": "yellow", "mode": "fixed"}}]},
+          {"matcher": {"id": "byName", "options": "Clean"}, "properties": [{"id": "color", "value": {"fixedColor": "green", "mode": "fixed"}}]}
+        ]
+      },
+      "gridPos": {"h": 6, "w": 24, "x": 0, "y": 25},
+      "id": 19,
+      "options": {
+        "legend": {
+          "calcs": ["sum"],
+          "displayMode": "list",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {"mode": "multi", "sort": "none"}
+      },
+      "targets": [
+        {
+          "expr": "sum(count_over_time({job=\"rspamd\"} |~ \"\\\\(reject\\\\)\" [$__interval]))",
+          "legendFormat": "Rejected",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(count_over_time({job=\"rspamd\"} |~ \"\\\\(greylist\\\\)\" [$__interval]))",
+          "legendFormat": "Greylisted",
+          "refId": "B"
+        },
+        {
+          "expr": "sum(count_over_time({job=\"rspamd\"} |~ \"\\\\(add header\\\\)\" [$__interval]))",
+          "legendFormat": "Spam Header",
+          "refId": "C"
+        },
+        {
+          "expr": "sum(count_over_time({job=\"rspamd\"} |~ \"\\\\(no action\\\\)\" [$__interval]))",
+          "legendFormat": "Clean",
+          "refId": "D"
+        }
+      ],
+      "title": "Rspamd Actions",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["overview"],
+  "templating": {
+    "list": [
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": ["error", "err", "fatal", "panic", "critical", "crit"],
+          "value": ["error", "err", "fatal", "panic", "critical", "crit"]
+        },
+        "description": "Log levels to display (extracted from log content)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Level",
+        "multi": true,
+        "name": "level",
+        "options": [
+          {"selected": true, "text": "fatal", "value": "fatal"},
+          {"selected": true, "text": "panic", "value": "panic"},
+          {"selected": true, "text": "critical", "value": "critical"},
+          {"selected": true, "text": "crit", "value": "crit"},
+          {"selected": true, "text": "error", "value": "error"},
+          {"selected": true, "text": "err", "value": "err"},
+          {"selected": false, "text": "warn", "value": "warn"},
+          {"selected": false, "text": "warning", "value": "warning"},
+          {"selected": false, "text": "notice", "value": "notice"},
+          {"selected": false, "text": "info", "value": "info"},
+          {"selected": false, "text": "debug", "value": "debug"}
+        ],
+        "query": "fatal, panic, critical, crit, error, err, warn, warning, notice, info, debug",
+        "type": "custom"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "",
+          "value": ""
+        },
+        "description": "Filter logs by text (regex supported)",
+        "hide": 0,
+        "label": "Search",
+        "name": "search",
+        "options": [],
+        "query": "",
+        "type": "textbox"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Service Overview",
+  "uid": "service-overview",
+  "version": 1
+}

roles/monitoring/files/system-metrics-dashboard.json

@@ -0,0 +1,553 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 60
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
+          "refId": "A",
+          "legendFormat": "CPU Usage"
+        }
+      ],
+      "title": "CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 60
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
+          "refId": "A",
+          "legendFormat": "Memory Usage"
+        }
+      ],
+      "title": "Memory Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 100,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 70
+              },
+              {
+                "color": "red",
+                "value": 85
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 - ((node_filesystem_avail_bytes{mountpoint=\"/\"} * 100) / node_filesystem_size_bytes{mountpoint=\"/\"})",
+          "refId": "A",
+          "legendFormat": "Root Disk Usage"
+        }
+      ],
+      "title": "Disk Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "tooltip": false,
+              "viz": false,
+              "legend": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "Bps"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": ".*Receive.*"
+            },
+            "properties": [
+              {
+                "id": "custom.transform",
+                "value": "negative-Y"
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "irate(node_network_receive_bytes_total{device!=\"lo\"}[5m])",
+          "refId": "A",
+          "legendFormat": "{{device}} Receive"
+        },
+        {
+          "expr": "irate(node_network_transmit_bytes_total{device!=\"lo\"}[5m])",
+          "refId": "B",
+          "legendFormat": "{{device}} Transmit"
+        }
+      ],
+      "title": "Network Traffic",
+      "type": "timeseries"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 2
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 0,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "node_load1",
+          "refId": "A"
+        }
+      ],
+      "title": "Load Average (1m)",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 2,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 8,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
+          "refId": "A"
+        }
+      ],
+      "title": "Disk Space Available",
+      "type": "stat"
+    },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 2,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 8,
+        "x": 16,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "node_memory_MemAvailable_bytes",
+          "refId": "A"
+        }
+      ],
+      "title": "Memory Available",
+      "type": "stat"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["system"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "System Metrics",
+  "uid": "system-metrics",
+  "version": 1
+}

roles/monitoring/handlers/main.yml

@@ -0,0 +1,30 @@
+---
+- name: restart prometheus
+  community.docker.docker_container:
+    name: prometheus
+    state: started
+    restart: true
+
+- name: restart alloy
+  community.docker.docker_container:
+    name: alloy
+    state: started
+    restart: true
+
+- name: restart loki
+  community.docker.docker_container:
+    name: loki
+    state: started
+    restart: true
+
+- name: restart grafana
+  community.docker.docker_container:
+    name: grafana
+    state: started
+    restart: true
+
+- name: reload monitoring stack
+  community.docker.docker_compose_v2:
+    project_src: "{{ monitoring_data_path }}"
+    state: present
+    build: never

roles/monitoring/tasks/main.yml

@@ -0,0 +1,127 @@
+---
+- name: Create monitoring directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - "{{ monitoring_data_path }}"
+    - "{{ monitoring_data_path }}/prometheus"
+    - "{{ monitoring_data_path }}/alloy"
+    - "{{ monitoring_data_path }}/grafana"
+    - "{{ monitoring_data_path }}/grafana/provisioning"
+    - "{{ monitoring_data_path }}/grafana/provisioning/datasources"
+    - "{{ monitoring_data_path }}/grafana/provisioning/dashboards"
+    - "{{ monitoring_data_path }}/loki"
+
+- name: Create textfile collector directory for node_exporter metrics
+  file:
+    path: /var/lib/node_exporter/textfile_collector
+    state: directory
+    mode: '0755'
+    owner: root
+    group: root
+
+- name: Create container data directories with proper ownership
+  ansible.builtin.shell: |
+    mkdir -p "{{ item.path }}"
+    chmod 755 "{{ item.path }}"
+    chown {{ item.uid }}:{{ item.gid }} "{{ item.path }}"
+  args:
+    creates: "{{ item.path }}"
+  loop:
+    - { path: "{{ monitoring_data_path }}/prometheus/data", uid: 65534, gid: 65534 }
+    - { path: "{{ monitoring_data_path }}/grafana/data", uid: 472, gid: 472 }
+    - { path: "{{ monitoring_data_path }}/loki/data", uid: 10001, gid: 10001 }
+  loop_control:
+    label: "{{ item.path }}"
+
+- name: Create Prometheus configuration
+  template:
+    src: prometheus.yml.j2
+    dest: "{{ monitoring_data_path }}/prometheus/prometheus.yml"
+    mode: '0644'
+  notify: restart prometheus
+
+- name: Create Alloy configuration
+  template:
+    src: config.alloy.j2
+    dest: "{{ monitoring_data_path }}/alloy/config.alloy"
+    mode: '0644'
+    force: true
+  notify: restart alloy
+
+- name: Create Loki configuration
+  template:
+    src: loki-config.yaml.j2
+    dest: "{{ monitoring_data_path }}/loki/loki-config.yaml"
+    mode: '0644'
+    force: true
+  notify: restart loki
+
+- name: Create Grafana datasource configuration
+  template:
+    src: datasources.yml.j2
+    dest: "{{ monitoring_data_path }}/grafana/provisioning/datasources/datasources.yml"
+    mode: '0644'
+
+- name: Create Grafana dashboard provisioning config
+  template:
+    src: dashboards.yml.j2
+    dest: "{{ monitoring_data_path }}/grafana/provisioning/dashboards/dashboards.yml"
+    mode: '0644'
+
+- name: Copy Node Exporter dashboard
+  copy:
+    src: node-exporter-dashboard.json
+    dest: "{{ monitoring_data_path }}/grafana/provisioning/dashboards/node-exporter.json"
+    mode: '0644'
+
+- name: Copy System Metrics dashboard
+  copy:
+    src: system-metrics-dashboard.json
+    dest: "{{ monitoring_data_path }}/grafana/provisioning/dashboards/system-metrics.json"
+    mode: '0644'
+
+- name: Copy Caddy dashboard
+  copy:
+    src: caddy-dashboard.json
+    dest: "{{ monitoring_data_path }}/grafana/provisioning/dashboards/caddy.json"
+    mode: '0644'
+
+- name: Copy Mailserver dashboard
+  copy:
+    src: mailserver-dashboard.json
+    dest: "{{ monitoring_data_path }}/grafana/provisioning/dashboards/mailserver.json"
+    mode: '0644'
+
+- name: Copy Forgejo dashboard
+  copy:
+    src: forgejo-dashboard.json
+    dest: "{{ monitoring_data_path }}/grafana/provisioning/dashboards/forgejo.json"
+    mode: '0644'
+
+- name: Copy Service Overview dashboard
+  copy:
+    src: service-overview-dashboard.json
+    dest: "{{ monitoring_data_path }}/grafana/provisioning/dashboards/service-overview.json"
+    mode: '0644'
+
+- name: Create compose file
+  template:
+    src: compose.yml.j2
+    dest: "{{ monitoring_data_path }}/compose.yml"
+    mode: '0644'
+  register: compose_file
+  notify: reload monitoring stack
+
+- name: Deploy monitoring stack
+  community.docker.docker_compose_v2:
+    project_src: "{{ monitoring_data_path }}"
+    state: present
+    build: never
+  register: compose_output
+
+- name: Show deployment status
+  debug:
+    msg: "Monitoring stack deployed. Grafana available at {{ grafana_root_url }}"

roles/monitoring/templates/compose.yml.j2

@@ -0,0 +1,93 @@
+services:
+  prometheus:
+    image: prom/prometheus:{{ prometheus_version }}
+    container_name: prometheus
+    restart: unless-stopped
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time={{ prometheus_retention_days }}d'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+      - '--web.enable-remote-write-receiver'
+    volumes:
+      - {{ monitoring_data_path }}/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - {{ monitoring_data_path }}/prometheus/data:/prometheus
+    networks:
+      - monitoring
+      - caddy
+{% if prometheus_expose_port | default(true) %}
+    ports:
+      - "{{ prometheus_port }}:9090"
+{% endif %}
+
+  alloy:
+    image: grafana/alloy:{{ alloy_version }}
+    container_name: alloy
+    restart: unless-stopped
+    privileged: true
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:{{ alloy_port }}
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
+    volumes:
+      - {{ monitoring_data_path }}/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - /:/host/root:ro
+      - /sys:/host/sys:ro
+      - /proc:/host/proc:ro
+      - /var/log:/var/log:ro
+      - /run/log/journal:/run/log/journal:ro
+      - /etc/machine-id:/etc/machine-id:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+      - /var/lib/docker:/var/lib/docker:ro
+    environment:
+      HOSTNAME: {{ ansible_facts["hostname"] }}
+    networks:
+      - monitoring
+{% if alloy_expose_port | default(true) %}
+    ports:
+      - "{{ alloy_port }}:{{ alloy_port }}"
+{% endif %}
+
+  grafana:
+    image: grafana/grafana:{{ grafana_version }}
+    container_name: grafana
+    restart: unless-stopped
+    volumes:
+      - {{ monitoring_data_path }}/grafana/data:/var/lib/grafana
+      - {{ monitoring_data_path }}/grafana/provisioning:/etc/grafana/provisioning
+    environment:
+      GF_SECURITY_ADMIN_PASSWORD: {{ grafana_admin_password }}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_SERVER_ROOT_URL: {{ grafana_root_url | default('http://localhost:3000') }}
+      GF_SERVER_SERVE_FROM_SUB_PATH: "false"
+    networks:
+      - monitoring
+      - caddy
+{% if grafana_expose_port | default([]) %}
+    ports:
+      - "{{ grafana_expose_port }}:3000"
+{% endif %}
+
+  loki:
+    image: grafana/loki:{{ loki_version }}
+    container_name: loki
+    restart: unless-stopped
+    command: -config.file=/etc/loki/local-config.yaml
+    volumes:
+      - {{ monitoring_data_path }}/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
+      - {{ monitoring_data_path }}/loki/data:/loki
+    networks:
+      - monitoring
+{% if loki_expose_port | default(true) %}
+    ports:
+      - "{{ loki_port | default(3100) }}:3100"
+{% endif %}
+
+networks:
+  monitoring:
+    external: true
+  caddy:
+    external: true

roles/monitoring/templates/config.alloy.j2

@@ -0,0 +1,163 @@
+// Prometheus metrics collection
+prometheus.exporter.unix "node" {
+  rootfs_path    = "/host/root"
+  sysfs_path     = "/host/sys"
+  procfs_path    = "/host/proc"
+  textfile {
+    directory = "/host/root/var/lib/node_exporter/textfile_collector"
+  }
+  
+  set_collectors = ["cpu", "loadavg", "meminfo", "diskstats", "filesystem", "netdev", "textfile"]
+}
+
+prometheus.scrape "node_exporter" {
+  targets    = prometheus.exporter.unix.node.targets
+  forward_to = [prometheus.remote_write.metrics.receiver]
+}
+
+prometheus.scrape "alloy" {
+  targets = [{
+    __address__ = "localhost:{{ alloy_port }}",
+  }]
+  forward_to = [prometheus.remote_write.metrics.receiver]
+}
+
+prometheus.exporter.cadvisor "docker" {
+  docker_host = "unix:///var/run/docker.sock"
+  docker_only = true
+}
+
+prometheus.scrape "cadvisor" {
+  targets    = prometheus.exporter.cadvisor.docker.targets
+  forward_to = [prometheus.remote_write.metrics.receiver]
+}
+
+prometheus.remote_write "metrics" {
+  endpoint {
+    url = "http://prometheus:{{ prometheus_port }}/api/v1/write"
+  }
+}
+
+// Journal log collection (includes both system logs and Docker containers)
+loki.source.journal "journal" {
+  forward_to = [loki.process.journal.receiver]
+  relabel_rules = loki.relabel.journal.rules
+  labels = {
+    job = "journal",
+  }
+}
+
+loki.relabel "journal" {
+  forward_to = []
+
+  // Systemd unit (e.g., ssh.service, docker.service)
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+
+  // Container name for Docker containers
+  rule {
+    source_labels = ["__journal_container_name"]
+    target_label  = "container"
+  }
+
+  // Syslog priority (0=emerg, 1=alert, 2=crit, 3=err, 4=warn, 5=notice, 6=info, 7=debug)
+  rule {
+    source_labels = ["__journal_priority"]
+    target_label  = "priority"
+  }
+
+  // Syslog identifier (program name)
+  rule {
+    source_labels = ["__journal_syslog_identifier"]
+    target_label  = "syslog_identifier"
+  }
+
+  // Tag tuwunel container with its own job label
+  rule {
+    source_labels = ["__journal_container_name"]
+    regex         = "tuwunel"
+    target_label  = "job"
+    replacement   = "tuwunel"
+  }
+}
+
+loki.process "journal" {
+  forward_to = [loki.write.logs.receiver]
+
+  // Extract log level from common formats: level=info, "level":"info", [INFO], etc.
+  stage.regex {
+    expression = "(?i)(level=|\"level\":\\s*\"|\\[)(?P<extracted_level>debug|info|warn|warning|error|err|fatal|panic|critical|crit|notice)(\\]|\"|\\s|$)"
+  }
+
+  // Map extracted level to numeric priority for consistent filtering
+  stage.template {
+    source   = "level"
+    template = "{% raw %}{{ if .extracted_level }}{{ .extracted_level }}{{ else }}{{ .priority }}{{ end }}{% endraw %}"
+  }
+
+  stage.labels {
+    values = {
+      level = "",
+    }
+  }
+}
+
+loki.write "logs" {
+  endpoint {
+    url = "http://loki:{{ loki_port }}/loki/api/v1/push"
+  }
+}
+
+// Fail2ban log file collection (ban/unban details go to file, not journald)
+local.file_match "fail2ban" {
+  path_targets = [{"__path__" = "/host/root/var/log/fail2ban.log"}]
+}
+
+loki.source.file "fail2ban" {
+  targets    = local.file_match.fail2ban.targets
+  forward_to = [loki.process.fail2ban.receiver]
+}
+
+loki.process "fail2ban" {
+  forward_to = [loki.write.logs.receiver]
+
+  stage.static_labels {
+    values = {
+      job  = "fail2ban",
+      unit = "fail2ban.service",
+    }
+  }
+
+  stage.regex {
+    expression = "(?i)\\s(?P<extracted_level>notice|warning|error|info)\\s"
+  }
+
+  stage.labels {
+    values = {
+      level = "extracted_level",
+    }
+  }
+}
+
+// Rspamd log file collection (logs to file inside mailserver, not stdout)
+local.file_match "rspamd" {
+  path_targets = [{"__path__" = "/host/root/srv/mail/maillogs/rspamd.log"}]
+}
+
+loki.source.file "rspamd" {
+  targets    = local.file_match.rspamd.targets
+  forward_to = [loki.process.rspamd.receiver]
+}
+
+loki.process "rspamd" {
+  forward_to = [loki.write.logs.receiver]
+
+  stage.static_labels {
+    values = {
+      container = "mailserver",
+      job       = "rspamd",
+    }
+  }
+}

roles/monitoring/templates/dashboards.yml.j2

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+  - name: 'Default'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/provisioning/dashboards
+      foldersFromFilesStructure: true

roles/monitoring/templates/datasources.yml.j2

@@ -0,0 +1,17 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:{{ prometheus_port }}
+    isDefault: true
+    editable: false
+    jsonData:
+      timeInterval: 15s
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:{{ loki_port }}
+    editable: false

roles/monitoring/templates/loki-config.yaml.j2

@@ -0,0 +1,43 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: {{ loki_retention_days | default(prometheus_retention_days) }}d
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+compactor:
+  working_directory: /loki/compactor
+  compaction_interval: 10m

roles/monitoring/templates/prometheus.yml.j2

@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:{{ prometheus_port }}']
+
+  - job_name: 'alloy'
+    static_configs:
+      - targets: ['alloy:{{ alloy_port }}']
+
+  - job_name: 'caddy'
+    static_configs:
+      - targets: ['caddy:{{ caddy_metrics_port }}']
+
+  - job_name: 'forgejo'
+    static_configs:
+      - targets: ['forgejo:{{ forgejo_port }}']

roles/nebula/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# UDP port for nebula tunnel traffic
+nebula_port: 4242

roles/nebula/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: Restart nebula
+  service:
+    name: nebula
+    state: restarted

roles/nebula/tasks/main.yml

@@ -0,0 +1,69 @@
+- name: Allow Nebula UDP traffic
+  ufw:
+    rule: allow
+    port: "{{ nebula_port }}"
+    proto: udp
+
+- name: Download Nebula release
+  unarchive:
+    src: "https://github.com/slackhq/nebula/releases/download/v{{ nebula_version }}/nebula-linux-amd64.tar.gz"
+    dest: /usr/local/bin/
+    remote_src: true
+    creates: /usr/local/bin/nebula
+    include:
+      - nebula
+      - nebula-cert
+
+- name: Create Nebula config directory
+  file:
+    path: /etc/nebula
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+
+- name: Generate Nebula CA
+  command: >
+    nebula-cert ca
+    -name "linderhof"
+    -out-crt /etc/nebula/ca.crt
+    -out-key /etc/nebula/ca.key
+  args:
+    creates: /etc/nebula/ca.key
+
+- name: Generate host certificate
+  command: >
+    nebula-cert sign
+    -ca-crt /etc/nebula/ca.crt
+    -ca-key /etc/nebula/ca.key
+    -name "lighthouse"
+    -ip "{{ nebula_lighthouse_ip }}/{{ nebula_subnet.split('/')[1] }}"
+    -out-crt /etc/nebula/host.crt
+    -out-key /etc/nebula/host.key
+  args:
+    creates: /etc/nebula/host.key
+
+- name: Deploy Nebula config
+  template:
+    src: config.yml.j2
+    dest: /etc/nebula/config.yml
+    owner: root
+    group: root
+    mode: "0600"
+  notify: Restart nebula
+
+- name: Deploy Nebula systemd unit
+  template:
+    src: nebula.service.j2
+    dest: /etc/systemd/system/nebula.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart nebula
+
+- name: Enable and start Nebula
+  systemd:
+    name: nebula
+    enabled: true
+    state: started
+    daemon_reload: true

roles/nebula/templates/config.yml.j2

@@ -0,0 +1,40 @@
+pki:
+  ca: /etc/nebula/ca.crt
+  cert: /etc/nebula/host.crt
+  key: /etc/nebula/host.key
+
+static_host_map: {}
+
+lighthouse:
+  am_lighthouse: true
+  interval: 60
+
+listen:
+  host: 0.0.0.0
+  port: {{ nebula_port }}
+
+punchy:
+  punch: true
+
+tun:
+  dev: nebula1
+  drop_local_broadcast: false
+  drop_multicast: false
+
+logging:
+  level: info
+  format: text
+
+firewall:
+  conntrack:
+    tcp_timeout: 12m
+    udp_timeout: 3m
+    default_timeout: 10m
+  outbound:
+    - port: any
+      proto: any
+      host: any
+  inbound:
+    - port: any
+      proto: any
+      host: any

roles/nebula/templates/nebula.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=Nebula Overlay Network
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/provision/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+hcloud_server_type: cx22
+hcloud_image: ubuntu-24.04
+hcloud_location: fsn1

roles/provision/tasks/hetzner.yml

@@ -0,0 +1,40 @@
+---
+- name: Register SSH key with Hetzner
+  hetzner.hcloud.ssh_key:
+    name: "{{ admin_user }}"
+    public_key: "{{ admin_ssh_key }}"
+    api_token: "{{ hcloud_token }}"
+    state: present
+
+- name: Create server
+  hetzner.hcloud.server:
+    name: "{{ server_name }}"
+    server_type: "{{ hcloud_server_type }}"
+    image: "{{ hcloud_image }}"
+    location: "{{ hcloud_location }}"
+    ssh_keys:
+      - "{{ admin_user }}"
+    api_token: "{{ hcloud_token }}"
+    state: present
+  register: server_result
+
+- name: Set server IP fact
+  ansible.builtin.set_fact:
+    server_ip: "{{ server_result.hcloud_server.ipv4_address }}"
+
+- name: Wait for SSH to become available
+  ansible.builtin.wait_for:
+    host: "{{ server_ip }}"
+    port: 22
+    timeout: 300
+
+- name: Update inventory with new IP
+  ansible.builtin.lineinfile:
+    path: "{{ inventory_dir }}/hosts.yml"
+    regexp: '^\s+ansible_host:'
+    line: "      ansible_host: {{ server_ip }}"
+  delegate_to: localhost
+
+- name: Print server IP
+  ansible.builtin.debug:
+    msg: "Server '{{ server_name }}' provisioned at {{ server_ip }}"

roles/provision/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Include provider tasks
+  ansible.builtin.include_tasks: "{{ cloud_provider }}.yml"

roles/radicale/defaults/main.yml

@@ -0,0 +1,2 @@
+radicale_version: "latest"
+radicale_port: 5232

roles/radicale/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+- name: restart radicale
+  community.docker.docker_compose_v2:
+    project_src: /srv/radicale
+    state: present
+    recreate: always
+    build: never

roles/radicale/tasks/main.yml

@@ -0,0 +1,81 @@
+---
+- name: Create radicale directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - /srv/radicale
+    - /srv/radicale/data
+    - /srv/radicale/config
+
+- name: Create radicale configuration
+  template:
+    src: config.j2
+    dest: /srv/radicale/config/config
+    mode: '0644'
+  notify: restart radicale
+
+- name: Check for cached radicale hash
+  ansible.builtin.stat:
+    path: /srv/radicale/config/.radicale_hash
+  register: _radicale_hash_stat
+
+- name: Read radicale hash from cache
+  ansible.builtin.slurp:
+    src: /srv/radicale/config/.radicale_hash
+  register: _radicale_hash_file
+  when: _radicale_hash_stat.stat.exists
+
+- name: Set radicale hash fact from cache
+  ansible.builtin.set_fact:
+    _radicale_hash: "{{ _radicale_hash_file.content | b64decode | trim }}"
+  when: _radicale_hash_stat.stat.exists
+
+- name: Generate radicale password hash
+  command:
+    argv:
+      - docker
+      - run
+      - --rm
+      - caddy:2
+      - caddy
+      - hash-password
+      - --plaintext
+      - "{{ radicale_password }}"
+  register: _radicale_hash_result
+  changed_when: false
+  no_log: true
+  when: not _radicale_hash_stat.stat.exists
+
+- name: Cache radicale hash
+  ansible.builtin.copy:
+    content: "{{ _radicale_hash_result.stdout }}"
+    dest: /srv/radicale/config/.radicale_hash
+    mode: "0600"
+  when: not _radicale_hash_stat.stat.exists
+
+- name: Set radicale hash fact from generation
+  ansible.builtin.set_fact:
+    _radicale_hash: "{{ _radicale_hash_result.stdout }}"
+  when: not _radicale_hash_stat.stat.exists
+
+- name: Create radicale users file
+  copy:
+    content: "{{ admin_user }}:{{ _radicale_hash }}"
+    dest: /srv/radicale/config/users
+    mode: '0644'
+  notify: restart radicale
+
+- name: Create compose file
+  template:
+    src: compose.yml.j2
+    dest: /srv/radicale/compose.yml
+    mode: '0644'
+  notify: restart radicale
+
+- name: Deploy radicale
+  community.docker.docker_compose_v2:
+    project_src: /srv/radicale
+    state: present
+    build: never

roles/radicale/templates/compose.yml.j2

@@ -0,0 +1,17 @@
+services:
+  radicale:
+    image: tomsquest/docker-radicale:{{ radicale_version }}
+    container_name: radicale
+    restart: unless-stopped
+    volumes:
+      - /srv/radicale/data:/data
+      - /srv/radicale/config:/config:ro
+    networks:
+      - radicale
+      - monitoring
+
+networks:
+  radicale:
+    external: true
+  monitoring:
+    external: true

roles/radicale/templates/config.j2

@@ -0,0 +1,10 @@
+[server]
+hosts = 0.0.0.0:{{ radicale_port }}
+
+[auth]
+type = htpasswd
+htpasswd_filename = /config/users
+htpasswd_encryption = bcrypt
+
+[storage]
+filesystem_folder = /data/collections

roles/restic/defaults/main.yml

@@ -0,0 +1,29 @@
+restic_backend_type: "sftp"
+restic_password: ""
+# restic_repo: set explicitly when restic_backend_type is not 'sftp'
+
+restic_backup_paths: >-
+  {{
+    ['/etc/letsencrypt', '/srv/caddy']
+    + (['/etc/nebula'] if (enable_nebula | default(false)) else [])
+    + (['/srv/forgejo'] if (enable_forgejo | default(false)) else [])
+    + (['/srv/goaccess'] if (enable_goaccess | default(false)) else [])
+    + (['/srv/mail'] if (enable_mail | default(false)) else [])
+    + (['/srv/monitoring'] if (enable_monitoring | default(false)) else [])
+    + (['/srv/tuwunel'] if (enable_tuwunel | default(false)) else [])
+    + (['/srv/radicale'] if (enable_radicale | default(false)) else [])
+    + (['/srv/diun'] if (enable_diun | default(false)) else [])
+  }}
+
+restic_exclude_patterns:
+  - "**/tmp"
+  - "**/cache"
+  - "**/*.gz"
+
+restic_backup_time: "02:00:00"
+restic_prune_time: "04:00:00"
+
+restic_retention:
+  daily: 7
+  weekly: 4
+  monthly: 6

roles/restic/files/restic_backup

@@ -0,0 +1 @@
+/home/matthias/.ssh/island_restic_backup

roles/restic/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: Reload systemd
+  systemd:
+    daemon_reload: true
+

roles/restic/tasks/backend.yml

@@ -0,0 +1,19 @@
+---
+- name: Ensure restic local repo directory exists
+  file:
+    path: "{{ restic_repo }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  when: restic_repo is defined and restic_repo.startswith('/')  # only local path
+
+- name: Ensure restic repo is initialized
+  ansible.builtin.shell: |
+    set -euo pipefail
+    source /etc/restic/restic.env
+    restic snapshots > /dev/null 2>&1 || restic init
+    touch /etc/restic/.initialized
+  args:
+    creates: /etc/restic/.initialized
+

roles/restic/tasks/backend_sftp.yml

@@ -0,0 +1,30 @@
+---
+- name: Deploy Restic SSH key
+  ansible.builtin.copy:
+    src: restic_backup         # local path in your playbook repo
+    dest: "{{ restic_ssh_key }}"    # e.g. /root/.ssh/restic_backup
+    owner: root
+    group: root
+    mode: '0600'
+
+- name: Ensure restic repo directory exists on Storage Box
+  ansible.builtin.shell: |
+    ssh -i {{ restic_ssh_key }} -o BatchMode=yes -o StrictHostKeyChecking=no -p {{ restic_ssh_port }} {{ restic_user }}@{{ restic_host }} \
+    "mkdir -p {{ restic_remote_path }} && chmod 700 {{ restic_remote_path }}" < /dev/null
+  changed_when: false
+
+- name: Write the ssh config for the root user
+  # TODO: this replaces roots config and should be much smarter, safe for me currently
+  template:
+    src: restic-ssh-config.j2
+    dest: /root/.ssh/config
+    mode: "0644"
+
+- name: Initialize restic repo on Storage Box (if needed)
+  ansible.builtin.shell: |
+    source /etc/restic/restic.env
+    restic snapshots > /dev/null 2>&1 || restic init
+    touch /etc/restic/.initialized
+  args:
+    creates: /etc/restic/.initialized
+

roles/restic/tasks/backup.yml

@@ -0,0 +1,34 @@
+- name: Install restic backup service
+  template:
+    src: restic-backup.service.j2
+    dest: /etc/systemd/system/restic-backup.service
+
+- name: Install restic backup timer
+  template:
+    src: restic-backup.timer.j2
+    dest: /etc/systemd/system/restic-backup.timer
+
+- name: Enable and start restic backup timer
+  systemd:
+    name: restic-backup.timer
+    enabled: true
+    state: started
+    daemon_reload: true
+
+- name: Install restic prune service
+  template:
+    src: restic-prune.service.j2
+    dest: /etc/systemd/system/restic-prune.service
+
+- name: Install restic prune timer
+  template:
+    src: restic-prune.timer.j2
+    dest: /etc/systemd/system/restic-prune.timer
+
+- name: Enable and start restic prune timer
+  systemd:
+    name: restic-prune.timer
+    enabled: true
+    state: started
+    daemon_reload: true
+

roles/restic/tasks/config.yml

@@ -0,0 +1,24 @@
+- name: Create restic config directory
+  file:
+    path: /etc/restic
+    state: directory
+    mode: "0700"
+
+- name: Write restic environment file
+  template:
+    src: restic.env.j2
+    dest: /etc/restic/restic.env
+    mode: "0600"
+
+- name: Write restic backup script
+  template:
+    src: restic-backup.sh.j2
+    dest: /usr/local/bin/restic-backup
+    mode: "0750"
+
+- name: Write restic prune script
+  template:
+    src: restic-prune.sh.j2
+    dest: /usr/local/bin/restic-prune
+    mode: "0750"
+

roles/restic/tasks/install.yml

@@ -0,0 +1,6 @@
+- name: Install restic
+  apt:
+    name: restic
+    state: present
+    update_cache: true
+

roles/restic/tasks/main.yml

@@ -0,0 +1,15 @@
+---
+- name: Install restic binary
+  include_tasks: install.yml
+
+- name: Configure restic environment
+  include_tasks: config.yml
+
+- name: Prepare backup repository
+  include_tasks: "{{ backend_file }}"
+  vars:
+    backend_file: "{{ 'backend_sftp.yml' if restic_backend_type == 'sftp' else 'backend.yml' }}"
+
+- name: Create systemd backup timer and service
+  include_tasks: backup.yml
+

roles/restic/templates/restic-backup.service.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Restic Backup
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/restic-backup
+

roles/restic/templates/restic-backup.sh.j2

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source /etc/restic/restic.env
+
+# Metrics file for node_exporter
+METRICS_DIR="/var/lib/node_exporter/textfile_collector"
+METRICS_FILE="${METRICS_DIR}/restic_backup.prom"
+mkdir -p "${METRICS_DIR}"
+
+# Temporary file for atomic writes
+TEMP_FILE=$(mktemp)
+
+# Start backup
+START_TIME=$(date +%s)
+
+if restic backup \
+{% for path in restic_backup_paths %}
+  {{ path }} \
+{% endfor %}
+{% for pattern in restic_exclude_patterns %}
+  --exclude '{{ pattern }}' \
+{% endfor %}
+  --host {{ ansible_facts["hostname"] }}; then
+  
+  # Backup succeeded
+  STATUS=1
+  echo "# HELP restic_backup_success Whether the last backup succeeded (1=success, 0=failure)" > "${TEMP_FILE}"
+  echo "# TYPE restic_backup_success gauge" >> "${TEMP_FILE}"
+  echo "restic_backup_success ${STATUS}" >> "${TEMP_FILE}"
+  
+  echo "# HELP restic_backup_timestamp_seconds Timestamp of last backup completion" >> "${TEMP_FILE}"
+  echo "# TYPE restic_backup_timestamp_seconds gauge" >> "${TEMP_FILE}"
+  echo "restic_backup_timestamp_seconds $(date +%s)" >> "${TEMP_FILE}"
+  
+  echo "# HELP restic_backup_duration_seconds Duration of last backup in seconds" >> "${TEMP_FILE}"
+  echo "# TYPE restic_backup_duration_seconds gauge" >> "${TEMP_FILE}"
+  echo "restic_backup_duration_seconds $(($(date +%s) - START_TIME))" >> "${TEMP_FILE}"
+  
+  # Move temp file to final location atomically
+  mv "${TEMP_FILE}" "${METRICS_FILE}"
+  
+  exit 0
+else
+  # Backup failed
+  STATUS=0
+  echo "# HELP restic_backup_success Whether the last backup succeeded (1=success, 0=failure)" > "${TEMP_FILE}"
+  echo "# TYPE restic_backup_success gauge" >> "${TEMP_FILE}"
+  echo "restic_backup_success ${STATUS}" >> "${TEMP_FILE}"
+  
+  echo "# HELP restic_backup_timestamp_seconds Timestamp of last backup attempt" >> "${TEMP_FILE}"
+  echo "# TYPE restic_backup_timestamp_seconds gauge" >> "${TEMP_FILE}"
+  echo "restic_backup_timestamp_seconds $(date +%s)" >> "${TEMP_FILE}"
+  
+  # Move temp file to final location atomically
+  mv "${TEMP_FILE}" "${METRICS_FILE}"
+  
+  exit 1
+fi

roles/restic/templates/restic-backup.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily Restic Backup
+
+[Timer]
+OnCalendar=*-*-* {{ restic_backup_time }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+

roles/restic/templates/restic-prune.service.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Restic Prune
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/restic-prune
+

roles/restic/templates/restic-prune.sh.j2

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source /etc/restic/restic.env
+
+# Metrics file for node_exporter
+METRICS_DIR="/var/lib/node_exporter/textfile_collector"
+METRICS_FILE="${METRICS_DIR}/restic_prune.prom"
+mkdir -p "${METRICS_DIR}"
+
+# Temporary file for atomic writes
+TEMP_FILE=$(mktemp)
+
+# Start prune
+START_TIME=$(date +%s)
+
+if restic forget \
+  --keep-daily {{ restic_retention.daily }} \
+  --keep-weekly {{ restic_retention.weekly }} \
+  --keep-monthly {{ restic_retention.monthly }} \
+  --prune; then
+  
+  # Prune succeeded
+  STATUS=1
+  echo "# HELP restic_prune_success Whether the last prune succeeded (1=success, 0=failure)" > "${TEMP_FILE}"
+  echo "# TYPE restic_prune_success gauge" >> "${TEMP_FILE}"
+  echo "restic_prune_success ${STATUS}" >> "${TEMP_FILE}"
+  
+  echo "# HELP restic_prune_timestamp_seconds Timestamp of last prune completion" >> "${TEMP_FILE}"
+  echo "# TYPE restic_prune_timestamp_seconds gauge" >> "${TEMP_FILE}"
+  echo "restic_prune_timestamp_seconds $(date +%s)" >> "${TEMP_FILE}"
+  
+  echo "# HELP restic_prune_duration_seconds Duration of last prune in seconds" >> "${TEMP_FILE}"
+  echo "# TYPE restic_prune_duration_seconds gauge" >> "${TEMP_FILE}"
+  echo "restic_prune_duration_seconds $(($(date +%s) - START_TIME))" >> "${TEMP_FILE}"
+  
+  # Move temp file to final location atomically
+  mv "${TEMP_FILE}" "${METRICS_FILE}"
+  
+  exit 0
+else
+  # Prune failed
+  STATUS=0
+  echo "# HELP restic_prune_success Whether the last prune succeeded (1=success, 0=failure)" > "${TEMP_FILE}"
+  echo "# TYPE restic_prune_success gauge" >> "${TEMP_FILE}"
+  echo "restic_prune_success ${STATUS}" >> "${TEMP_FILE}"
+  
+  echo "# HELP restic_prune_timestamp_seconds Timestamp of last prune attempt" >> "${TEMP_FILE}"
+  echo "# TYPE restic_prune_timestamp_seconds gauge" >> "${TEMP_FILE}"
+  echo "restic_prune_timestamp_seconds $(date +%s)" >> "${TEMP_FILE}"
+  
+  # Move temp file to final location atomically
+  mv "${TEMP_FILE}" "${METRICS_FILE}"
+  
+  exit 1
+fi

roles/restic/templates/restic-prune.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily Restic Prune
+
+[Timer]
+OnCalendar=*-*-* {{ restic_prune_time }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+

roles/restic/templates/restic-ssh-config.j2

@@ -0,0 +1,4 @@
+Host {{ restic_host }}
+  IdentityFile {{ restic_ssh_key }}
+  User {{ restic_user }}
+  Port {{ restic_ssh_port }}

roles/restic/templates/restic.env.j2

@@ -0,0 +1,7 @@
+{% if restic_backend_type == 'sftp' %}
+export RESTIC_REPOSITORY="sftp:{{ restic_user }}@{{ restic_host }}:{{ restic_remote_path }}"
+{% else %}
+export RESTIC_REPOSITORY="{{ restic_repo }}"
+{% endif %}
+export RESTIC_PASSWORD="{{ restic_password }}"
+export RESTIC_CACHE_DIR=/var/cache/restic

roles/tuwunel/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+# Port (internal to docker network)
+tuwunel_port: 6167
+
+# Trusted Matrix servers for federation
+tuwunel_trusted_servers:
+  - matrix.org

roles/tuwunel/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart tuwunel
+  community.docker.docker_compose_v2:
+    project_src: /srv/tuwunel
+    state: restarted
+    build: never

roles/tuwunel/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: Create Tuwunel directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - /srv/tuwunel
+    - /srv/tuwunel/data
+
+- name: Deploy Tuwunel configuration
+  ansible.builtin.template:
+    src: tuwunel.toml.j2
+    dest: /srv/tuwunel/tuwunel.toml
+    mode: '0644'
+  notify: Restart tuwunel
+
+- name: Deploy Tuwunel docker-compose file
+  ansible.builtin.template:
+    src: compose.yml.j2
+    dest: /srv/tuwunel/compose.yml
+    mode: '0644'
+  notify: Restart tuwunel
+
+- name: Start Tuwunel service
+  community.docker.docker_compose_v2:
+    project_src: /srv/tuwunel
+    state: present
+    build: never
+  register: tuwunel_output

roles/tuwunel/templates/compose.yml.j2

@@ -0,0 +1,16 @@
+services:
+  tuwunel:
+    image: ghcr.io/matrix-construct/tuwunel:{{ tuwunel_version }}
+    container_name: tuwunel
+    restart: unless-stopped
+    environment:
+      TUWUNEL_CONFIG: /etc/tuwunel.toml
+    volumes:
+      - /srv/tuwunel/data:/var/lib/tuwunel
+      - /srv/tuwunel/tuwunel.toml:/etc/tuwunel.toml:ro
+    networks:
+      - tuwunel
+
+networks:
+  tuwunel:
+    external: true