2026-05-16 09:46:12 +02:00
10 changed files with 323 additions and 39 deletions
--- a/.env.example
+++ b/.env.example
@@ -29,16 +29,17 @@ OCR_TRAINING_TOKEN=change-me-in-production
 # --- Observability ---
 # Optional stack — start with: docker compose -f docker-compose.observability.yml up -d
 # Requires the main stack to already be running (docker compose up -d creates archiv-net).
+# In production the stack is managed from /opt/familienarchiv/ (see docs/DEPLOYMENT.md §4).

 # Ports for host access
-PORT_GRAFANA=3001
+PORT_GRAFANA=3003
 PORT_GLITCHTIP=3002
 PORT_PROMETHEUS=9090

 # Grafana admin password — change this before exposing Grafana beyond localhost
 GRAFANA_ADMIN_PASSWORD=changeme

-# GlitchTip domain — production: use https://grafana.raddatz.cloud (must match Caddy vhost)
+# GlitchTip domain — production: use https://glitchtip.archiv.raddatz.cloud (must match Caddy vhost)
 GLITCHTIP_DOMAIN=http://localhost:3002

 # GlitchTip secret key — Django SECRET_KEY equivalent, used to sign sessions and tokens.
@@ -47,6 +48,15 @@ GLITCHTIP_DOMAIN=http://localhost:3002
 # Generate with: python3 -c "import secrets; print(secrets.token_hex(50))"
 GLITCHTIP_SECRET_KEY=changeme-generate-a-real-secret

+# PostgreSQL hostname for GlitchTip's db-init job and workers.
+# Override when only the staging stack is running (container name differs from archive-db).
+# Default (archive-db) is correct for production with the full stack up.
+POSTGRES_HOST=archive-db
+
+# $$ escaping note: passwords in /opt/familienarchiv/.env that contain a literal '$' must
+# use '$$' so Docker Compose does not expand them as variable references.
+# Example: a password 'p@$$word' should be written as 'p@$$$$word' in the .env file.
+
 # Error reporting DSNs — leave empty to disable the SDK (safe default).
 # SENTRY_DSN: backend (Spring Boot) — used by the GlitchTip/Sentry Java SDK
 SENTRY_DSN=
--- a/.gitea/workflows/nightly.yml
+++ b/.gitea/workflows/nightly.yml
@@ -78,12 +78,6 @@ jobs:
          APP_MAIL_FROM=noreply@staging.raddatz.cloud
          IMPORT_HOST_DIR=/srv/familienarchiv-staging/import
          POSTGRES_USER=archiv
-          PORT_GRAFANA=3003
-          PORT_GLITCHTIP=3002
-          PORT_PROMETHEUS=9090
-          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
-          GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
-          GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud
          SENTRY_DSN=${{ secrets.SENTRY_DSN }}
          EOF

@@ -131,13 +125,76 @@ jobs:
            --profile staging \
            up -d --wait --remove-orphans

-      - name: Start observability stack
+      - name: Deploy observability configs
+        # Copies the compose file and config tree from the workspace checkout
+        # into /opt/familienarchiv/ — the permanent location that persists
+        # between CI runs. Containers started in the next step bind-mount
+        # from there, so a future workspace wipe cannot corrupt a running
+        # config file.
+        #
+        # obs-secrets.env is written fresh from Gitea secrets on every run so
+        # Gitea is always the single source of truth for secret rotation.
+        # Non-secret config lives in infra/observability/obs.env (tracked in git).
+        run: |
+          mkdir -p /opt/familienarchiv/infra/observability
+          rsync -a --delete infra/observability/ /opt/familienarchiv/infra/observability/
+          cp docker-compose.observability.yml /opt/familienarchiv/
+          cat > /opt/familienarchiv/obs-secrets.env <<'EOF'
+          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
+          POSTGRES_PASSWORD=${{ secrets.STAGING_POSTGRES_PASSWORD }}
+          POSTGRES_HOST=archiv-staging-db-1
+          EOF
+          # Note: POSTGRES_HOST is derived from the Compose project name (archiv-staging)
+          # and service name (db). A project rename requires updating this value.
+          chmod 600 /opt/familienarchiv/obs-secrets.env
+
+      - name: Validate observability compose config
+        # Dry-run: resolves all variable substitutions and reports any missing
+        # required keys before containers start. Catches undefined variables and
+        # YAML errors in config files updated by the previous step.
+        # --env-file order: obs.env first (git-tracked defaults), obs-secrets.env
+        # second (CI-written secrets). Later files win on duplicate keys, so
+        # obs-secrets.env overrides POSTGRES_HOST set in obs.env.
        run: |
          docker compose \
-            -f docker-compose.observability.yml \
-            --env-file .env.staging \
+            -f /opt/familienarchiv/docker-compose.observability.yml \
+            --env-file /opt/familienarchiv/infra/observability/obs.env \
+            --env-file /opt/familienarchiv/obs-secrets.env \
+            config --quiet
+
+      - name: Start observability stack
+        # Runs with absolute paths so bind mounts resolve to stable host paths
+        # that survive workspace wipes between nightly runs (see ADR-016).
+        # Non-secret config from obs.env (git-tracked); secrets from obs-secrets.env
+        # (written fresh from Gitea secrets above). --env-file order: obs.env first,
+        # obs-secrets.env second — later file wins on duplicate keys.
+        run: |
+          docker compose \
+            -f /opt/familienarchiv/docker-compose.observability.yml \
+            --env-file /opt/familienarchiv/infra/observability/obs.env \
+            --env-file /opt/familienarchiv/obs-secrets.env \
            up -d --wait --remove-orphans

+      - name: Assert observability stack health
+        # docker compose up --wait covers services WITH healthcheck directives only.
+        # obs-promtail, obs-cadvisor, obs-node-exporter, and obs-glitchtip-worker have
+        # no healthcheck — they are considered "started" as soon as the process runs.
+        # This step explicitly asserts the five healthchecked critical services are
+        # healthy before the smoke test proceeds.
+        run: |
+          set -e
+          unhealthy=""
+          for svc in obs-loki obs-prometheus obs-grafana obs-tempo obs-glitchtip; do
+            status=$(docker inspect "$svc" --format '{{.State.Health.Status}}' 2>/dev/null || echo "missing")
+            if [ "$status" != "healthy" ]; then
+              echo "::error::$svc is not healthy (status: $status)"
+              unhealthy="$unhealthy $svc"
+            fi
+          done
+          [ -z "$unhealthy" ] || exit 1
+          echo "All critical observability services are healthy"
+
      - name: Reload Caddy
        # Apply any committed Caddyfile changes before smoke-testing the
        # public surface. Without this step, a Caddyfile edit lands in the
--- a/.gitea/workflows/release.yml
+++ b/.gitea/workflows/release.yml
@@ -76,12 +76,6 @@ jobs:
          APP_MAIL_FROM=noreply@raddatz.cloud
          IMPORT_HOST_DIR=/srv/familienarchiv-production/import
          POSTGRES_USER=archiv
-          PORT_GRAFANA=3003
-          PORT_GLITCHTIP=3002
-          PORT_PROMETHEUS=9090
-          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
-          GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
-          GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud
          SENTRY_DSN=${{ secrets.SENTRY_DSN }}
          EOF

@@ -104,13 +98,74 @@ jobs:
            --env-file .env.production \
            up -d --wait --remove-orphans

-      - name: Start observability stack
+      - name: Deploy observability configs
+        # Mirrors the nightly approach: copies obs compose file and config tree
+        # to /opt/familienarchiv/ (permanent path, survives workspace wipes — ADR-016),
+        # then writes obs-secrets.env fresh from Gitea secrets.
+        # Non-secret config lives in infra/observability/obs.env (tracked in git).
+        run: |
+          mkdir -p /opt/familienarchiv/infra/observability
+          rsync -a --delete infra/observability/ /opt/familienarchiv/infra/observability/
+          cp docker-compose.observability.yml /opt/familienarchiv/
+          cat > /opt/familienarchiv/obs-secrets.env <<'EOF'
+          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
+          POSTGRES_PASSWORD=${{ secrets.PROD_POSTGRES_PASSWORD }}
+          POSTGRES_HOST=archiv-production-db-1
+          EOF
+          # Note: POSTGRES_HOST is derived from the Compose project name (archiv-production)
+          # and service name (db). A project rename requires updating this value.
+          chmod 600 /opt/familienarchiv/obs-secrets.env
+
+      - name: Validate observability compose config
+        # Dry-run: resolves all variable substitutions and reports any missing
+        # required keys before containers start. Catches undefined variables and
+        # YAML errors in config files updated by the previous step.
+        # --env-file order: obs.env first (git-tracked defaults), obs-secrets.env
+        # second (CI-written secrets). Later files win on duplicate keys, so
+        # obs-secrets.env overrides POSTGRES_HOST set in obs.env.
+        # Keep in sync with the equivalent step in nightly.yml (#603).
        run: |
          docker compose \
-            -f docker-compose.observability.yml \
-            --env-file .env.production \
+            -f /opt/familienarchiv/docker-compose.observability.yml \
+            --env-file /opt/familienarchiv/infra/observability/obs.env \
+            --env-file /opt/familienarchiv/obs-secrets.env \
+            config --quiet
+
+      - name: Start observability stack
+        # Runs with absolute paths so bind mounts resolve to stable host paths
+        # that survive workspace wipes between runs (see ADR-016).
+        # Non-secret config from obs.env (git-tracked); secrets from obs-secrets.env
+        # (written fresh from Gitea secrets above). --env-file order: obs.env first,
+        # obs-secrets.env second — later file wins on duplicate keys.
+        # Keep in sync with the equivalent step in nightly.yml (#603).
+        run: |
+          docker compose \
+            -f /opt/familienarchiv/docker-compose.observability.yml \
+            --env-file /opt/familienarchiv/infra/observability/obs.env \
+            --env-file /opt/familienarchiv/obs-secrets.env \
            up -d --wait --remove-orphans

+      - name: Assert observability stack health
+        # docker compose up --wait covers services WITH healthcheck directives only.
+        # obs-promtail, obs-cadvisor, obs-node-exporter, and obs-glitchtip-worker have
+        # no healthcheck — they are considered "started" as soon as the process runs.
+        # This step explicitly asserts the five healthchecked critical services are
+        # healthy before the smoke test proceeds.
+        # Keep in sync with the equivalent step in nightly.yml (#603).
+        run: |
+          set -e
+          unhealthy=""
+          for svc in obs-loki obs-prometheus obs-grafana obs-tempo obs-glitchtip; do
+            status=$(docker inspect "$svc" --format '{{.State.Health.Status}}' 2>/dev/null || echo "missing")
+            if [ "$status" != "healthy" ]; then
+              echo "::error::$svc is not healthy (status: $status)"
+              unhealthy="$unhealthy $svc"
+            fi
+          done
+          [ -z "$unhealthy" ] || exit 1
+          echo "All critical observability services are healthy"
+
      - name: Reload Caddy
        # See nightly.yml — same rationale and mechanism: DooD job containers
        # cannot call systemctl directly; nsenter via a privileged sibling
--- a/docker-compose.observability.yml
+++ b/docker-compose.observability.yml
@@ -142,10 +142,11 @@ services:
    container_name: obs-grafana
    restart: unless-stopped
    ports:
-      - "127.0.0.1:${PORT_GRAFANA:-3001}:3000"
+      - "127.0.0.1:${PORT_GRAFANA:-3003}:3000"
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-changeme}
      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL:-http://localhost:3003}
    volumes:
      - grafana_data:/var/lib/grafana
      - ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro
@@ -193,7 +194,7 @@ services:
      obs-glitchtip-db-init:
        condition: service_completed_successfully
    environment:
-      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@archive-db:5432/glitchtip
+      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-archive-db}:5432/glitchtip
      REDIS_URL: redis://obs-redis:6379/0
      SECRET_KEY: ${GLITCHTIP_SECRET_KEY}
      GLITCHTIP_DOMAIN: ${GLITCHTIP_DOMAIN:-http://localhost:3002}
@@ -202,6 +203,12 @@ services:
      GLITCHTIP_MAX_EVENT_LIFE_DAYS: 90
    ports:
      - "127.0.0.1:${PORT_GLITCHTIP:-3002}:8080"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:8080/_health/ || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
    networks:
      - archiv-net
      - obs-net
@@ -215,7 +222,7 @@ services:
      obs-redis:
        condition: service_healthy
    environment:
-      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@archive-db:5432/glitchtip
+      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-archive-db}:5432/glitchtip
      REDIS_URL: redis://obs-redis:6379/0
      SECRET_KEY: ${GLITCHTIP_SECRET_KEY}
    networks:
@@ -229,10 +236,10 @@ services:
    environment:
      PGPASSWORD: ${POSTGRES_PASSWORD}
    command: >
-      sh -c "psql -h archive-db -U ${POSTGRES_USER} -tc
+      sh -c "psql -h ${POSTGRES_HOST:-archive-db} -U ${POSTGRES_USER} -tc
      \"SELECT 1 FROM pg_database WHERE datname = 'glitchtip'\" |
      grep -q 1 ||
-      psql -h archive-db -U ${POSTGRES_USER} -c \"CREATE DATABASE glitchtip;\""
+      psql -h ${POSTGRES_HOST:-archive-db} -U ${POSTGRES_USER} -c \"CREATE DATABASE glitchtip;\""
    networks:
      - archiv-net

--- a/docs/DEPLOYMENT.md
+++ b/docs/DEPLOYMENT.md
@@ -43,7 +43,7 @@ graph TD
 - SSE notifications transit Caddy (browser → Caddy → backend); the backend is never reachable directly from the public internet. The SvelteKit SSR layer is bypassed for SSE, but Caddy is not.
 - The Caddyfile responds `404` on `/actuator/*` (defense in depth). Internal monitoring scrapes the backend on the docker network, not through Caddy.
 - Production and staging cohabit on the same host via docker compose project names: `archiv-production` (ports 8080/3000) and `archiv-staging` (ports 8081/3001).
- An optional observability stack (Prometheus, Node Exporter, cAdvisor) runs as a separate compose file: `docker compose -f docker-compose.observability.yml up -d`. It joins `archiv-net` and scrapes the backend's management port (`:8081`). Configuration lives under `infra/observability/`.
+- An optional observability stack (Prometheus, Node Exporter, cAdvisor, Loki, Tempo, Grafana, GlitchTip) runs as a separate compose file. Configuration lives under `infra/observability/`. In production and CI, the stack is managed from `/opt/familienarchiv/` (CI copies it there on every nightly run) so bind mounts survive workspace wipes — see §4 for the ops procedure.

 ### OCR memory requirements

@@ -142,7 +142,8 @@ All vars are set in `.env` at the repo root (copy from `.env.example`). The back
 | Variable | Purpose | Default | Required? | Sensitive? |
 |---|---|---|---|---|
 | `PORT_PROMETHEUS` | Host port for the Prometheus UI (bound to `127.0.0.1` only) | `9090` | — | — |
-| `PORT_GRAFANA` | Host port for the Grafana UI (bound to `127.0.0.1` only) | `3001` | — | — |
+| `PORT_GRAFANA` | Host port for the Grafana UI (bound to `127.0.0.1` only) | `3003` | — | — |
+| `POSTGRES_HOST` | PostgreSQL hostname for GlitchTip's db-init job and workers. Override when only the staging stack is running and `archive-db` is not resolvable by that name. | `archive-db` | — | — |
 | `GRAFANA_ADMIN_PASSWORD` | Grafana `admin` user password | `changeme` | YES (prod) | YES |
 | `PORT_GLITCHTIP` | Host port for the GlitchTip UI (bound to `127.0.0.1` only) | `3002` | — | — |
 | `GLITCHTIP_DOMAIN` | Public-facing base URL for GlitchTip (used in email links and CORS) | `http://localhost:3002` | YES (prod) | — |
@@ -202,6 +203,18 @@ mkdir -p /srv/gitea-workspace
 #   volumes:
 #     - /srv/gitea-workspace:/srv/gitea-workspace
 # See runner-config.yaml (workdir_parent + valid_volumes + options) and ADR-015.
+
+# Observability config permanent directory — the nightly CI job copies
+# docker-compose.observability.yml and infra/observability/ here on every run.
+# The obs stack is always started from this path, not from the workspace.
+# See ADR-016 for why this directory is used instead of a server-pull approach.
+mkdir -p /opt/familienarchiv/infra
+
+# ⚠ IMPORTANT: after any change to runner-config.yaml (valid_volumes, options, workdir_parent),
+# restart the Gitea Act runner on the host for the new config to take effect:
+#   systemctl restart gitea-runner
+# Until restarted, job containers are spawned with the old config and any new bind mounts
+# (e.g. /opt/familienarchiv) will not be available inside job steps.
 ```

 ### 3.2 DNS records
@@ -284,13 +297,70 @@ docker compose logs --tail=200 <service>

 ### Observability stack

-An observability stack is available via `docker-compose.observability.yml`. Configuration lives under `infra/observability/`. Start it after the main stack is up (which creates `archiv-net`):
+An observability stack is available via `docker-compose.observability.yml`. Configuration lives under `infra/observability/`.
+
+#### Dev — start from the workspace

 ```bash
 docker compose up -d                                    # creates archiv-net
 docker compose -f docker-compose.observability.yml up -d
 ```

+#### Why the obs stack is managed differently from the main app stack
+
+The main app stack (`docker-compose.prod.yml`) has no config-file bind mounts — its containers read config from env vars and image defaults. The workspace is wiped after each CI run but that does not affect running containers, because they hold no references to workspace paths.
+
+The obs stack is different: `prometheus.yml`, `tempo.yml`, Loki config, Grafana provisioning files, and Promtail config are all bind-mounted from the host filesystem into their containers. If those source paths disappear (workspace wipe), the containers can restart fine until a `docker compose up` is run again — at that point Docker tries to re-resolve the bind-mount source and fails because the workspace path no longer exists.
+
+The fix is to keep the obs compose file and config tree at a **permanent path** that CI copies to on every run but which survives between runs: `/opt/familienarchiv/` (see ADR-016).
+
+#### Production — managed from `/opt/familienarchiv/`
+
+Every CI run (nightly + release) copies `docker-compose.observability.yml` and `infra/observability/` to `/opt/familienarchiv/` before starting the stack. Bind mounts then resolve to `/opt/familienarchiv/infra/observability/…` — a stable path that outlasts any workspace wipe.
+
+**Environment variables** follow the same two-source model as the main stack:
+
+| Source | What it contains | Managed by |
+|---|---|---|
+| `infra/observability/obs.env` | All non-secret config (ports, URLs, hostnames) | Git — reviewed in PRs |
+| `/opt/familienarchiv/obs-secrets.env` | Passwords and secret keys only | CI — written fresh from Gitea secrets on every deploy |
+
+Both files are passed explicitly via `--env-file` to the compose command, so there is no implicit auto-read `.env` and no operator-managed file to keep in sync.
+
+**Non-secret config** (`infra/observability/obs.env`):
+
+| Key | Value | Notes |
+|---|---|---|
+| `PORT_GRAFANA` | `3003` | Avoids collision with staging frontend on port 3001 |
+| `PORT_GLITCHTIP` | `3002` | |
+| `PORT_PROMETHEUS` | `9090` | |
+| `GF_SERVER_ROOT_URL` | `https://grafana.archiv.raddatz.cloud` | Required for alert email links and OAuth redirects |
+| `GLITCHTIP_DOMAIN` | `https://glitchtip.archiv.raddatz.cloud` | Must match the Caddy vhost |
+| `POSTGRES_HOST` | `archive-db` | Override if only the staging stack is running |
+
+**Secret keys** (set in Gitea secrets, injected by CI into `obs-secrets.env`):
+
+| Gitea secret | Notes |
+|---|---|
+| `GRAFANA_ADMIN_PASSWORD` | Strong unique password; shared by nightly and release |
+| `GLITCHTIP_SECRET_KEY` | `openssl rand -hex 32`; shared by nightly and release |
+| `STAGING_POSTGRES_PASSWORD` / `PROD_POSTGRES_PASSWORD` | Must match the running PostgreSQL container |
+
+To start or restart the obs stack manually on the server (after CI has run at least once):
+
+```bash
+docker compose \
+  -f /opt/familienarchiv/docker-compose.observability.yml \
+  --env-file /opt/familienarchiv/infra/observability/obs.env \
+  --env-file /opt/familienarchiv/obs-secrets.env \
+  up -d --wait --remove-orphans
+```
+
+> **Note:** `cp -r` does not remove deleted files. If a config file is removed from the repo, its stale copy persists at `/opt/familienarchiv/infra/observability/` until manually deleted:
+> ```bash
+> rm /opt/familienarchiv/infra/observability/<path-to-removed-file>
+> ```
+
 Current services:

 | Service | Image | Purpose |
@@ -311,7 +381,7 @@ Current services:

 | Item | Value |
 |---|---|
-| URL | `http://localhost:3001` (or `http://localhost:$PORT_GRAFANA`) |
+| URL | `http://localhost:3003` (or `http://localhost:$PORT_GRAFANA`) |
 | Username | `admin` |
 | Password | `$GRAFANA_ADMIN_PASSWORD` (default: `changeme` — **change before exposing to a network**) |

@@ -341,7 +411,7 @@ docker exec obs-loki wget -qO- \

 **Prefer `compose_service` over `container_name` in LogQL queries** — `container_name` differs between dev (`archive-backend`) and prod (`archiv-production-backend-1`), while `compose_service` is stable (`backend`, `db`, `minio`, etc.).

-Prometheus port `9090` and Grafana port `3001` are bound to `127.0.0.1` on the host. No other observability ports are host-bound.
+Prometheus port `9090` and Grafana port `3003` (default; configurable via `PORT_GRAFANA`) are bound to `127.0.0.1` on the host. No other observability ports are host-bound.

 #### GlitchTip

--- a/docs/adr/016-obs-stack-co-location-ci-push.md
+++ b/docs/adr/016-obs-stack-co-location-ci-push.md
@@ -0,0 +1,57 @@
+# ADR-016: Observability stack co-location at `/opt/familienarchiv/` with CI-push config sync
+
+## Status
+
+Accepted
+
+## Context
+
+Issue #601 established that the observability stack must survive Gitea CI workspace wipes between nightly runs. When the nightly job completes, act_runner deletes the job workspace. Any Docker container that bind-mounts a config file from a workspace path (`/srv/gitea-workspace/…/infra/observability/prometheus/prometheus.yml`) then references a path that no longer exists on the host. On the next nightly run, Docker Compose either auto-creates an empty directory in its place (causing the container to fail to start because a file mount receives a directory) or finds a stale file from a previous run if the workspace happened to land at the same path.
+
+ADR-015 solved the workspace bind-mount resolution problem: job workspaces are stored at `/srv/gitea-workspace` so `$(pwd)` inside the job container maps to a real host path. But it did not address persistence: the workspace is still wiped after the job, so bind mounts from workspace-relative paths remain fragile across runs.
+
+### Decision drivers
+
+1. Bind-mount sources must point to a host path that persists indefinitely, not to a path that disappears after each CI run.
+2. Config files must reflect the committed state of the repo after every nightly run (no manual sync steps).
+3. Secrets must not be written to the workspace or to any path managed by CI; they must survive independently of deployments.
+4. The solution must not introduce new infrastructure dependencies (no SSH access from CI, no external registry, no additional server-side daemon).
+
+### Alternatives considered
+
+**A: Server-pull model** — a systemd timer or cron job on the server does `git pull` from the repo into `/opt/familienarchiv/` and then runs `docker compose up`. Rejected because: (1) requires git credentials on the server and a registered deploy key, (2) adds a second deployment mechanism that diverges from the CI-push model used for the main app stack, (3) timing coupling — the server pull must complete before CI's health checks run, requiring polling or a webhook.
+
+**B: Separate directory (e.g. `/opt/obs/`)** — keeps obs configs isolated from the app stack. Rejected because: (1) the main app compose files are already in `/opt/familienarchiv/` (managed the same way), and (2) GlitchTip shares the `archive-db` PostgreSQL instance and `archiv-net` Docker network — it is architecturally part of the same deployment unit, not a separate one. Co-location reflects the actual coupling.
+
+**C: Named Docker configs (Swarm)** — Docker Swarm supports first-class config objects that persist in the cluster. Rejected because the project does not use Swarm and introducing it solely for config persistence is a disproportionate dependency.
+
+## Decision
+
+The observability stack is co-located with the main application deployment at `/opt/familienarchiv/`:
+
+- `docker-compose.observability.yml` → `/opt/familienarchiv/docker-compose.observability.yml`
+- `infra/observability/` → `/opt/familienarchiv/infra/observability/`
+
+Both the nightly CI job (`nightly.yml`) and the release job (`release.yml`) copy these files from the workspace checkout to `/opt/familienarchiv/` using `cp -r` on every run (CI-push model). Containers always read config from the permanent location; a workspace wipe has no effect on running containers.
+
+Environment variables follow a two-source model:
+
+- `infra/observability/obs.env` (git-tracked, non-secret): all non-sensitive config — host ports, public URLs (`GLITCHTIP_DOMAIN`, `GF_SERVER_ROOT_URL`), and the default `POSTGRES_HOST`. Changes go through PR review. No credentials.
+- `/opt/familienarchiv/obs-secrets.env` (CI-written, per-deploy): passwords and secret keys only (`GRAFANA_ADMIN_PASSWORD`, `GLITCHTIP_SECRET_KEY`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_HOST`), injected fresh from Gitea secrets on every nightly and release deploy. Gitea is the single source of truth for secrets — rotating a secret takes effect on the next deploy without manual server action.
+
+Both files are passed explicitly via `--env-file` to every obs compose command (config dry-run and `up`). There is no implicit auto-read `.env`. The required key inventory is documented in `docs/DEPLOYMENT.md §4`.
+
+The CI runner mounts `/opt/familienarchiv` as a bind mount into job containers (see `runner-config.yaml`). This requires a one-time `mkdir -p /opt/familienarchiv/infra` on the server and a runner restart after updating `runner-config.yaml` (see ADR-015 and `docs/DEPLOYMENT.md §3.1`).
+
+## Consequences
+
+**Positive:**
+- Bind-mount sources survive workspace wipes by definition — they are on a persistent host path.
+- Config is always in sync with the repo after each nightly run.
+- No new infrastructure dependencies; the CI-push model mirrors how the main app stack is deployed.
+- Secret rotation requires no manual server action — Gitea secrets are the authoritative store; `obs-secrets.env` is rewritten from scratch on every deploy so a secret change takes effect on the next nightly or release run.
+
+**Negative:**
+- `cp -r` does not remove deleted files; a config file removed from the repo persists in `/opt/familienarchiv/infra/observability/` until manually deleted. Acceptable for this project's change frequency. A `rsync -a --delete` would give a clean mirror if this becomes a problem.
+- Mounting `/opt/familienarchiv/` into CI job containers expands the blast radius of a compromised workflow step — a malicious step could overwrite app compose files and Caddy config. Acceptable because the runner is single-tenant (trusted code only). See `runner-config.yaml` security comment.
+- Runner must be restarted (`systemctl restart gitea-runner`) after any change to `runner-config.yaml` for the new mount to take effect.
--- a/docs/architecture/c4/l2-containers.puml
+++ b/docs/architecture/c4/l2-containers.puml
@@ -17,7 +17,7 @@ System_Boundary(archiv, "Familienarchiv (Docker Compose)") {
    Container(mc, "Bucket / Service-Account Init", "MinIO Client (mc)", "One-shot container on startup. Idempotent: creates the archive bucket, the archiv-app service account, and attaches the readwrite policy.")
 }

-System_Boundary(observability, "Observability Stack (docker-compose.observability.yml)") {
+System_Boundary(observability, "Observability Stack (/opt/familienarchiv/docker-compose.observability.yml)") {
    Container(prometheus, "Prometheus", "prom/prometheus:v3.4.0", "Scrapes metrics from backend management port 8081 (/actuator/prometheus), node-exporter, and cAdvisor. Retention: 30 days.")
    Container(node_exporter, "Node Exporter", "prom/node-exporter:v1.9.0", "Host-level CPU, memory, disk, and network metrics.")
    Container(cadvisor, "cAdvisor", "gcr.io/cadvisor/cadvisor:v0.52.1", "Per-container resource metrics.")
--- a/infra/observability/obs.env
+++ b/infra/observability/obs.env
@@ -0,0 +1,24 @@
+# Non-secret observability stack configuration — tracked in git.
+# Secret values (passwords, keys) are injected by CI from Gitea secrets
+# into /opt/familienarchiv/obs-secrets.env at deploy time.
+#
+# For local dev the main .env file supplies these values instead;
+# this file is only used in the CI/production path.
+
+# Host ports (all bound to 127.0.0.1 — Caddy is the external entry point)
+PORT_GRAFANA=3003
+PORT_GLITCHTIP=3002
+PORT_PROMETHEUS=9090
+
+# Public URLs — used for internal redirects, alert email links, OAuth callbacks
+GF_SERVER_ROOT_URL=https://grafana.archiv.raddatz.cloud
+GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud
+
+POSTGRES_USER=archiv
+
+# PostgreSQL hostname for GlitchTip db-init and workers.
+# The actual value depends on the Compose project name — it is not a fixed string.
+# CI sets POSTGRES_HOST in obs-secrets.env per environment:
+#   staging:    archiv-staging-db-1    (project archiv-staging + service db)
+#   production: archiv-production-db-1 (project archiv-production + service db)
+# For local dev, set POSTGRES_HOST in your .env file (defaults to archive-db there).
--- a/infra/observability/tempo/tempo.yml
+++ b/infra/observability/tempo/tempo.yml
@@ -36,9 +36,6 @@ metrics_generator:
      source: tempo
  storage:
    path: /var/tempo/generator/wal
-  processors:
-    - service-graphs
-    - span-metrics

 # Tempo HTTP API (port 3200) is unauthenticated. Access is controlled entirely
 # by network isolation: only Grafana (on obs-net) should reach this port.
--- a/runner-config.yaml
+++ b/runner-config.yaml
@@ -15,12 +15,19 @@ container:
  valid_volumes:
    - "/var/run/docker.sock"
    - "/srv/gitea-workspace"
+    - "/opt/familienarchiv"
  # appended to `docker run` when the runner spawns a job container
-  # SECURITY: Mounting the Docker socket grants job containers root-equivalent
-  # access to the host Docker daemon. Acceptable here because only trusted code
-  # from this private repo runs on this runner. Do NOT use on a runner that
-  # accepts untrusted PRs from external contributors.
-  options: "-v /var/run/docker.sock:/var/run/docker.sock -v /srv/gitea-workspace:/srv/gitea-workspace"
+  # SECURITY WARNING: This mount configuration grants CI job containers:
+  #   1. Root-equivalent access to the host Docker daemon (via the socket).
+  #   2. Read/write access to /opt/familienarchiv/ — including the main app's
+  #      compose files, Caddy config, and observability configs. A malicious
+  #      workflow step could overwrite any file in that directory.
+  # Both are acceptable ONLY because this runner is single-tenant: it executes
+  # code exclusively from this private repo with a fixed set of trusted authors.
+  # WARNING: Do NOT add this runner to any repo with external contributors or
+  # untrusted PRs — the blast radius includes the entire production deployment.
+  # See ADR-016 for the reasoning behind the /opt/familienarchiv mount.
+  options: "-v /var/run/docker.sock:/var/run/docker.sock -v /srv/gitea-workspace:/srv/gitea-workspace -v /opt/familienarchiv:/opt/familienarchiv"
  # keep network mode default (bridge) — Testcontainers handles its own networking
  force_pull: false