fix(obs): set POSTGRES_HOST per environment — staging/prod use compose auto-names not archive-db

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.gitea/workflows/nightly.yml

@@ -78,12 +78,6 @@ jobs:
           APP_MAIL_FROM=noreply@staging.raddatz.cloud
           IMPORT_HOST_DIR=/srv/familienarchiv-staging/import
           POSTGRES_USER=archiv
-          PORT_GRAFANA=3003
-          PORT_GLITCHTIP=3002
-          PORT_PROMETHEUS=9090
-          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
-          GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
-          GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud
           SENTRY_DSN=${{ secrets.SENTRY_DSN }}
           EOF
 
@@ -136,30 +130,44 @@ jobs:
         # into /opt/familienarchiv/ — the permanent location that persists
         # between CI runs. Containers started in the next step bind-mount
         # from there, so a future workspace wipe cannot corrupt a running
-        # config file. Secrets are read from /opt/familienarchiv/.env (managed
-        # separately on the server; not written or deleted by CI).
+        # config file.
+        #
+        # obs-secrets.env is written fresh from Gitea secrets on every run so
+        # Gitea is always the single source of truth for secret rotation.
+        # Non-secret config lives in infra/observability/obs.env (tracked in git).
         run: |
           mkdir -p /opt/familienarchiv/infra
           cp -r infra/observability /opt/familienarchiv/infra/
           cp docker-compose.observability.yml /opt/familienarchiv/
+          cat > /opt/familienarchiv/obs-secrets.env <<EOF
+          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
+          POSTGRES_USER=archiv
+          POSTGRES_PASSWORD=${{ secrets.STAGING_POSTGRES_PASSWORD }}
+          POSTGRES_HOST=archiv-staging-db-1
+          EOF
 
       - name: Validate observability compose config
-        # Dry-run: resolves all variable substitutions from /opt/familienarchiv/.env
-        # and reports any missing required keys before containers start. Catches
-        # truncated passwords (missing $$ escaping), undefined variables, and YAML
-        # errors in config files updated by the previous step.
+        # Dry-run: resolves all variable substitutions and reports any missing
+        # required keys before containers start. Catches undefined variables and
+        # YAML errors in config files updated by the previous step.
         run: |
           docker compose \
             -f /opt/familienarchiv/docker-compose.observability.yml \
+            --env-file /opt/familienarchiv/infra/observability/obs.env \
+            --env-file /opt/familienarchiv/obs-secrets.env \
             config --quiet
 
       - name: Start observability stack
-        # Runs from /opt/familienarchiv/ so bind mounts resolve to stable
-        # host paths that survive workspace wipes between nightly runs.
-        # Docker Compose reads /opt/familienarchiv/.env automatically.
+        # Runs with absolute paths so bind mounts resolve to stable host paths
+        # that survive workspace wipes between nightly runs (see ADR-016).
+        # Non-secret config from obs.env (git-tracked); secrets from obs-secrets.env
+        # (written fresh from Gitea secrets above).
         run: |
           docker compose \
             -f /opt/familienarchiv/docker-compose.observability.yml \
+            --env-file /opt/familienarchiv/infra/observability/obs.env \
+            --env-file /opt/familienarchiv/obs-secrets.env \
             up -d --wait --remove-orphans
 
       - name: Assert observability stack health

.gitea/workflows/release.yml

@@ -76,12 +76,6 @@ jobs:
           APP_MAIL_FROM=noreply@raddatz.cloud
           IMPORT_HOST_DIR=/srv/familienarchiv-production/import
           POSTGRES_USER=archiv
-          PORT_GRAFANA=3003
-          PORT_GLITCHTIP=3002
-          PORT_PROMETHEUS=9090
-          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
-          GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
-          GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud
           SENTRY_DSN=${{ secrets.SENTRY_DSN }}
           EOF
 
@@ -104,11 +98,29 @@ jobs:
             --env-file .env.production \
             up -d --wait --remove-orphans
 
+      - name: Deploy observability configs
+        # Mirrors the nightly approach: copies obs compose file and config tree
+        # to /opt/familienarchiv/ (permanent path, survives workspace wipes — ADR-016),
+        # then writes obs-secrets.env fresh from Gitea secrets.
+        # Non-secret config lives in infra/observability/obs.env (tracked in git).
+        run: |
+          mkdir -p /opt/familienarchiv/infra
+          cp -r infra/observability /opt/familienarchiv/infra/
+          cp docker-compose.observability.yml /opt/familienarchiv/
+          cat > /opt/familienarchiv/obs-secrets.env <<EOF
+          GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
+          POSTGRES_USER=archiv
+          POSTGRES_PASSWORD=${{ secrets.PROD_POSTGRES_PASSWORD }}
+          POSTGRES_HOST=archiv-production-db-1
+          EOF
+
       - name: Start observability stack
         run: |
           docker compose \
-            -f docker-compose.observability.yml \
-            --env-file .env.production \
+            -f /opt/familienarchiv/docker-compose.observability.yml \
+            --env-file /opt/familienarchiv/infra/observability/obs.env \
+            --env-file /opt/familienarchiv/obs-secrets.env \
             up -d --wait --remove-orphans
 
       - name: Reload Caddy

docker-compose.observability.yml

@@ -146,6 +146,7 @@ services:
     environment:
       GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-changeme}
       GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL:-http://localhost:3003}
     volumes:
       - grafana_data:/var/lib/grafana
       - ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro

docs/DEPLOYMENT.md

@@ -306,32 +306,54 @@ docker compose up -d                                    # creates archiv-net
 docker compose -f docker-compose.observability.yml up -d
 ```
 
+#### Why the obs stack is managed differently from the main app stack
+
+The main app stack (`docker-compose.prod.yml`) has no config-file bind mounts — its containers read config from env vars and image defaults. The workspace is wiped after each CI run but that does not affect running containers, because they hold no references to workspace paths.
+
+The obs stack is different: `prometheus.yml`, `tempo.yml`, Loki config, Grafana provisioning files, and Promtail config are all bind-mounted from the host filesystem into their containers. If those source paths disappear (workspace wipe), the containers can restart fine until a `docker compose up` is run again — at that point Docker tries to re-resolve the bind-mount source and fails because the workspace path no longer exists.
+
+The fix is to keep the obs compose file and config tree at a **permanent path** that CI copies to on every run but which survives between runs: `/opt/familienarchiv/` (see ADR-016).
+
 #### Production — managed from `/opt/familienarchiv/`
 
-The nightly CI job copies `docker-compose.observability.yml` and `infra/observability/` to `/opt/familienarchiv/` on every run, then starts the stack from there. Bind mounts in the compose file resolve to `/opt/familienarchiv/infra/observability/…` on the host, which survives workspace wipes between CI runs (see ADR-016).
+Every CI run (nightly + release) copies `docker-compose.observability.yml` and `infra/observability/` to `/opt/familienarchiv/` before starting the stack. Bind mounts then resolve to `/opt/familienarchiv/infra/observability/…` — a stable path that outlasts any workspace wipe.
 
-The obs stack reads secrets from `/opt/familienarchiv/.env` (Docker Compose auto-reads this file when launched from that directory). This file is managed by the operator — CI does **not** write or delete it.
+**Environment variables** follow the same two-source model as the main stack:
 
-**Required keys in `/opt/familienarchiv/.env`:**
+| Source | What it contains | Managed by |
+|---|---|---|
+| `infra/observability/obs.env` | All non-secret config (ports, URLs, hostnames) | Git — reviewed in PRs |
+| `/opt/familienarchiv/obs-secrets.env` | Passwords and secret keys only | CI — written fresh from Gitea secrets on every deploy |
 
-| Key | Example / notes |
+Both files are passed explicitly via `--env-file` to the compose command, so there is no implicit auto-read `.env` and no operator-managed file to keep in sync.
+
+**Non-secret config** (`infra/observability/obs.env`):
+
+| Key | Value | Notes |
+|---|---|---|
+| `PORT_GRAFANA` | `3003` | Avoids collision with staging frontend on port 3001 |
+| `PORT_GLITCHTIP` | `3002` | |
+| `PORT_PROMETHEUS` | `9090` | |
+| `GF_SERVER_ROOT_URL` | `https://grafana.archiv.raddatz.cloud` | Required for alert email links and OAuth redirects |
+| `GLITCHTIP_DOMAIN` | `https://glitchtip.archiv.raddatz.cloud` | Must match the Caddy vhost |
+| `POSTGRES_HOST` | `archive-db` | Override if only the staging stack is running |
+
+**Secret keys** (set in Gitea secrets, injected by CI into `obs-secrets.env`):
+
+| Gitea secret | Notes |
 |---|---|
-| `GRAFANA_ADMIN_PASSWORD` | Strong unique password |
-| `GLITCHTIP_SECRET_KEY` | `python3 -c "import secrets; print(secrets.token_hex(32))"` |
-| `GLITCHTIP_DOMAIN` | `https://glitchtip.archiv.raddatz.cloud` — must match the Caddy vhost |
-| `POSTGRES_USER` | Must match the `archiv` user set in `.env.staging` / `.env.production` |
-| `POSTGRES_PASSWORD` | Must match the running PostgreSQL container's password |
-| `PORT_GRAFANA` | `3003` (staging default; 3001 was used by staging frontend) |
-| `PORT_GLITCHTIP` | `3002` |
-| `PORT_PROMETHEUS` | `9090` |
-| `SENTRY_DSN` | Set after GlitchTip first-run; leave empty to disable |
+| `GRAFANA_ADMIN_PASSWORD` | Strong unique password; shared by nightly and release |
+| `GLITCHTIP_SECRET_KEY` | `openssl rand -hex 32`; shared by nightly and release |
+| `STAGING_POSTGRES_PASSWORD` / `PROD_POSTGRES_PASSWORD` | Must match the running PostgreSQL container |
 
-**`$$` escaping rule:** passwords that contain a literal `$` must use `$$` in this file so Docker Compose does not expand them as variable references. Example: a password `p@$word` must be written as `p@$$word`. Failure to escape produces a silently truncated password — Grafana or GlitchTip will start but reject logins.
-
-To start or restart the obs stack manually on the server:
+To start or restart the obs stack manually on the server (after CI has run at least once):
 
 ```bash
-docker compose -f /opt/familienarchiv/docker-compose.observability.yml up -d --wait --remove-orphans
+docker compose \
+  -f /opt/familienarchiv/docker-compose.observability.yml \
+  --env-file /opt/familienarchiv/infra/observability/obs.env \
+  --env-file /opt/familienarchiv/obs-secrets.env \
+  up -d --wait --remove-orphans
 ```
 
 Current services:

infra/observability/obs.env

@@ -0,0 +1,22 @@
+# Non-secret observability stack configuration — tracked in git.
+# Secret values (passwords, keys) are injected by CI from Gitea secrets
+# into /opt/familienarchiv/obs-secrets.env at deploy time.
+#
+# For local dev the main .env file supplies these values instead;
+# this file is only used in the CI/production path.
+
+# Host ports (all bound to 127.0.0.1 — Caddy is the external entry point)
+PORT_GRAFANA=3003
+PORT_GLITCHTIP=3002
+PORT_PROMETHEUS=9090
+
+# Public URLs — used for internal redirects, alert email links, OAuth callbacks
+GF_SERVER_ROOT_URL=https://grafana.archiv.raddatz.cloud
+GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud
+
+# PostgreSQL hostname for GlitchTip db-init and workers.
+# The actual value depends on the Compose project name — it is not a fixed string.
+# CI sets POSTGRES_HOST in obs-secrets.env per environment:
+#   staging:    archiv-staging-db-1    (project archiv-staging + service db)
+#   production: archiv-production-db-1 (project archiv-production + service db)
+# For local dev, set POSTGRES_HOST in your .env file (defaults to archive-db there).