From 1fc47888d50128f8d6020305c6fe3c997d2a36c0 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 15 May 2026 19:02:53 +0200 Subject: [PATCH] fix(ci): sync observability configs to host before docker compose up (#598) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DooD runner only shares /var/run/docker.sock — no workspace directory is mapped to the host daemon. Relative bind mounts in docker-compose.observability.yml resolved to paths that didn't exist on the host; Docker auto-created directories in their place, causing 'not a directory' mount failures for all five config files. Fix: - docker-compose.observability.yml: replace hardcoded ./infra/observability/ prefix with ${OBS_CONFIG_DIR:-./infra/observability} so the path is configurable while remaining backwards-compatible for local use. - nightly.yml / release.yml: add a 'Sync observability configs to host' step that finds the job container's overlay2 MergedDir (the container's full filesystem as seen from the host mount namespace), then uses the existing nsenter/alpine pattern to cp the config tree into a stable host path (/srv/familienarchiv-{staging,production}/obs-configs). OBS_CONFIG_DIR is injected into the env file so Compose picks it up. Co-Authored-By: Claude Sonnet 4.6 --- .gitea/workflows/nightly.yml | 19 +++++++++++++++++++ .gitea/workflows/release.yml | 11 +++++++++++ docker-compose.observability.yml | 10 +++++----- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/nightly.yml b/.gitea/workflows/nightly.yml index 81cf885c..af5e1750 100644 --- a/.gitea/workflows/nightly.yml +++ b/.gitea/workflows/nightly.yml @@ -85,6 +85,7 @@ jobs: GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }} GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud SENTRY_DSN=${{ secrets.SENTRY_DSN }} + OBS_CONFIG_DIR=/srv/familienarchiv-staging/obs-configs EOF - name: Verify backend /import:ro mount is wired @@ -131,6 +132,24 @@ jobs: --profile staging \ up -d --wait --remove-orphans + - name: Sync observability configs to host + # DooD: runner-config.yaml only shares /var/run/docker.sock with the host + # daemon — no workspace directory is mapped. Relative bind mounts in + # docker-compose.observability.yml would resolve to paths that don't + # exist on the host; Docker auto-creates directories in their place, + # causing "not a directory" mount failures at container startup. + # + # Fix: find the job container's overlay2 merged directory (visible in the + # host's mount namespace), then use nsenter to copy from there into a + # stable host path. The overlay path is the job container's full + # filesystem as seen from the host — no socket tricks needed. + run: | + OVERLAY=$(docker inspect "$(hostname)" --format '{{.GraphDriver.Data.MergedDir}}') + SRC="${OVERLAY}$(pwd)/infra/observability" + docker run --rm --privileged --pid=host \ + alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d \ + sh -c "nsenter -t 1 -m -- sh -c 'mkdir -p /srv/familienarchiv-staging/obs-configs && cp -r \"${SRC}/.\" /srv/familienarchiv-staging/obs-configs/'" + - name: Start observability stack run: | docker compose \ diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 2645dc15..4237c9cf 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -83,6 +83,7 @@ jobs: GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }} GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud SENTRY_DSN=${{ secrets.SENTRY_DSN }} + OBS_CONFIG_DIR=/srv/familienarchiv-production/obs-configs EOF - name: Build images @@ -104,6 +105,16 @@ jobs: --env-file .env.production \ up -d --wait --remove-orphans + - name: Sync observability configs to host + # DooD: same overlay2 trick as nightly.yml — see that file for the + # full rationale. Production path: /srv/familienarchiv-production/obs-configs. + run: | + OVERLAY=$(docker inspect "$(hostname)" --format '{{.GraphDriver.Data.MergedDir}}') + SRC="${OVERLAY}$(pwd)/infra/observability" + docker run --rm --privileged --pid=host \ + alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d \ + sh -c "nsenter -t 1 -m -- sh -c 'mkdir -p /srv/familienarchiv-production/obs-configs && cp -r \"${SRC}/.\" /srv/familienarchiv-production/obs-configs/'" + - name: Start observability stack run: | docker compose \ diff --git a/docker-compose.observability.yml b/docker-compose.observability.yml index b83cb439..2da53c3b 100644 --- a/docker-compose.observability.yml +++ b/docker-compose.observability.yml @@ -16,7 +16,7 @@ services: container_name: obs-prometheus restart: unless-stopped volumes: - - ./infra/observability/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ${OBS_CONFIG_DIR:-./infra/observability}/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus_data:/prometheus command: - '--config.file=/etc/prometheus/prometheus.yml' @@ -79,7 +79,7 @@ services: container_name: obs-loki restart: unless-stopped volumes: - - ./infra/observability/loki/loki-config.yml:/etc/loki/loki-config.yml:ro + - ${OBS_CONFIG_DIR:-./infra/observability}/loki/loki-config.yml:/etc/loki/loki-config.yml:ro - loki_data:/loki command: -config.file=/etc/loki/loki-config.yml expose: @@ -98,7 +98,7 @@ services: container_name: obs-promtail restart: unless-stopped volumes: - - ./infra/observability/promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro + - ${OBS_CONFIG_DIR:-./infra/observability}/promtail/promtail-config.yml:/etc/promtail/promtail-config.yml:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro # :ro restricts file-system access but NOT Docker API permissions — a compromised Promtail has full daemon access. Accepted risk on single-operator self-hosted archive. - /var/run/docker.sock:/var/run/docker.sock:ro @@ -118,7 +118,7 @@ services: container_name: obs-tempo restart: unless-stopped volumes: - - ./infra/observability/tempo/tempo.yml:/etc/tempo.yml:ro + - ${OBS_CONFIG_DIR:-./infra/observability}/tempo/tempo.yml:/etc/tempo.yml:ro - tempo_data:/var/tempo command: -config.file=/etc/tempo.yml expose: @@ -148,7 +148,7 @@ services: GF_USERS_ALLOW_SIGN_UP: "false" volumes: - grafana_data:/var/lib/grafana - - ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro + - ${OBS_CONFIG_DIR:-./infra/observability}/grafana/provisioning:/etc/grafana/provisioning:ro healthcheck: test: ["CMD-SHELL", "wget -qO- http://localhost:3000/api/health | grep -q ok || exit 1"] interval: 30s