devops(observability): add Tempo for distributed trace storage (OTLP receiver)

Closes #575

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

infra/observability/tempo/tempo.yml

@@ -0,0 +1,51 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    # 30 days — matches Loki retention. Compactor enforces this automatically;
+    # no manual intervention needed under normal trace volumes.
+    block_retention: 720h
+
+storage:
+  trace:
+    # Local filesystem storage — single-VPS deployment, no S3 backend needed.
+    # Both paths are on the same named Docker volume (tempo_data) so they
+    # survive container restarts without split-brain between WAL and blocks.
+    backend: local
+    local:
+      path: /var/tempo/blocks
+    wal:
+      path: /var/tempo/wal
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /var/tempo/generator/wal
+  processors:
+    - service-graphs
+    - span-metrics
+
+# Tempo HTTP API (port 3200) is unauthenticated. Access is controlled entirely
+# by network isolation: only Grafana (on obs-net) should reach this port.
+# The OTLP receivers (4317 gRPC, 4318 HTTP) are internal to archiv-net only.
+overrides:
+  defaults:
+    metrics_generator:
+      processors:
+        - service-graphs
+        - span-metrics