Compare commits
246 Commits
fdb9ae31ae
...
docs/impor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5efe3b8a7c | ||
|
|
0f1f9055c3 | ||
|
|
8cac63e938 | ||
|
|
97db718f81 | ||
|
|
06127724de | ||
|
|
7c017eca2a | ||
|
|
97ab9e38df | ||
|
|
f10b80a03f | ||
|
|
6478cc58ae | ||
|
|
a7c45b3a0e | ||
|
|
5ff0c25e10 | ||
|
|
7ba3a29592 | ||
|
|
d314fd9338 | ||
|
|
18d5a1e2da | ||
|
|
df00ea4238 | ||
|
|
ff1a7c07f1 | ||
|
|
366b484815 | ||
|
|
88c8063227 | ||
|
|
3066d3d3ff | ||
|
|
3e7ddea90a | ||
|
|
75b3ca8b9e | ||
|
|
74c4c390fc | ||
|
|
29087319e6 | ||
|
|
53457d9319 | ||
|
|
2d97595e9c | ||
|
|
a177077b40 | ||
|
|
b7a2332861 | ||
|
|
1da1a8d223 | ||
|
|
59715bdccd | ||
|
|
53a661adb6 | ||
|
|
4942c0ea07 | ||
|
|
7edc002ebb | ||
|
|
b43dd6cdd4 | ||
|
|
cff486dda7 | ||
|
|
df14e6b1ee | ||
|
|
1908dde859 | ||
|
|
4845e7a3c1 | ||
|
|
c6cceec6e9 | ||
|
|
8f6f4f2d62 | ||
|
|
6f7aa643c9 | ||
|
|
adfff420a5 | ||
|
|
8e9e3bba06 | ||
|
|
627fc44d99 | ||
|
|
6583226d79 | ||
|
|
41b205becc | ||
|
|
f22dcaecb7 | ||
|
|
1109ab917b | ||
|
|
769984608b | ||
|
|
c282f38170 | ||
|
|
3ea7f0b5b2 | ||
|
|
bcba4dab80 | ||
|
|
a4a3e3b105 | ||
|
|
cac00ed711 | ||
|
|
637829cebc | ||
|
|
4e636b3253 | ||
|
|
ab2708e63b | ||
|
|
ed8e9576e4 | ||
|
|
0958df7768 | ||
|
|
f4ffd8acee | ||
|
|
0801da8df0 | ||
|
|
e0e1578bdd | ||
|
|
2df71beb7e | ||
|
|
2dbb3c37b4 | ||
|
|
67368b4413 | ||
|
|
ddf6cf4cbc | ||
|
|
df952861c4 | ||
|
|
22a5ee816a | ||
|
|
0179e93a4b | ||
|
|
0fc0cbcffd | ||
|
|
549cb15845 | ||
|
|
74ddf16b01 | ||
|
|
ebaedb1af0 | ||
|
|
e75ac8ec45 | ||
|
|
525f091b3a | ||
|
|
d6abf990c7 | ||
|
|
77d59c5d83 | ||
|
|
6c2b9af10b | ||
|
|
2e3744d9ef | ||
|
|
131ed336bc | ||
|
|
3fa3460dbf | ||
|
|
79edb94558 | ||
|
|
52d8dc2b20 | ||
|
|
696b71da5a | ||
|
|
f3e3545d06 | ||
|
|
4bb6685edb | ||
|
|
18c93d4eaa | ||
|
|
eca4f1f0e8 | ||
|
|
4e33f52add | ||
|
|
890f014bb3 | ||
|
|
429ff32eda | ||
|
|
38a4ca2e34 | ||
|
|
b63a2040e3 | ||
|
|
0c4b22291f | ||
|
|
f1a61278f9 | ||
|
|
2914010b68 | ||
|
|
1a7e4ce536 | ||
|
|
3fa0f59529 | ||
|
|
36d50222ec | ||
|
|
d47326d01c | ||
|
|
0af43043ba | ||
|
|
51f7efe333 | ||
|
|
8f0fb89e22 | ||
|
|
9d812572c8 | ||
|
|
4ee36b2047 | ||
|
|
1253e89887 | ||
|
|
197a3e71d5 | ||
|
|
4f469db02e | ||
|
|
9886f2bcac | ||
|
|
006d02a137 | ||
|
|
c89441278f | ||
|
|
5301820a88 | ||
|
|
feb5275a94 | ||
|
|
4037564e65 | ||
|
|
0ef50d0ae1 | ||
|
|
9579391e27 | ||
|
|
720615bb1a | ||
|
|
6fbec80414 | ||
|
|
12416e7704 | ||
|
|
d56e6eadab | ||
|
|
510e406a5e | ||
|
|
711d170607 | ||
|
|
55617722f6 | ||
|
|
47afb9e181 | ||
|
|
db951d80cf | ||
|
|
a47027d67a | ||
|
|
1c94a43cb5 | ||
|
|
a1fc7b13d9 | ||
|
|
033d430688 | ||
|
|
640bdc12db | ||
|
|
93e58be141 | ||
|
|
96e8a07a8c | ||
|
|
f46ae2658f | ||
|
|
6125f50d6d | ||
|
|
197c948a35 | ||
|
|
4a4248e726 | ||
|
|
8210984fe3 | ||
|
|
e1e6d2d4b2 | ||
|
|
5ad5f82864 | ||
|
|
19e2f65a21 | ||
|
|
909f960b2e | ||
|
|
7b282f699d | ||
|
|
392097287c | ||
|
|
728f9cd1b0 | ||
|
|
35fbaf8154 | ||
|
|
978a2b3cdb | ||
|
|
30efb54aac | ||
|
|
dbf74cb91a | ||
|
|
261cbbd867 | ||
|
|
6f862243fd | ||
|
|
3d3c111c2b | ||
|
|
cdd5bfa318 | ||
|
|
85c13b3d46 | ||
|
|
9a460b3c90 | ||
|
|
cdc3e2e4c8 | ||
|
|
e89a90ff66 | ||
|
|
0c0a4830cd | ||
|
|
dd843d76c2 | ||
|
|
9601974db0 | ||
|
|
1782526c99 | ||
|
|
76ef54e064 | ||
|
|
f1d1ac3f1a | ||
|
|
0f48ffede5 | ||
|
|
3e72157ee1 | ||
|
|
e2d3975524 | ||
|
|
59e99f862a | ||
|
|
bb39ca59ec | ||
|
|
6b53cbfc5b | ||
|
|
e3e8373526 | ||
|
|
907a6a6b53 | ||
|
|
f27e2d33a5 | ||
|
|
6832300a4b | ||
|
|
9c5267e1f0 | ||
|
|
4979ae1867 | ||
|
|
29ef82f7b4 | ||
|
|
f458c11a0d | ||
|
|
e615ba1bbf | ||
|
|
1bec7dd17e | ||
|
|
a0339a5526 | ||
|
|
65cae4a5e8 | ||
|
|
c8cc0646cb | ||
|
|
e8057fe517 | ||
|
|
378023c53d | ||
|
|
ff3e863032 | ||
|
|
8fc32f18ce | ||
|
|
0cd9ea915e | ||
|
|
f0e7f73ec1 | ||
|
|
567f9267e8 | ||
|
|
1dc5bf4377 | ||
|
|
31d3ec8367 | ||
|
|
d739f58bb5 | ||
|
|
18e675a5b2 | ||
|
|
a3fc838855 | ||
|
|
d5043053e0 | ||
|
|
c932dd19d9 | ||
|
|
c532ad21bf | ||
|
|
0e95bd9160 | ||
|
|
e312cce4e1 | ||
|
|
5587722800 | ||
|
|
0451b6630c | ||
|
|
f77fb79cd2 | ||
|
|
1247b51d9e | ||
|
|
7342c60952 | ||
|
|
328bd2c3b4 | ||
|
|
db87a214fd | ||
|
|
ad95b09046 | ||
|
|
1e95ca979b | ||
|
|
1cae9ac311 | ||
|
|
72bd2e11b4 | ||
|
|
69b3c663c0 | ||
|
|
f470a39ad2 | ||
|
|
e2f287d3d8 | ||
|
|
914e438793 | ||
|
|
6266c5f721 | ||
|
|
f564c30ae2 | ||
|
|
a5ce46359a | ||
|
|
b45953e567 | ||
|
|
36d1b9c038 | ||
|
|
56bcbcdd5c | ||
|
|
9b9bfde843 | ||
|
|
164a917d95 | ||
|
|
96c0aa592c | ||
|
|
d8520d9714 | ||
|
|
873d668653 | ||
|
|
4e257a7ca4 | ||
|
|
d0bb6729cd | ||
|
|
32ede3e3ce | ||
|
|
5da78e5e30 | ||
|
|
cb108faaf8 | ||
|
|
611b82ccde | ||
|
|
64d8f9d904 | ||
|
|
6f452a9a8b | ||
|
|
20fe5637c1 | ||
|
|
9bf8cf831d | ||
|
|
9f4a1141ef | ||
|
|
cb818f4bfa | ||
|
|
9c195ff5cb | ||
|
|
54d32c9163 | ||
|
|
0b5ab73963 | ||
|
|
956387471d | ||
|
|
78fd9e026e | ||
|
|
4d6fb06e02 | ||
|
|
8944f8bb44 | ||
|
|
1b178767ab | ||
|
|
7d10653c41 | ||
|
|
b7a03614bc | ||
|
|
49c5324352 |
@@ -39,6 +39,12 @@ PORT_PROMETHEUS=9090
|
||||
# Grafana admin password — change this before exposing Grafana beyond localhost
|
||||
GRAFANA_ADMIN_PASSWORD=changeme
|
||||
|
||||
# Password for the read-only grafana_reader PostgreSQL role used by the PO
|
||||
# Overview dashboard. Consumed by Flyway V68 (to set the role's password) and
|
||||
# by Grafana's PostgreSQL datasource (to connect). REQUIRED in production —
|
||||
# generate with: openssl rand -hex 32
|
||||
GRAFANA_DB_PASSWORD=changeme-generate-with-openssl-rand-hex-32
|
||||
|
||||
# GlitchTip domain — production: use https://glitchtip.archiv.raddatz.cloud (must match Caddy vhost)
|
||||
GLITCHTIP_DOMAIN=http://localhost:3002
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
name: Unit & Component Tests
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: mcr.microsoft.com/playwright:v1.58.2-noble
|
||||
image: mcr.microsoft.com/playwright:v1.60.0-noble
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -29,6 +29,10 @@ jobs:
|
||||
run: npm ci
|
||||
working-directory: frontend
|
||||
|
||||
- name: Security audit (no dev deps)
|
||||
run: npm audit --audit-level=high --omit=dev
|
||||
working-directory: frontend
|
||||
|
||||
- name: Compile Paraglide i18n
|
||||
run: npx @inlang/paraglide-js compile --project ./project.inlang --outdir ./src/lib/paraglide
|
||||
working-directory: frontend
|
||||
|
||||
@@ -31,6 +31,7 @@ name: nightly
|
||||
# STAGING_APP_ADMIN_USERNAME
|
||||
# STAGING_APP_ADMIN_PASSWORD
|
||||
# GRAFANA_ADMIN_PASSWORD
|
||||
# GRAFANA_DB_PASSWORD (read-only grafana_reader DB role, issue #651)
|
||||
# GLITCHTIP_SECRET_KEY
|
||||
# SENTRY_DSN (set after GlitchTip first-run; empty = Sentry disabled)
|
||||
|
||||
@@ -79,6 +80,8 @@ jobs:
|
||||
IMPORT_HOST_DIR=/srv/familienarchiv-staging/import
|
||||
POSTGRES_USER=archiv
|
||||
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
|
||||
VITE_SENTRY_DSN=${{ secrets.VITE_SENTRY_DSN }}
|
||||
GRAFANA_DB_PASSWORD=${{ secrets.GRAFANA_DB_PASSWORD }}
|
||||
EOF
|
||||
|
||||
- name: Verify backend /import:ro mount is wired
|
||||
@@ -142,6 +145,7 @@ jobs:
|
||||
cp docker-compose.observability.yml /opt/familienarchiv/
|
||||
cat > /opt/familienarchiv/obs-secrets.env <<'EOF'
|
||||
GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
||||
GRAFANA_DB_PASSWORD=${{ secrets.GRAFANA_DB_PASSWORD }}
|
||||
GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
|
||||
POSTGRES_PASSWORD=${{ secrets.STAGING_POSTGRES_PASSWORD }}
|
||||
POSTGRES_HOST=archiv-staging-db-1
|
||||
@@ -252,20 +256,20 @@ jobs:
|
||||
URL="https://$HOST"
|
||||
HOST_IP=$(awk 'NR>1 && $2=="00000000"{h=$3;printf "%d.%d.%d.%d\n",strtonum("0x"substr(h,7,2)),strtonum("0x"substr(h,5,2)),strtonum("0x"substr(h,3,2)),strtonum("0x"substr(h,1,2));exit}' /proc/net/route)
|
||||
[ -n "$HOST_IP" ] || { echo "ERROR: could not detect Docker bridge gateway via /proc/net/route"; exit 1; }
|
||||
RESOLVE="--resolve $HOST:443:$HOST_IP"
|
||||
RESOLVE=(--resolve "$HOST:443:$HOST_IP")
|
||||
echo "Smoke test: $URL (pinned to $HOST_IP via bridge gateway)"
|
||||
curl -fsS "$RESOLVE" --max-time 10 "$URL/login" -o /dev/null
|
||||
curl -fsS "${RESOLVE[@]}" --max-time 10 "$URL/login" -o /dev/null
|
||||
# Pin the preload-list-eligible HSTS value, not just header presence:
|
||||
# a degraded `max-age=1` or a dropped `includeSubDomains; preload` must
|
||||
# fail this check rather than pass it silently.
|
||||
curl -fsS "$RESOLVE" --max-time 10 -I "$URL/" \
|
||||
curl -fsS "${RESOLVE[@]}" --max-time 10 -I "$URL/" \
|
||||
| grep -Eqi 'strict-transport-security:[[:space:]]*max-age=31536000.*includeSubDomains.*preload'
|
||||
# Permissions-Policy denies APIs the app does not use (camera,
|
||||
# microphone, geolocation). A regression that loosens or drops the
|
||||
# header now fails the smoke step.
|
||||
curl -fsS "$RESOLVE" --max-time 10 -I "$URL/" \
|
||||
curl -fsS "${RESOLVE[@]}" --max-time 10 -I "$URL/" \
|
||||
| grep -Eqi 'permissions-policy:[[:space:]]*camera=\(\),[[:space:]]*microphone=\(\),[[:space:]]*geolocation=\(\)'
|
||||
status=$(curl -s "$RESOLVE" -o /dev/null -w "%{http_code}" --max-time 10 "$URL/actuator/health")
|
||||
status=$(curl -s "${RESOLVE[@]}" -o /dev/null -w "%{http_code}" --max-time 10 "$URL/actuator/health")
|
||||
[ "$status" = "404" ] || { echo "expected 404 from /actuator/health, got $status"; exit 1; }
|
||||
echo "All smoke checks passed"
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ name: release
|
||||
# MAIL_USERNAME
|
||||
# MAIL_PASSWORD
|
||||
# GRAFANA_ADMIN_PASSWORD
|
||||
# GRAFANA_DB_PASSWORD (read-only grafana_reader DB role, issue #651)
|
||||
# GLITCHTIP_SECRET_KEY
|
||||
# SENTRY_DSN (set after GlitchTip first-run; empty = Sentry disabled)
|
||||
|
||||
@@ -77,6 +78,7 @@ jobs:
|
||||
IMPORT_HOST_DIR=/srv/familienarchiv-production/import
|
||||
POSTGRES_USER=archiv
|
||||
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
|
||||
GRAFANA_DB_PASSWORD=${{ secrets.GRAFANA_DB_PASSWORD }}
|
||||
EOF
|
||||
|
||||
- name: Build images
|
||||
@@ -110,6 +112,7 @@ jobs:
|
||||
cp docker-compose.observability.yml /opt/familienarchiv/
|
||||
cat > /opt/familienarchiv/obs-secrets.env <<'EOF'
|
||||
GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
||||
GRAFANA_DB_PASSWORD=${{ secrets.GRAFANA_DB_PASSWORD }}
|
||||
GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
|
||||
POSTGRES_PASSWORD=${{ secrets.PROD_POSTGRES_PASSWORD }}
|
||||
POSTGRES_HOST=archiv-production-db-1
|
||||
@@ -181,28 +184,31 @@ jobs:
|
||||
|
||||
- name: Smoke test deployed environment
|
||||
# See nightly.yml — same three checks, against the prod vhost.
|
||||
# --resolve pins to the bridge gateway IP (the host), not 127.0.0.1
|
||||
# — see nightly.yml for the full network topology explanation.
|
||||
# --resolve stored as a Bash array so "${RESOLVE[@]}" expands to two
|
||||
# separate arguments; a quoted string would pass the flag and its value
|
||||
# as one token and curl would reject it as an unknown option.
|
||||
# Gateway detection via /proc/net/route — no iproute2 dependency.
|
||||
# See nightly.yml for the full network topology explanation.
|
||||
run: |
|
||||
set -e
|
||||
HOST="archiv.raddatz.cloud"
|
||||
URL="https://$HOST"
|
||||
HOST_IP=$(ip route show default | awk '/default/ {print $3}')
|
||||
[ -n "$HOST_IP" ] || { echo "ERROR: could not detect Docker bridge gateway via 'ip route'"; exit 1; }
|
||||
RESOLVE="--resolve $HOST:443:$HOST_IP"
|
||||
HOST_IP=$(awk 'NR>1 && $2=="00000000"{h=$3;printf "%d.%d.%d.%d\n",strtonum("0x"substr(h,7,2)),strtonum("0x"substr(h,5,2)),strtonum("0x"substr(h,3,2)),strtonum("0x"substr(h,1,2));exit}' /proc/net/route)
|
||||
[ -n "$HOST_IP" ] || { echo "ERROR: could not detect Docker bridge gateway via /proc/net/route"; exit 1; }
|
||||
RESOLVE=(--resolve "$HOST:443:$HOST_IP")
|
||||
echo "Smoke test: $URL (pinned to $HOST_IP via bridge gateway)"
|
||||
curl -fsS "$RESOLVE" --max-time 10 "$URL/login" -o /dev/null
|
||||
curl -fsS "${RESOLVE[@]}" --max-time 10 "$URL/login" -o /dev/null
|
||||
# Pin the preload-list-eligible HSTS value, not just header presence:
|
||||
# a degraded `max-age=1` or a dropped `includeSubDomains; preload` must
|
||||
# fail this check rather than pass it silently.
|
||||
curl -fsS "$RESOLVE" --max-time 10 -I "$URL/" \
|
||||
curl -fsS "${RESOLVE[@]}" --max-time 10 -I "$URL/" \
|
||||
| grep -Eqi 'strict-transport-security:[[:space:]]*max-age=31536000.*includeSubDomains.*preload'
|
||||
# Permissions-Policy denies APIs the app does not use (camera,
|
||||
# microphone, geolocation). A regression that loosens or drops the
|
||||
# header now fails the smoke step.
|
||||
curl -fsS "$RESOLVE" --max-time 10 -I "$URL/" \
|
||||
curl -fsS "${RESOLVE[@]}" --max-time 10 -I "$URL/" \
|
||||
| grep -Eqi 'permissions-policy:[[:space:]]*camera=\(\),[[:space:]]*microphone=\(\),[[:space:]]*geolocation=\(\)'
|
||||
status=$(curl -s "$RESOLVE" -o /dev/null -w "%{http_code}" --max-time 10 "$URL/actuator/health")
|
||||
status=$(curl -s "${RESOLVE[@]}" -o /dev/null -w "%{http_code}" --max-time 10 "$URL/actuator/health")
|
||||
[ "$status" = "404" ] || { echo "expected 404 from /actuator/health, got $status"; exit 1; }
|
||||
echo "All smoke checks passed"
|
||||
|
||||
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -26,3 +26,7 @@ node_modules/
|
||||
|
||||
# Repo uses npm; yarn.lock is ignored to avoid double-lockfile drift.
|
||||
frontend/yarn.lock
|
||||
|
||||
**/.venv/
|
||||
**/__pycache__/
|
||||
*.pyc
|
||||
|
||||
@@ -77,7 +77,7 @@ npm run generate:api # Regenerate TypeScript API types from OpenAPI spec
|
||||
```
|
||||
backend/src/main/java/org/raddatz/familienarchiv/
|
||||
├── audit/ Audit logging
|
||||
├── auth/ AuthService, AuthSessionController, LoginRequest (Spring Session JDBC)
|
||||
├── auth/ AuthService, AuthSessionController, LoginRequest, LoginRateLimiter, RateLimitProperties (Spring Session JDBC)
|
||||
├── config/ Infrastructure config (Minio, Async, Web)
|
||||
├── dashboard/ Dashboard analytics + StatsController/StatsService
|
||||
├── document/ Document domain (entities, controller, service, repository, DTOs)
|
||||
@@ -160,7 +160,7 @@ Input DTOs live flat in the domain package. Response types are the model entitie
|
||||
|
||||
→ See [CONTRIBUTING.md §Error handling](./CONTRIBUTING.md#error-handling)
|
||||
|
||||
**LLM reminder:** use `DomainException.notFound/forbidden/conflict/internal()` from service methods — never throw raw exceptions. When adding a new `ErrorCode`: (1) add to `ErrorCode.java`, (2) add to `ErrorCode` type in `frontend/src/lib/shared/errors.ts`, (3) add a `case` in `getErrorMessage()`, (4) add i18n keys in `messages/{de,en,es}.json`.
|
||||
**LLM reminder:** use `DomainException.notFound/forbidden/conflict/internal()` from service methods — never throw raw exceptions. When adding a new `ErrorCode`: (1) add to `ErrorCode.java`, (2) add to `ErrorCode` type in `frontend/src/lib/shared/errors.ts`, (3) add a `case` in `getErrorMessage()`, (4) add i18n keys in `messages/{de,en,es}.json`. Valid error codes include: `TOO_MANY_LOGIN_ATTEMPTS` (returned by `LoginRateLimiter` as HTTP 429 when a brute-force threshold is exceeded).
|
||||
|
||||
### Security / Permissions
|
||||
|
||||
@@ -267,7 +267,7 @@ Back button pattern — use the shared `<BackButton>` component from `$lib/share
|
||||
|
||||
→ See [CONTRIBUTING.md §Error handling](./CONTRIBUTING.md#error-handling)
|
||||
|
||||
**LLM reminder:** when adding a new `ErrorCode`: (1) add to `ErrorCode.java`, (2) add to `ErrorCode` type in `frontend/src/lib/shared/errors.ts`, (3) add a `case` in `getErrorMessage()`, (4) add i18n keys in `messages/{de,en,es}.json`.
|
||||
**LLM reminder:** when adding a new `ErrorCode`: (1) add to `ErrorCode.java`, (2) add to `ErrorCode` type in `frontend/src/lib/shared/errors.ts`, (3) add a `case` in `getErrorMessage()`, (4) add i18n keys in `messages/{de,en,es}.json`. Valid error codes include: `TOO_MANY_LOGIN_ATTEMPTS` (returned by `LoginRateLimiter` as HTTP 429 when a brute-force threshold is exceeded).
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -263,7 +263,7 @@ if (!result.response.ok) {
|
||||
return { person: result.data! }; // non-null assertion is safe after the ok check
|
||||
```
|
||||
|
||||
For multipart/form-data (file uploads): bypass the typed client and use raw `fetch` — the client cannot handle it.
|
||||
For multipart/form-data (file uploads): bypass the typed client and use `event.fetch` directly — never global `fetch`. The typed client cannot handle multipart bodies, but `event.fetch` is still required so that `handleFetch` injects the session cookie.
|
||||
|
||||
### Date handling
|
||||
|
||||
|
||||
@@ -97,7 +97,10 @@ public class MyEntity {
|
||||
|
||||
- Annotated with `@Service`, `@RequiredArgsConstructor`, optionally `@Slf4j`.
|
||||
- Write methods: `@Transactional`.
|
||||
- Read methods: no annotation (default non-transactional).
|
||||
- Read methods: no annotation (default non-transactional) — **except** when the method returns
|
||||
an entity whose lazy associations must remain accessible to the caller after the method
|
||||
returns. In that case, use `@Transactional(readOnly = true)` to keep the Hibernate session
|
||||
open. Removing this annotation causes `LazyInitializationException` in production. See ADR-022.
|
||||
- Cross-domain access goes through the other domain's service, never its repository.
|
||||
|
||||
## Error Handling
|
||||
|
||||
@@ -7,12 +7,10 @@ import org.raddatz.familienarchiv.audit.AuditService;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.user.AppUser;
|
||||
import org.raddatz.familienarchiv.user.UserService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.security.authentication.AuthenticationManager;
|
||||
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.security.core.AuthenticationException;
|
||||
import org.springframework.session.jdbc.JdbcIndexedSessionRepository;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Map;
|
||||
@@ -26,28 +24,17 @@ public class AuthService {
|
||||
private final AuthenticationManager authenticationManager;
|
||||
private final UserService userService;
|
||||
private final AuditService auditService;
|
||||
private final LoginRateLimiter loginRateLimiter;
|
||||
private final SessionRevocationPort sessionRevocationPort;
|
||||
|
||||
@Autowired(required = false)
|
||||
private JdbcIndexedSessionRepository sessionRepository;
|
||||
|
||||
@Autowired(required = false)
|
||||
private LoginRateLimiter loginRateLimiter;
|
||||
|
||||
/**
|
||||
* Validates credentials and returns the authenticated user plus the Spring Security
|
||||
* Authentication object. The caller is responsible for persisting the Authentication
|
||||
* to the session via SecurityContextRepository.
|
||||
*/
|
||||
public LoginResult login(String email, String password, String ip, String ua) {
|
||||
if (loginRateLimiter != null) {
|
||||
try {
|
||||
loginRateLimiter.checkAndConsume(ip, email);
|
||||
} catch (DomainException ex) {
|
||||
auditService.log(AuditKind.LOGIN_RATE_LIMITED, null, null, Map.of(
|
||||
"ip", ip,
|
||||
"email", email));
|
||||
throw ex;
|
||||
}
|
||||
try {
|
||||
loginRateLimiter.checkAndConsume(ip, email);
|
||||
} catch (DomainException ex) {
|
||||
auditService.log(AuditKind.LOGIN_RATE_LIMITED, null, null, Map.of(
|
||||
"ip", ip,
|
||||
"email", email));
|
||||
throw ex;
|
||||
}
|
||||
try {
|
||||
Authentication auth = authenticationManager.authenticate(
|
||||
@@ -58,9 +45,7 @@ public class AuthService {
|
||||
"userId", user.getId().toString(),
|
||||
"ip", ip,
|
||||
"ua", truncateUa(ua)));
|
||||
if (loginRateLimiter != null) {
|
||||
loginRateLimiter.invalidateOnSuccess(ip, email);
|
||||
}
|
||||
loginRateLimiter.invalidateOnSuccess(ip, email);
|
||||
return new LoginResult(user, auth);
|
||||
} catch (AuthenticationException ex) {
|
||||
// Audit login failure — intentionally does NOT log the attempted password.
|
||||
@@ -75,20 +60,11 @@ public class AuthService {
|
||||
}
|
||||
|
||||
public int revokeOtherSessions(String currentSessionId, String principalName) {
|
||||
int count = 0;
|
||||
for (String id : sessionRepository.findByPrincipalName(principalName).keySet()) {
|
||||
if (!id.equals(currentSessionId)) {
|
||||
sessionRepository.deleteById(id);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
return sessionRevocationPort.revokeOtherSessions(currentSessionId, principalName);
|
||||
}
|
||||
|
||||
public int revokeAllSessions(String principalName) {
|
||||
var sessions = sessionRepository.findByPrincipalName(principalName);
|
||||
sessions.keySet().forEach(sessionRepository::deleteById);
|
||||
return sessions.size();
|
||||
return sessionRevocationPort.revokeAllSessions(principalName);
|
||||
}
|
||||
|
||||
public void logout(String email, String ip, String ua) {
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package org.raddatz.familienarchiv.auth;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.session.jdbc.JdbcIndexedSessionRepository;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
class JdbcSessionRevocationAdapter implements SessionRevocationPort {
|
||||
|
||||
private final JdbcIndexedSessionRepository sessionRepository;
|
||||
|
||||
@Override
|
||||
public int revokeOtherSessions(String currentSessionId, String principalName) {
|
||||
int count = 0;
|
||||
for (String id : sessionRepository.findByPrincipalName(principalName).keySet()) {
|
||||
if (!id.equals(currentSessionId)) {
|
||||
sessionRepository.deleteById(id);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int revokeAllSessions(String principalName) {
|
||||
var sessions = sessionRepository.findByPrincipalName(principalName);
|
||||
sessions.keySet().forEach(sessionRepository::deleteById);
|
||||
return sessions.size();
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
@Service
|
||||
@@ -36,17 +37,27 @@ public class LoginRateLimiter {
|
||||
.build(key -> newBucket(maxPerIp, windowMinutes));
|
||||
}
|
||||
|
||||
// NOTE: This cache is node-local (in-memory). In a multi-replica deployment,
|
||||
// effective limits would be multiplied by replica count.
|
||||
// For the current single-VPS setup this is the correct, simplest implementation.
|
||||
|
||||
public void checkAndConsume(String ip, String email) {
|
||||
boolean ipEmailOk = byIpEmail.get(ip + ":" + email).tryConsume(1);
|
||||
boolean ipOk = byIp.get(ip).tryConsume(1);
|
||||
if (!ipEmailOk || !ipOk) {
|
||||
long retryAfterSeconds = windowMinutes * 60L;
|
||||
String key = ip + ":" + email.toLowerCase(Locale.ROOT);
|
||||
if (!byIpEmail.get(key).tryConsume(1)) {
|
||||
throw DomainException.tooManyRequests(ErrorCode.TOO_MANY_LOGIN_ATTEMPTS,
|
||||
"Too many login attempts from " + ip);
|
||||
"Too many login attempts from " + ip, retryAfterSeconds);
|
||||
}
|
||||
if (!byIp.get(ip).tryConsume(1)) {
|
||||
// Refund the ipEmail token so IP-level blocking does not erode the per-email quota.
|
||||
byIpEmail.get(key).addTokens(1);
|
||||
throw DomainException.tooManyRequests(ErrorCode.TOO_MANY_LOGIN_ATTEMPTS,
|
||||
"Too many login attempts from " + ip, retryAfterSeconds);
|
||||
}
|
||||
}
|
||||
|
||||
public void invalidateOnSuccess(String ip, String email) {
|
||||
byIpEmail.invalidate(ip + ":" + email);
|
||||
byIpEmail.invalidate(ip + ":" + email.toLowerCase(Locale.ROOT));
|
||||
byIp.invalidate(ip);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package org.raddatz.familienarchiv.auth;
|
||||
|
||||
class NoOpSessionRevocationAdapter implements SessionRevocationPort {
|
||||
|
||||
@Override
|
||||
public int revokeOtherSessions(String currentSessionId, String principalName) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int revokeAllSessions(String principalName) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package org.raddatz.familienarchiv.auth;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.session.jdbc.JdbcIndexedSessionRepository;
|
||||
|
||||
@Configuration
|
||||
class SessionRevocationConfig {
|
||||
|
||||
@Bean
|
||||
SessionRevocationPort sessionRevocationPort(
|
||||
@Autowired(required = false) JdbcIndexedSessionRepository sessionRepository) {
|
||||
if (sessionRepository != null) {
|
||||
return new JdbcSessionRevocationAdapter(sessionRepository);
|
||||
}
|
||||
return new NoOpSessionRevocationAdapter();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package org.raddatz.familienarchiv.auth;
|
||||
|
||||
public interface SessionRevocationPort {
|
||||
int revokeOtherSessions(String currentSessionId, String principalName);
|
||||
int revokeAllSessions(String principalName);
|
||||
}
|
||||
@@ -5,8 +5,10 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.core.env.Environment;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
import java.util.Map;
|
||||
|
||||
@Configuration
|
||||
@RequiredArgsConstructor
|
||||
@@ -14,6 +16,7 @@ import javax.sql.DataSource;
|
||||
public class FlywayConfig {
|
||||
|
||||
private final DataSource dataSource;
|
||||
private final Environment environment;
|
||||
|
||||
@Bean(name = "flyway")
|
||||
public Flyway flyway() {
|
||||
@@ -21,6 +24,7 @@ public class FlywayConfig {
|
||||
Flyway flyway = Flyway.configure()
|
||||
.dataSource(dataSource)
|
||||
.locations("classpath:db/migration")
|
||||
.placeholders(Map.of("grafanaDbPassword", resolveGrafanaDbPassword()))
|
||||
.baselineOnMigrate(true)
|
||||
.baselineVersion("4")
|
||||
.load();
|
||||
@@ -28,4 +32,22 @@ public class FlywayConfig {
|
||||
log.info("Flyway: {} migration(s) applied.", result.migrationsExecuted);
|
||||
return flyway;
|
||||
}
|
||||
|
||||
// Fail-closed: refuse to boot when GRAFANA_DB_PASSWORD is unset. The
|
||||
// grafana_reader role's password is (re)set on every boot by
|
||||
// R__grafana_reader_password.sql, so a missing env var means we'd either
|
||||
// skip the rotation silently or — with a hardcoded fallback — publish a
|
||||
// well-known credential for a role with SELECT on audit_log, documents,
|
||||
// and transcription_blocks. Same shape as UserDataInitializer's refusal
|
||||
// to seed default admin credentials outside dev/test/e2e.
|
||||
String resolveGrafanaDbPassword() {
|
||||
String value = environment.getProperty("GRAFANA_DB_PASSWORD");
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalStateException(
|
||||
"GRAFANA_DB_PASSWORD is required: it is consumed by "
|
||||
+ "R__grafana_reader_password.sql to (re)set the grafana_reader "
|
||||
+ "role's password on every boot. Generate with: openssl rand -hex 32");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ public class RateLimitInterceptor implements HandlerInterceptor {
|
||||
AtomicInteger count = requestCounts.get(ip, k -> new AtomicInteger(0));
|
||||
if (count.incrementAndGet() > MAX_REQUESTS_PER_MINUTE) {
|
||||
response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value());
|
||||
response.setHeader("Retry-After", "60");
|
||||
response.getWriter().write("{\"code\":\"RATE_LIMIT_EXCEEDED\",\"message\":\"Too many requests\"}");
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package org.raddatz.familienarchiv.document;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import lombok.*;
|
||||
import org.hibernate.annotations.BatchSize;
|
||||
import org.hibernate.annotations.CreationTimestamp;
|
||||
import org.hibernate.annotations.UpdateTimestamp;
|
||||
|
||||
@@ -21,6 +22,17 @@ import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
@NamedEntityGraph(name = "Document.full", attributeNodes = {
|
||||
@NamedAttributeNode("sender"),
|
||||
@NamedAttributeNode("receivers"),
|
||||
@NamedAttributeNode("tags"),
|
||||
@NamedAttributeNode("trainingLabels")
|
||||
})
|
||||
@NamedEntityGraph(name = "Document.list", attributeNodes = {
|
||||
@NamedAttributeNode("sender"),
|
||||
@NamedAttributeNode("receivers"),
|
||||
@NamedAttributeNode("tags")
|
||||
})
|
||||
@Entity
|
||||
@Table(name = "documents")
|
||||
@Data // Lombok: Generiert Getter, Setter, ToString, etc.
|
||||
@@ -118,24 +130,27 @@ public class Document {
|
||||
@Builder.Default
|
||||
private ScriptType scriptType = ScriptType.UNKNOWN;
|
||||
|
||||
@ManyToMany(fetch = FetchType.EAGER)
|
||||
@ManyToMany(fetch = FetchType.LAZY)
|
||||
@JoinTable(name = "document_receivers", joinColumns = @JoinColumn(name = "document_id"), inverseJoinColumns = @JoinColumn(name = "person_id"))
|
||||
@BatchSize(size = 50)
|
||||
@Builder.Default
|
||||
private Set<Person> receivers = new HashSet<>();
|
||||
|
||||
@ManyToOne
|
||||
@ManyToOne(fetch = FetchType.LAZY)
|
||||
@JoinColumn(name = "sender_id")
|
||||
private Person sender;
|
||||
|
||||
@ManyToMany(fetch = FetchType.EAGER)
|
||||
@ManyToMany(fetch = FetchType.LAZY)
|
||||
@JoinTable(name = "document_tags", joinColumns = @JoinColumn(name = "document_id"), inverseJoinColumns = @JoinColumn(name = "tag_id"))
|
||||
@BatchSize(size = 50)
|
||||
@Builder.Default
|
||||
private Set<Tag> tags = new HashSet<>();
|
||||
|
||||
@ElementCollection(fetch = FetchType.EAGER)
|
||||
@ElementCollection(fetch = FetchType.LAZY)
|
||||
@CollectionTable(name = "document_training_labels", joinColumns = @JoinColumn(name = "document_id"))
|
||||
@Column(name = "label")
|
||||
@Enumerated(EnumType.STRING)
|
||||
@BatchSize(size = 50)
|
||||
@Builder.Default
|
||||
private Set<TrainingLabel> trainingLabels = new HashSet<>();
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package org.raddatz.familienarchiv.document;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import org.raddatz.familienarchiv.audit.ActivityActorDTO;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
public record DocumentListItem(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
UUID id,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
String title,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
String originalFilename,
|
||||
String thumbnailUrl,
|
||||
LocalDate documentDate,
|
||||
Person sender,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
List<Person> receivers,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
List<Tag> tags,
|
||||
String archiveBox,
|
||||
String archiveFolder,
|
||||
String location,
|
||||
String summary,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int completionPercentage,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
List<ActivityActorDTO> contributors,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
SearchMatchData matchData
|
||||
) {}
|
||||
@@ -7,6 +7,8 @@ import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.Pageable;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.data.jpa.domain.Specification;
|
||||
import org.springframework.data.jpa.repository.EntityGraph;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
@@ -23,6 +25,18 @@ import java.util.UUID;
|
||||
@Repository
|
||||
public interface DocumentRepository extends JpaRepository<Document, UUID>, JpaSpecificationExecutor<Document> {
|
||||
|
||||
@EntityGraph("Document.full")
|
||||
Optional<Document> findById(UUID id);
|
||||
|
||||
@EntityGraph("Document.list")
|
||||
Page<Document> findAll(Specification<Document> spec, Pageable pageable);
|
||||
|
||||
@EntityGraph("Document.list")
|
||||
List<Document> findAll(Specification<Document> spec);
|
||||
|
||||
@EntityGraph("Document.list")
|
||||
Page<Document> findAll(Pageable pageable);
|
||||
|
||||
// Findet ein Dokument anhand des ursprünglichen Dateinamens
|
||||
// Wichtig für den Abgleich beim Excel-Import & Datei-Upload
|
||||
Optional<Document> findByOriginalFilename(String originalFilename);
|
||||
@@ -30,17 +44,21 @@ public interface DocumentRepository extends JpaRepository<Document, UUID>, JpaSp
|
||||
// Wie oben, gibt aber nur das erste Ergebnis zurück — sicher wenn doppelte Dateinamen existieren
|
||||
Optional<Document> findFirstByOriginalFilename(String originalFilename);
|
||||
|
||||
// Findet alle Dokumente mit einem bestimmten Status
|
||||
// z.B. um alle offenen "PLACEHOLDER" zu finden
|
||||
// Callers access only status/id scalar fields — no graph needed.
|
||||
List<Document> findByStatus(DocumentStatus status);
|
||||
|
||||
// Prüft effizient, ob ein Dateiname schon existiert (gibt true/false zurück)
|
||||
boolean existsByOriginalFilename(String originalFilename);
|
||||
|
||||
// lazy – @BatchSize(50) fallback active; see ADR-022
|
||||
@EntityGraph("Document.full")
|
||||
List<Document> findBySenderId(UUID senderId);
|
||||
|
||||
// lazy – @BatchSize(50) fallback active; see ADR-022
|
||||
@EntityGraph("Document.full")
|
||||
List<Document> findByReceiversId(UUID receiverId);
|
||||
|
||||
// Callers access only doc.getTags() to mutate the set — receivers/sender not touched; no graph needed.
|
||||
List<Document> findByTags_Id(UUID tagId);
|
||||
|
||||
@Query("SELECT d FROM Document d WHERE d.id NOT IN (SELECT DISTINCT dv.documentId FROM DocumentVersion dv)")
|
||||
@@ -55,12 +73,15 @@ public interface DocumentRepository extends JpaRepository<Document, UUID>, JpaSp
|
||||
|
||||
long countByMetadataCompleteFalse();
|
||||
|
||||
// No production callers — only used if a future export path iterates the full list; no graph needed.
|
||||
List<Document> findByMetadataCompleteFalse(Sort sort);
|
||||
|
||||
// Callers map to IncompleteDocumentDTO using only scalar fields (id, title, createdAt) — no graph needed.
|
||||
Page<Document> findByMetadataCompleteFalse(Pageable pageable);
|
||||
|
||||
Optional<Document> findFirstByMetadataCompleteFalseAndIdNot(UUID id, Sort sort);
|
||||
|
||||
@EntityGraph("Document.full")
|
||||
@Query("SELECT DISTINCT d FROM Document d " +
|
||||
"JOIN d.receivers r " +
|
||||
"WHERE " +
|
||||
@@ -75,6 +96,7 @@ public interface DocumentRepository extends JpaRepository<Document, UUID>, JpaSp
|
||||
@Param("to") LocalDate to,
|
||||
Sort sort);
|
||||
|
||||
@EntityGraph("Document.full")
|
||||
@Query("SELECT DISTINCT d FROM Document d " +
|
||||
"LEFT JOIN d.receivers r " +
|
||||
"WHERE (d.sender.id = :personId OR r.id = :personId) " +
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
package org.raddatz.familienarchiv.document;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import org.raddatz.familienarchiv.audit.ActivityActorDTO;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record DocumentSearchItem(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
Document document,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
SearchMatchData matchData,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int completionPercentage,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
List<ActivityActorDTO> contributors
|
||||
) {}
|
||||
@@ -7,7 +7,7 @@ import java.util.List;
|
||||
|
||||
public record DocumentSearchResult(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
List<DocumentSearchItem> items,
|
||||
List<DocumentListItem> items,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
long totalElements,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
@@ -21,16 +21,16 @@ public record DocumentSearchResult(
|
||||
* Single-page convenience factory used by empty-result shortcuts and by tests that
|
||||
* don't care about paging. Treats the whole list as page 0 of itself.
|
||||
*/
|
||||
public static DocumentSearchResult of(List<DocumentSearchItem> items) {
|
||||
public static DocumentSearchResult of(List<DocumentListItem> items) {
|
||||
int size = items.size();
|
||||
return new DocumentSearchResult(items, size, 0, size, size == 0 ? 0 : 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Paged factory used by the service when it has a real Pageable + full match count
|
||||
* (e.g. from Spring's Page<T> or from an in-memory sort-then-slice).
|
||||
* (e.g. from Spring's Page<T> or from an in-memory sort-then-slice).
|
||||
*/
|
||||
public static DocumentSearchResult paged(List<DocumentSearchItem> slice, Pageable pageable, long totalElements) {
|
||||
public static DocumentSearchResult paged(List<DocumentListItem> slice, Pageable pageable, long totalElements) {
|
||||
int pageSize = pageable.getPageSize();
|
||||
int totalPages = pageSize == 0 ? 0 : (int) ((totalElements + pageSize - 1) / pageSize);
|
||||
return new DocumentSearchResult(slice, totalElements, pageable.getPageNumber(), pageSize, totalPages);
|
||||
|
||||
@@ -10,7 +10,6 @@ import org.raddatz.familienarchiv.audit.AuditService;
|
||||
import org.raddatz.familienarchiv.document.DocumentBatchMetadataDTO;
|
||||
import org.raddatz.familienarchiv.document.DocumentBatchSummary;
|
||||
import org.raddatz.familienarchiv.document.DocumentBulkEditDTO;
|
||||
import org.raddatz.familienarchiv.document.DocumentSearchItem;
|
||||
import org.raddatz.familienarchiv.document.DocumentSearchResult;
|
||||
import org.raddatz.familienarchiv.document.DocumentSort;
|
||||
import org.raddatz.familienarchiv.document.DocumentUpdateDTO;
|
||||
@@ -447,6 +446,7 @@ public class DocumentService {
|
||||
return saved;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Document updateDocumentTags(UUID docId, List<String> tagNames) {
|
||||
Document doc = documentRepository.findById(docId)
|
||||
.orElseThrow(() -> DomainException.notFound(ErrorCode.DOCUMENT_NOT_FOUND, "Document not found: " + docId));
|
||||
@@ -635,7 +635,7 @@ public class DocumentService {
|
||||
return saved;
|
||||
}
|
||||
|
||||
// 0. Zuletzt aktive Dokumente (sortiert nach updatedAt DESC)
|
||||
@Transactional(readOnly = true)
|
||||
public List<Document> getRecentActivity(int size) {
|
||||
return documentRepository.findAll(
|
||||
PageRequest.of(0, size, Sort.by(Sort.Direction.DESC, "updatedAt"))
|
||||
@@ -735,7 +735,7 @@ public class DocumentService {
|
||||
return DocumentSearchResult.paged(enrichItems(slice, text), pageable, totalElements);
|
||||
}
|
||||
|
||||
private List<DocumentSearchItem> enrichItems(List<Document> documents, String text) {
|
||||
private List<DocumentListItem> enrichItems(List<Document> documents, String text) {
|
||||
List<Document> colorResolved = resolveDocumentTagColors(documents);
|
||||
Map<UUID, SearchMatchData> matchData = enrichWithMatchData(colorResolved, text);
|
||||
|
||||
@@ -743,7 +743,7 @@ public class DocumentService {
|
||||
Map<UUID, Integer> completionByDoc = fetchCompletionPercentages(docIds);
|
||||
Map<UUID, List<ActivityActorDTO>> contributorsByDoc = auditLogQueryService.findRecentContributorsPerDocument(docIds);
|
||||
|
||||
return colorResolved.stream().map(doc -> new DocumentSearchItem(
|
||||
return colorResolved.stream().map(doc -> toListItem(
|
||||
doc,
|
||||
matchData.getOrDefault(doc.getId(), SearchMatchData.empty()),
|
||||
completionByDoc.getOrDefault(doc.getId(), 0),
|
||||
@@ -751,6 +751,26 @@ public class DocumentService {
|
||||
)).toList();
|
||||
}
|
||||
|
||||
private DocumentListItem toListItem(Document doc, SearchMatchData match, int completionPct, List<ActivityActorDTO> contributors) {
|
||||
return new DocumentListItem(
|
||||
doc.getId(),
|
||||
doc.getTitle(),
|
||||
doc.getOriginalFilename(),
|
||||
doc.getThumbnailUrl(),
|
||||
doc.getDocumentDate(),
|
||||
doc.getSender(),
|
||||
List.copyOf(doc.getReceivers()),
|
||||
List.copyOf(doc.getTags()),
|
||||
doc.getArchiveBox(),
|
||||
doc.getArchiveFolder(),
|
||||
doc.getLocation(),
|
||||
doc.getSummary(),
|
||||
completionPct,
|
||||
contributors,
|
||||
match
|
||||
);
|
||||
}
|
||||
|
||||
private Map<UUID, Integer> fetchCompletionPercentages(List<UUID> docIds) {
|
||||
return transcriptionBlockQueryService.getCompletionStats(docIds);
|
||||
}
|
||||
@@ -843,6 +863,7 @@ public class DocumentService {
|
||||
documentRepository.save(doc);
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public Document getDocumentById(UUID id) {
|
||||
Document doc = documentRepository.findById(id)
|
||||
.orElseThrow(() -> DomainException.notFound(ErrorCode.DOCUMENT_NOT_FOUND, "Document not found: " + id));
|
||||
|
||||
@@ -43,7 +43,7 @@ public class TranscriptionBlockController {
|
||||
|
||||
@PostMapping
|
||||
@ResponseStatus(HttpStatus.CREATED)
|
||||
@RequirePermission(Permission.WRITE_ALL)
|
||||
@RequirePermission({Permission.ANNOTATE_ALL, Permission.WRITE_ALL})
|
||||
public TranscriptionBlock createBlock(
|
||||
@PathVariable UUID documentId,
|
||||
@Valid @RequestBody CreateTranscriptionBlockDTO dto,
|
||||
@@ -53,7 +53,7 @@ public class TranscriptionBlockController {
|
||||
}
|
||||
|
||||
@PutMapping("/{blockId}")
|
||||
@RequirePermission(Permission.WRITE_ALL)
|
||||
@RequirePermission({Permission.ANNOTATE_ALL, Permission.WRITE_ALL})
|
||||
public TranscriptionBlock updateBlock(
|
||||
@PathVariable UUID documentId,
|
||||
@PathVariable UUID blockId,
|
||||
@@ -65,7 +65,7 @@ public class TranscriptionBlockController {
|
||||
|
||||
@DeleteMapping("/{blockId}")
|
||||
@ResponseStatus(HttpStatus.NO_CONTENT)
|
||||
@RequirePermission(Permission.WRITE_ALL)
|
||||
@RequirePermission({Permission.ANNOTATE_ALL, Permission.WRITE_ALL})
|
||||
public void deleteBlock(
|
||||
@PathVariable UUID documentId,
|
||||
@PathVariable UUID blockId) {
|
||||
@@ -73,7 +73,7 @@ public class TranscriptionBlockController {
|
||||
}
|
||||
|
||||
@PutMapping("/reorder")
|
||||
@RequirePermission(Permission.WRITE_ALL)
|
||||
@RequirePermission({Permission.ANNOTATE_ALL, Permission.WRITE_ALL})
|
||||
public List<TranscriptionBlock> reorderBlocks(
|
||||
@PathVariable UUID documentId,
|
||||
@RequestBody ReorderTranscriptionBlocksDTO dto) {
|
||||
@@ -82,7 +82,7 @@ public class TranscriptionBlockController {
|
||||
}
|
||||
|
||||
@PutMapping("/{blockId}/review")
|
||||
@RequirePermission(Permission.WRITE_ALL)
|
||||
@RequirePermission({Permission.ANNOTATE_ALL, Permission.WRITE_ALL})
|
||||
public TranscriptionBlock reviewBlock(
|
||||
@PathVariable UUID documentId,
|
||||
@PathVariable UUID blockId,
|
||||
@@ -92,7 +92,7 @@ public class TranscriptionBlockController {
|
||||
}
|
||||
|
||||
@PutMapping("/review-all")
|
||||
@RequirePermission(Permission.WRITE_ALL)
|
||||
@RequirePermission({Permission.ANNOTATE_ALL, Permission.WRITE_ALL})
|
||||
public List<TranscriptionBlock> markAllBlocksReviewed(
|
||||
@PathVariable UUID documentId,
|
||||
Authentication authentication) {
|
||||
|
||||
@@ -10,11 +10,21 @@ public class DomainException extends RuntimeException {
|
||||
|
||||
private final ErrorCode code;
|
||||
private final HttpStatus status;
|
||||
/** Seconds until the rate-limit window resets; {@code null} when not applicable. */
|
||||
private final Long retryAfterSeconds;
|
||||
|
||||
public DomainException(ErrorCode code, HttpStatus status, String developerMessage) {
|
||||
super(developerMessage);
|
||||
this.code = code;
|
||||
this.status = status;
|
||||
this.retryAfterSeconds = null;
|
||||
}
|
||||
|
||||
private DomainException(ErrorCode code, HttpStatus status, String developerMessage, Long retryAfterSeconds) {
|
||||
super(developerMessage);
|
||||
this.code = code;
|
||||
this.status = status;
|
||||
this.retryAfterSeconds = retryAfterSeconds;
|
||||
}
|
||||
|
||||
public ErrorCode getCode() {
|
||||
@@ -25,6 +35,11 @@ public class DomainException extends RuntimeException {
|
||||
return status;
|
||||
}
|
||||
|
||||
/** Returns the {@code Retry-After} value in seconds, or {@code null} if not set. */
|
||||
public Long getRetryAfterSeconds() {
|
||||
return retryAfterSeconds;
|
||||
}
|
||||
|
||||
// --- Static factories for common cases ---
|
||||
|
||||
public static DomainException notFound(ErrorCode code, String message) {
|
||||
@@ -59,4 +74,8 @@ public class DomainException extends RuntimeException {
|
||||
public static DomainException tooManyRequests(ErrorCode code, String message) {
|
||||
return new DomainException(code, HttpStatus.TOO_MANY_REQUESTS, message);
|
||||
}
|
||||
|
||||
public static DomainException tooManyRequests(ErrorCode code, String message, long retryAfterSeconds) {
|
||||
return new DomainException(code, HttpStatus.TOO_MANY_REQUESTS, message, retryAfterSeconds);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,9 +23,11 @@ public class GlobalExceptionHandler {
|
||||
|
||||
@ExceptionHandler(DomainException.class)
|
||||
public ResponseEntity<ErrorResponse> handleDomain(DomainException ex) {
|
||||
return ResponseEntity
|
||||
.status(ex.getStatus())
|
||||
.body(new ErrorResponse(ex.getCode(), ex.getMessage()));
|
||||
var builder = ResponseEntity.status(ex.getStatus());
|
||||
if (ex.getRetryAfterSeconds() != null) {
|
||||
builder = builder.header("Retry-After", String.valueOf(ex.getRetryAfterSeconds()));
|
||||
}
|
||||
return builder.body(new ErrorResponse(ex.getCode(), ex.getMessage()));
|
||||
}
|
||||
|
||||
@ExceptionHandler(MethodArgumentNotValidException.class)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.*;
|
||||
@@ -31,6 +33,7 @@ import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
@@ -53,9 +56,41 @@ public class MassImportService {
|
||||
|
||||
public enum State { IDLE, RUNNING, DONE, FAILED }
|
||||
|
||||
public record ImportStatus(State state, String statusCode, @JsonIgnore String message, int processed, LocalDateTime startedAt) {}
|
||||
public enum SkipReason {
|
||||
INVALID_FILENAME_PATH_TRAVERSAL,
|
||||
INVALID_PDF_SIGNATURE,
|
||||
FILE_READ_ERROR,
|
||||
ALREADY_EXISTS,
|
||||
S3_UPLOAD_FAILED
|
||||
}
|
||||
|
||||
private volatile ImportStatus currentStatus = new ImportStatus(State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, null);
|
||||
public record SkippedFile(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) String filename,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) SkipReason reason
|
||||
) {}
|
||||
|
||||
public record ImportStatus(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) State state,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) String statusCode,
|
||||
@JsonIgnore String message,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) int processed,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) List<SkippedFile> skippedFiles,
|
||||
LocalDateTime startedAt
|
||||
) {
|
||||
// Note: @Schema on a record accessor method is not picked up by SpringDoc; the
|
||||
// "skipped" count is a computed convenience field derived from skippedFiles.size().
|
||||
@JsonProperty("skipped")
|
||||
public int skipped() { return skippedFiles.size(); }
|
||||
|
||||
/** Defensive-copy constructor — callers cannot mutate the stored list after construction. */
|
||||
public ImportStatus {
|
||||
skippedFiles = List.copyOf(skippedFiles);
|
||||
}
|
||||
}
|
||||
|
||||
record ProcessResult(int processed, List<SkippedFile> skippedFiles) {}
|
||||
|
||||
private volatile ImportStatus currentStatus = new ImportStatus(State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
|
||||
public ImportStatus getStatus() {
|
||||
return currentStatus;
|
||||
@@ -117,22 +152,22 @@ public class MassImportService {
|
||||
if (currentStatus.state() == State.RUNNING) {
|
||||
throw DomainException.conflict(ErrorCode.IMPORT_ALREADY_RUNNING, "A mass import is already in progress");
|
||||
}
|
||||
currentStatus = new ImportStatus(State.RUNNING, "IMPORT_RUNNING", "Import läuft...", 0, LocalDateTime.now());
|
||||
currentStatus = new ImportStatus(State.RUNNING, "IMPORT_RUNNING", "Import läuft...", 0, List.of(), LocalDateTime.now());
|
||||
try {
|
||||
File spreadsheet = findSpreadsheetFile();
|
||||
log.info("Starte Massenimport aus: {}", spreadsheet.getAbsolutePath());
|
||||
int processed = processRows(readSpreadsheet(spreadsheet));
|
||||
ProcessResult result = processRows(readSpreadsheet(spreadsheet));
|
||||
currentStatus = new ImportStatus(State.DONE, "IMPORT_DONE",
|
||||
"Import abgeschlossen. " + processed + " Dokumente verarbeitet.",
|
||||
processed, currentStatus.startedAt());
|
||||
"Import abgeschlossen. " + result.processed() + " Dokumente verarbeitet.",
|
||||
result.processed(), result.skippedFiles(), currentStatus.startedAt());
|
||||
} catch (NoSpreadsheetException e) {
|
||||
log.error("Massenimport fehlgeschlagen: keine Tabellendatei", e);
|
||||
currentStatus = new ImportStatus(State.FAILED, "IMPORT_FAILED_NO_SPREADSHEET",
|
||||
"Fehler: " + e.getMessage(), 0, currentStatus.startedAt());
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
} catch (Exception e) {
|
||||
log.error("Massenimport fehlgeschlagen", e);
|
||||
currentStatus = new ImportStatus(State.FAILED, "IMPORT_FAILED_INTERNAL",
|
||||
"Fehler: " + e.getMessage(), 0, currentStatus.startedAt());
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,30 +289,94 @@ public class MassImportService {
|
||||
|
||||
// --- Import logic (works on neutral List<String> rows) ---
|
||||
|
||||
private int processRows(List<List<String>> rows) {
|
||||
int count = 0;
|
||||
private ProcessResult processRows(List<List<String>> rows) {
|
||||
int processed = 0;
|
||||
List<SkippedFile> skippedFiles = new ArrayList<>();
|
||||
|
||||
for (int i = 1; i < rows.size(); i++) { // skip header row
|
||||
List<String> cells = rows.get(i);
|
||||
String index = getCell(cells, colIndex);
|
||||
if (index.isBlank()) continue;
|
||||
|
||||
String filename = index.contains(".") ? index : index + ".pdf";
|
||||
if (!isValidImportFilename(filename)) {
|
||||
log.warn("Skipping import row {}: filename rejected — {}", i, filename);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.INVALID_FILENAME_PATH_TRAVERSAL));
|
||||
continue;
|
||||
}
|
||||
Optional<File> fileOnDisk = findFileRecursive(filename);
|
||||
if (fileOnDisk.isEmpty()) {
|
||||
log.warn("Datei nicht gefunden, importiere nur Metadaten: {}", filename);
|
||||
}
|
||||
importSingleDocument(cells, fileOnDisk, filename, index);
|
||||
count++;
|
||||
|
||||
if (fileOnDisk.isPresent()) {
|
||||
try {
|
||||
if (!isPdfMagicBytes(fileOnDisk.get())) {
|
||||
log.warn("Überspringe {}: Datei beginnt nicht mit %PDF-Signatur", filename);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.INVALID_PDF_SIGNATURE));
|
||||
continue;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Fehler beim Prüfen der Magic-Bytes für {}", filename, e);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.FILE_READ_ERROR));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Optional<SkipReason> skipReason = importSingleDocument(cells, fileOnDisk, filename, index);
|
||||
if (skipReason.isPresent()) {
|
||||
skippedFiles.add(new SkippedFile(filename, skipReason.get()));
|
||||
} else {
|
||||
processed++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
return new ProcessResult(processed, skippedFiles);
|
||||
}
|
||||
|
||||
private boolean isValidImportFilename(String filename) {
|
||||
if (filename == null || filename.isBlank()) return false;
|
||||
if (filename.contains("/")) return false;
|
||||
if (filename.contains("\\")) return false;
|
||||
if (filename.contains("∕")) return false; // U+2215 DIVISION SLASH
|
||||
if (filename.contains("/")) return false; // U+FF0F FULLWIDTH SOLIDUS
|
||||
if (filename.contains("⧵")) return false; // U+29F5 REVERSE SOLIDUS OPERATOR
|
||||
if (filename.contains("..")) return false;
|
||||
if (filename.equals(".")) return false;
|
||||
if (filename.contains("\0")) return false;
|
||||
// Paths.get() is safe here on Linux for all inputs that passed the checks above;
|
||||
// it may throw InvalidPathException for OS-specific illegal chars on Windows,
|
||||
// but those are not reachable in production.
|
||||
if (Paths.get(filename).isAbsolute()) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// package-private: Mockito spy in tests can override to inject IOException
|
||||
InputStream openFileStream(File file) throws IOException {
|
||||
return new FileInputStream(file);
|
||||
}
|
||||
|
||||
private boolean isPdfMagicBytes(File file) throws IOException {
|
||||
try (InputStream is = openFileStream(file)) {
|
||||
byte[] header = is.readNBytes(4);
|
||||
return header.length == 4
|
||||
&& header[0] == 0x25 // %
|
||||
&& header[1] == 0x50 // P
|
||||
&& header[2] == 0x44 // D
|
||||
&& header[3] == 0x46; // F
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports a single document row.
|
||||
*
|
||||
* @return empty Optional on success; an Optional containing the skip reason on failure/skip.
|
||||
*/
|
||||
@Transactional
|
||||
protected void importSingleDocument(List<String> cells, Optional<File> file, String originalFilename, String index) {
|
||||
protected Optional<SkipReason> importSingleDocument(List<String> cells, Optional<File> file, String originalFilename, String index) {
|
||||
Optional<Document> existing = documentService.findByOriginalFilename(originalFilename);
|
||||
if (existing.isPresent() && existing.get().getStatus() != DocumentStatus.PLACEHOLDER) {
|
||||
log.info("Dokument {} existiert bereits, überspringe.", originalFilename);
|
||||
return;
|
||||
return Optional.of(SkipReason.ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
String archiveBox = getCell(cells, colBox);
|
||||
@@ -313,7 +412,7 @@ public class MassImportService {
|
||||
status = DocumentStatus.UPLOADED;
|
||||
} catch (Exception e) {
|
||||
log.error("S3 Upload Fehler für {}", file.get().getName(), e);
|
||||
return;
|
||||
return Optional.of(SkipReason.S3_UPLOAD_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -355,6 +454,7 @@ public class MassImportService {
|
||||
thumbnailAsyncRunner.dispatchAfterCommit(saved.getId());
|
||||
}
|
||||
log.info("Importiert{}: {}", file.isEmpty() ? " (nur Metadaten)" : "", originalFilename);
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
@@ -390,11 +490,18 @@ public class MassImportService {
|
||||
}
|
||||
|
||||
private Optional<File> findFileRecursive(String filename) {
|
||||
try (Stream<Path> walk = Files.walk(Paths.get(importDir))) {
|
||||
return walk.filter(p -> !Files.isDirectory(p))
|
||||
File baseDir = new File(importDir);
|
||||
try (Stream<Path> walk = Files.walk(baseDir.toPath())) {
|
||||
Optional<Path> match = walk.filter(p -> !Files.isDirectory(p))
|
||||
.filter(p -> p.getFileName().toString().equals(filename))
|
||||
.map(Path::toFile)
|
||||
.findFirst();
|
||||
if (match.isEmpty()) return Optional.empty();
|
||||
File candidate = match.get().toFile();
|
||||
String baseDirCanonical = baseDir.getCanonicalPath();
|
||||
if (!candidate.getCanonicalPath().startsWith(baseDirCanonical + File.separator)) {
|
||||
throw DomainException.internal(ErrorCode.INTERNAL_ERROR, "Path escape detected: " + candidate);
|
||||
}
|
||||
return Optional.of(candidate);
|
||||
} catch (IOException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package org.raddatz.familienarchiv.person;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.persistence.*;
|
||||
import lombok.*;
|
||||
@@ -9,6 +10,9 @@ import org.raddatz.familienarchiv.user.DisplayNameFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
// prevents infinite recursion in JSON serialization; see ADR-022 for lazy-fetch context
|
||||
@JsonIgnoreProperties({"hibernateLazyInitializer", "handler"})
|
||||
@Entity
|
||||
@Table(name = "persons")
|
||||
@Data
|
||||
|
||||
@@ -32,6 +32,11 @@ import java.util.Map;
|
||||
@RequiredArgsConstructor
|
||||
public class SecurityConfig {
|
||||
|
||||
// @WebMvcTest slices do not include JacksonAutoConfiguration, so ObjectMapper
|
||||
// cannot be injected here. A static instance is safe because the response
|
||||
// only serializes fixed String keys — no custom naming strategy or module needed.
|
||||
private static final ObjectMapper ERROR_WRITER = new ObjectMapper();
|
||||
|
||||
private final CustomUserDetailsService userDetailsService;
|
||||
private final Environment environment;
|
||||
|
||||
@@ -88,7 +93,7 @@ public class SecurityConfig {
|
||||
// CSRF protection via CookieCsrfTokenRepository (NFR-SEC-103).
|
||||
// The backend sets an XSRF-TOKEN cookie (not HttpOnly so JS can read it).
|
||||
// All state-changing requests must include X-XSRF-TOKEN matching the cookie.
|
||||
// See ADR-020 and issue #524 for the full security rationale.
|
||||
// See ADR-022 and issue #524 for the full security rationale.
|
||||
.csrf(csrf -> csrf
|
||||
.csrfTokenRepository(CookieCsrfTokenRepository.withHttpOnlyFalse())
|
||||
.csrfTokenRequestHandler(new CsrfTokenRequestAttributeHandler()))
|
||||
@@ -127,7 +132,7 @@ public class SecurityConfig {
|
||||
ErrorCode code = (e instanceof CsrfException)
|
||||
? ErrorCode.CSRF_TOKEN_MISSING
|
||||
: ErrorCode.FORBIDDEN;
|
||||
res.getWriter().write(new ObjectMapper().writeValueAsString(Map.of("code", code.name())));
|
||||
res.getWriter().write(ERROR_WRITER.writeValueAsString(Map.of("code", code.name())));
|
||||
}));
|
||||
|
||||
return http.build();
|
||||
|
||||
@@ -2,10 +2,13 @@ package org.raddatz.familienarchiv.tag;
|
||||
|
||||
import java.util.UUID;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.persistence.*;
|
||||
import lombok.*;
|
||||
|
||||
// prevents infinite recursion in JSON serialization; see ADR-022 for lazy-fetch context
|
||||
@JsonIgnoreProperties({"hibernateLazyInitializer", "handler"})
|
||||
@Entity
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
|
||||
@@ -31,5 +31,6 @@ public class InviteListItemDTO {
|
||||
private String status;
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private LocalDateTime createdAt;
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private String shareableUrl;
|
||||
}
|
||||
|
||||
@@ -30,15 +30,15 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.ResponseStatus;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/")
|
||||
@AllArgsConstructor
|
||||
@RequiredArgsConstructor
|
||||
public class UserController {
|
||||
private UserService userService;
|
||||
private AuthService authService;
|
||||
private AuditService auditService;
|
||||
private final UserService userService;
|
||||
private final AuthService authService;
|
||||
private final AuditService auditService;
|
||||
|
||||
@GetMapping("users/me")
|
||||
public ResponseEntity<AppUser> getCurrentUser(Authentication authentication) {
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Repeatable migration: sets the grafana_reader role's password from the
|
||||
-- ${grafanaDbPassword} placeholder (resolved by FlywayConfig from the
|
||||
-- GRAFANA_DB_PASSWORD environment variable). Flyway computes the checksum on
|
||||
-- the resolved migration content, so any change to GRAFANA_DB_PASSWORD changes
|
||||
-- the checksum and re-applies this migration on the next boot. That makes
|
||||
-- password rotation a "change env var + restart" operation — no manual psql.
|
||||
--
|
||||
-- V68 created the role itself (without a usable password). This file owns the
|
||||
-- password lifecycle; nothing else writes it.
|
||||
DO $$
|
||||
BEGIN
|
||||
EXECUTE format('ALTER ROLE grafana_reader WITH PASSWORD %L', '${grafanaDbPassword}');
|
||||
END
|
||||
$$;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- Read-only role used by the Grafana PostgreSQL datasource for the PO Overview
|
||||
-- dashboard (issue #651). The role is created here without a usable password
|
||||
-- (LOGIN-capable but no password set); R__grafana_reader_password.sql sets the
|
||||
-- password from GRAFANA_DB_PASSWORD on every boot, so rotation is just "bump
|
||||
-- the env var and restart the backend" — see docs/adr/024-* and the rotation
|
||||
-- runbook in docs/DEPLOYMENT.md.
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'grafana_reader') THEN
|
||||
CREATE ROLE grafana_reader WITH LOGIN;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
|
||||
GRANT CONNECT ON DATABASE ${flyway:database} TO grafana_reader;
|
||||
GRANT USAGE ON SCHEMA public TO grafana_reader;
|
||||
GRANT SELECT ON audit_log, documents, transcription_blocks TO grafana_reader;
|
||||
@@ -15,17 +15,10 @@ import org.springframework.security.authentication.AuthenticationManager;
|
||||
import org.springframework.security.authentication.BadCredentialsException;
|
||||
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
|
||||
import org.springframework.security.core.Authentication;
|
||||
import org.springframework.session.jdbc.JdbcIndexedSessionRepository;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
@@ -37,19 +30,13 @@ class AuthServiceTest {
|
||||
@Mock AuthenticationManager authenticationManager;
|
||||
@Mock UserService userService;
|
||||
@Mock AuditService auditService;
|
||||
@Mock JdbcIndexedSessionRepository sessionRepository;
|
||||
@Mock LoginRateLimiter loginRateLimiter;
|
||||
@Mock SessionRevocationPort sessionRevocationPort;
|
||||
@InjectMocks AuthService authService;
|
||||
|
||||
private static final String IP = "127.0.0.1";
|
||||
private static final String UA = "Mozilla/5.0 (Test)";
|
||||
|
||||
@BeforeEach
|
||||
void injectOptionalFields() {
|
||||
ReflectionTestUtils.setField(authService, "sessionRepository", sessionRepository);
|
||||
ReflectionTestUtils.setField(authService, "loginRateLimiter", loginRateLimiter);
|
||||
}
|
||||
|
||||
@Test
|
||||
void login_returns_user_on_valid_credentials() {
|
||||
UUID userId = UUID.randomUUID();
|
||||
@@ -159,7 +146,6 @@ class AuthServiceTest {
|
||||
|
||||
@Test
|
||||
void login_fires_LOGIN_RATE_LIMITED_audit_when_rate_limited() {
|
||||
UUID userId = UUID.randomUUID();
|
||||
doThrow(DomainException.tooManyRequests(ErrorCode.TOO_MANY_LOGIN_ATTEMPTS, "rate limited"))
|
||||
.when(loginRateLimiter).checkAndConsume(IP, "user@test.de");
|
||||
|
||||
@@ -183,35 +169,23 @@ class AuthServiceTest {
|
||||
verify(loginRateLimiter).invalidateOnSuccess(IP, "user@test.de");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void revokeOtherSessions_preserves_current_and_deletes_N_minus_1() {
|
||||
var sessions = new HashMap<String, Object>();
|
||||
sessions.put("session-keep", null);
|
||||
sessions.put("session-del-1", null);
|
||||
sessions.put("session-del-2", null);
|
||||
doReturn(sessions).when(sessionRepository).findByPrincipalName("user@test.de");
|
||||
void revokeOtherSessions_delegates_to_port() {
|
||||
when(sessionRevocationPort.revokeOtherSessions("session-keep", "user@test.de")).thenReturn(2);
|
||||
|
||||
int count = authService.revokeOtherSessions("session-keep", "user@test.de");
|
||||
|
||||
assertThat(count).isEqualTo(2);
|
||||
verify(sessionRepository, never()).deleteById("session-keep");
|
||||
verify(sessionRepository).deleteById("session-del-1");
|
||||
verify(sessionRepository).deleteById("session-del-2");
|
||||
verify(sessionRevocationPort).revokeOtherSessions("session-keep", "user@test.de");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void revokeAllSessions_deletes_all_sessions_for_principal() {
|
||||
var sessions = new HashMap<String, Object>();
|
||||
sessions.put("session-1", null);
|
||||
sessions.put("session-2", null);
|
||||
doReturn(sessions).when(sessionRepository).findByPrincipalName("user@test.de");
|
||||
void revokeAllSessions_delegates_to_port() {
|
||||
when(sessionRevocationPort.revokeAllSessions("user@test.de")).thenReturn(3);
|
||||
|
||||
int count = authService.revokeAllSessions("user@test.de");
|
||||
|
||||
assertThat(count).isEqualTo(2);
|
||||
verify(sessionRepository).deleteById("session-1");
|
||||
verify(sessionRepository).deleteById("session-2");
|
||||
assertThat(count).isEqualTo(3);
|
||||
verify(sessionRevocationPort).revokeAllSessions("user@test.de");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +119,21 @@ class AuthSessionIntegrationTest {
|
||||
assertThat(me.getStatusCode().value()).isEqualTo(401);
|
||||
}
|
||||
|
||||
// ─── Task: CSRF rejection at integration layer ────────────────────────────
|
||||
|
||||
@Test
|
||||
void post_without_csrf_token_returns_403_CSRF_TOKEN_MISSING() {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
// Deliberately omit XSRF-TOKEN cookie and X-XSRF-TOKEN header
|
||||
ResponseEntity<String> response = http.postForEntity(
|
||||
baseUrl + "/api/auth/logout",
|
||||
new HttpEntity<>("{}", headers), String.class);
|
||||
|
||||
assertThat(response.getStatusCode().value()).isEqualTo(403);
|
||||
assertThat(response.getBody()).contains("CSRF_TOKEN_MISSING");
|
||||
}
|
||||
|
||||
// ─── helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package org.raddatz.familienarchiv.auth;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||
import org.springframework.transaction.support.TransactionTemplate;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Integration test for {@link JdbcSessionRevocationAdapter} that verifies
|
||||
* session rows are actually written to / removed from the {@code spring_session}
|
||||
* table backed by a real PostgreSQL container.
|
||||
*
|
||||
* <p>Sessions are inserted via raw JDBC to avoid the module-access restriction on
|
||||
* {@code JdbcIndexedSessionRepository.JdbcSession}. The {@link SessionRevocationPort}
|
||||
* bean injected here is the real {@link JdbcSessionRevocationAdapter} wired by Spring.
|
||||
*/
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@ActiveProfiles("test")
|
||||
@Import(PostgresContainerConfig.class)
|
||||
class JdbcSessionRevocationAdapterIntegrationTest {
|
||||
|
||||
@MockitoBean S3Client s3Client;
|
||||
|
||||
@Autowired SessionRevocationPort adapter;
|
||||
@Autowired JdbcTemplate jdbcTemplate;
|
||||
@Autowired TransactionTemplate transactionTemplate;
|
||||
|
||||
private static final String PRINCIPAL = "revocation-it@test.de";
|
||||
|
||||
@BeforeEach
|
||||
void clearSessions() {
|
||||
// spring_session_attributes cascades on delete
|
||||
transactionTemplate.execute(status -> {
|
||||
jdbcTemplate.update("DELETE FROM spring_session");
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
// ── helper ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Inserts a minimal {@code spring_session} row attributed to {@value #PRINCIPAL}
|
||||
* and returns its opaque primary-key ID (the value the repository uses as the
|
||||
* session identifier, not the {@code SESSION_ID} column which holds the public token).
|
||||
*
|
||||
* <p>Column layout mirrors the Flyway-managed schema shipped with the app:
|
||||
* PRIMARY_ID, SESSION_ID, CREATION_TIME, LAST_ACCESS_TIME, MAX_INACTIVE_INTERVAL,
|
||||
* EXPIRY_TIME, PRINCIPAL_NAME.
|
||||
*/
|
||||
/**
|
||||
* Inserts a persisted session row for {@value #PRINCIPAL} and returns the
|
||||
* {@code SESSION_ID} column value — this is the opaque identifier that
|
||||
* {@link JdbcIndexedSessionRepository} uses as the session's public key
|
||||
* (returned by {@code JdbcSession.getId()} and expected by
|
||||
* {@link JdbcIndexedSessionRepository#deleteById}).
|
||||
*
|
||||
* <p>The inserts run inside a {@link TransactionTemplate} so the rows are
|
||||
* committed before {@code findByPrincipalName} opens its own transaction and
|
||||
* can see the data via Read Committed isolation.
|
||||
*/
|
||||
private String insertSession() {
|
||||
String primaryId = UUID.randomUUID().toString();
|
||||
// SESSION_ID is the value used by JdbcSession.getId() and findByPrincipalName map keys.
|
||||
String sessionId = UUID.randomUUID().toString();
|
||||
long now = Instant.now().toEpochMilli();
|
||||
long expiry = now + 8L * 3600 * 1000; // 8-hour TTL
|
||||
transactionTemplate.execute(status -> {
|
||||
jdbcTemplate.update("""
|
||||
INSERT INTO spring_session
|
||||
(PRIMARY_ID, SESSION_ID, CREATION_TIME, LAST_ACCESS_TIME,
|
||||
MAX_INACTIVE_INTERVAL, EXPIRY_TIME, PRINCIPAL_NAME)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
primaryId, sessionId, now, now, 28800, expiry, PRINCIPAL);
|
||||
// Spring Session's listSessionsByPrincipalName query joins spring_session_attributes;
|
||||
// insert a minimal attribute row so the session appears in the result set.
|
||||
jdbcTemplate.update("""
|
||||
INSERT INTO spring_session_attributes
|
||||
(SESSION_PRIMARY_ID, ATTRIBUTE_NAME, ATTRIBUTE_BYTES)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
primaryId, "test_attr", new byte[]{0});
|
||||
return null;
|
||||
});
|
||||
return sessionId; // the public key used by JdbcSession.getId() and deleteById()
|
||||
}
|
||||
|
||||
// ── tests ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void revokeAllSessions_removes_every_row_from_spring_session_table() {
|
||||
insertSession();
|
||||
insertSession();
|
||||
|
||||
int count = adapter.revokeAllSessions(PRINCIPAL);
|
||||
|
||||
assertThat(count).isEqualTo(2);
|
||||
assertThat(jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM spring_session WHERE PRINCIPAL_NAME = ?",
|
||||
Long.class, PRINCIPAL))
|
||||
.isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void revokeOtherSessions_deletes_non_current_rows_and_keeps_current_session() {
|
||||
String keepId = insertSession();
|
||||
insertSession();
|
||||
insertSession();
|
||||
|
||||
int count = adapter.revokeOtherSessions(keepId, PRINCIPAL);
|
||||
|
||||
assertThat(count).isEqualTo(2);
|
||||
// The current session row must still be present (keyed by SESSION_ID)
|
||||
assertThat(jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM spring_session WHERE SESSION_ID = ?",
|
||||
Long.class, keepId))
|
||||
.isEqualTo(1L);
|
||||
// The total for this principal is now exactly 1
|
||||
assertThat(jdbcTemplate.queryForObject(
|
||||
"SELECT COUNT(*) FROM spring_session WHERE PRINCIPAL_NAME = ?",
|
||||
Long.class, PRINCIPAL))
|
||||
.isEqualTo(1L);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package org.raddatz.familienarchiv.auth;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.session.jdbc.JdbcIndexedSessionRepository;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class JdbcSessionRevocationAdapterTest {
|
||||
|
||||
@Mock JdbcIndexedSessionRepository sessionRepository;
|
||||
@InjectMocks JdbcSessionRevocationAdapter adapter;
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void revokeOtherSessions_preserves_current_and_deletes_N_minus_1() {
|
||||
var sessions = new HashMap<String, Object>();
|
||||
sessions.put("session-keep", null);
|
||||
sessions.put("session-del-1", null);
|
||||
sessions.put("session-del-2", null);
|
||||
doReturn(sessions).when(sessionRepository).findByPrincipalName("user@test.de");
|
||||
|
||||
int count = adapter.revokeOtherSessions("session-keep", "user@test.de");
|
||||
|
||||
assertThat(count).isEqualTo(2);
|
||||
verify(sessionRepository, never()).deleteById("session-keep");
|
||||
verify(sessionRepository).deleteById("session-del-1");
|
||||
verify(sessionRepository).deleteById("session-del-2");
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Test
|
||||
void revokeAllSessions_deletes_all_sessions_for_principal() {
|
||||
var sessions = new HashMap<String, Object>();
|
||||
sessions.put("session-1", null);
|
||||
sessions.put("session-2", null);
|
||||
doReturn(sessions).when(sessionRepository).findByPrincipalName("user@test.de");
|
||||
|
||||
int count = adapter.revokeAllSessions("user@test.de");
|
||||
|
||||
assertThat(count).isEqualTo(2);
|
||||
verify(sessionRepository).deleteById("session-1");
|
||||
verify(sessionRepository).deleteById("session-2");
|
||||
}
|
||||
}
|
||||
@@ -5,8 +5,9 @@ import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatNoException;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
class LoginRateLimiterTest {
|
||||
|
||||
@@ -37,10 +38,22 @@ class LoginRateLimiterTest {
|
||||
|
||||
assertThatThrownBy(() -> rateLimiter.checkAndConsume("1.2.3.4", "user@example.com"))
|
||||
.isInstanceOf(DomainException.class)
|
||||
.satisfies(ex -> org.assertj.core.api.Assertions.assertThat(((DomainException) ex).getCode())
|
||||
.satisfies(ex -> assertThat(((DomainException) ex).getCode())
|
||||
.isEqualTo(ErrorCode.TOO_MANY_LOGIN_ATTEMPTS));
|
||||
}
|
||||
|
||||
@Test
|
||||
void blocked_attempt_carries_retry_after_seconds_equal_to_window_duration() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
rateLimiter.checkAndConsume("1.2.3.4", "user@example.com");
|
||||
}
|
||||
|
||||
assertThatThrownBy(() -> rateLimiter.checkAndConsume("1.2.3.4", "user@example.com"))
|
||||
.isInstanceOf(DomainException.class)
|
||||
.satisfies(ex -> assertThat(((DomainException) ex).getRetryAfterSeconds())
|
||||
.isEqualTo(15 * 60L)); // windowMinutes=15 → 900 seconds
|
||||
}
|
||||
|
||||
@Test
|
||||
void success_after_10_failures_resets_ip_email_bucket() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
@@ -61,7 +74,75 @@ class LoginRateLimiterTest {
|
||||
|
||||
assertThatThrownBy(() -> rateLimiter.checkAndConsume("1.2.3.4", "attacker@example.com"))
|
||||
.isInstanceOf(DomainException.class)
|
||||
.satisfies(ex -> org.assertj.core.api.Assertions.assertThat(((DomainException) ex).getCode())
|
||||
.satisfies(ex -> assertThat(((DomainException) ex).getCode())
|
||||
.isEqualTo(ErrorCode.TOO_MANY_LOGIN_ATTEMPTS));
|
||||
}
|
||||
|
||||
@Test
|
||||
void different_email_from_same_ip_not_blocked_by_sibling_email_exhaustion() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
rateLimiter.checkAndConsume("1.2.3.4", "user@example.com");
|
||||
}
|
||||
|
||||
assertThatThrownBy(() -> rateLimiter.checkAndConsume("1.2.3.4", "user@example.com"))
|
||||
.isInstanceOf(DomainException.class);
|
||||
|
||||
assertThatNoException().isThrownBy(
|
||||
() -> rateLimiter.checkAndConsume("1.2.3.4", "other@example.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void email_lookup_is_case_insensitive_so_mixed_case_shares_the_same_bucket() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
rateLimiter.checkAndConsume("1.2.3.4", "User@Example.COM");
|
||||
}
|
||||
|
||||
assertThatThrownBy(() -> rateLimiter.checkAndConsume("1.2.3.4", "user@example.com"))
|
||||
.isInstanceOf(DomainException.class)
|
||||
.satisfies(ex -> assertThat(((DomainException) ex).getCode())
|
||||
.isEqualTo(ErrorCode.TOO_MANY_LOGIN_ATTEMPTS));
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidateOnSuccess_is_case_insensitive_so_mixed_case_clears_the_bucket() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
rateLimiter.checkAndConsume("1.2.3.4", "user@example.com");
|
||||
}
|
||||
|
||||
rateLimiter.invalidateOnSuccess("1.2.3.4", "User@Example.COM");
|
||||
|
||||
assertThatNoException().isThrownBy(
|
||||
() -> rateLimiter.checkAndConsume("1.2.3.4", "user@example.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ip_exhaustion_does_not_consume_ipEmail_tokens_for_blocked_attempts() {
|
||||
// Use a tighter limiter so the phantom-consumption effect is observable.
|
||||
// ipEmail=3, IP=3: exhausting IP via one email burns the other email's quota with the old code.
|
||||
RateLimitProperties props = new RateLimitProperties();
|
||||
props.setMaxAttemptsPerIpEmail(3);
|
||||
props.setMaxAttemptsPerIp(3);
|
||||
props.setWindowMinutes(15);
|
||||
LoginRateLimiter tightLimiter = new LoginRateLimiter(props);
|
||||
|
||||
// Exhaust the per-IP bucket using "user@"
|
||||
for (int i = 0; i < 3; i++) {
|
||||
tightLimiter.checkAndConsume("1.2.3.4", "user@example.com");
|
||||
}
|
||||
|
||||
// Three blocked attempts for "target@" while IP is exhausted
|
||||
for (int i = 0; i < 3; i++) {
|
||||
assertThatThrownBy(() -> tightLimiter.checkAndConsume("1.2.3.4", "target@example.com"))
|
||||
.isInstanceOf(DomainException.class);
|
||||
}
|
||||
|
||||
// A successful login for "user@" resets the IP bucket but NOT target@'s ipEmail bucket
|
||||
tightLimiter.invalidateOnSuccess("1.2.3.4", "user@example.com");
|
||||
|
||||
// After IP reset: "target@" must NOT be blocked by an exhausted ipEmail bucket.
|
||||
// With the old code, 3 blocked attempts burned all 3 ipEmail tokens → blocked here.
|
||||
// With the fix, tokens are refunded on each blocked attempt → still has capacity.
|
||||
assertThatNoException().isThrownBy(
|
||||
() -> tightLimiter.checkAndConsume("1.2.3.4", "target@example.com"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package org.raddatz.familienarchiv.config;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.mock.env.MockEnvironment;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
class FlywayConfigTest {
|
||||
|
||||
@Test
|
||||
void resolveGrafanaDbPassword_throws_when_env_unset() {
|
||||
FlywayConfig config = new FlywayConfig(null, new MockEnvironment());
|
||||
|
||||
assertThatThrownBy(config::resolveGrafanaDbPassword)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("GRAFANA_DB_PASSWORD is required");
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolveGrafanaDbPassword_throws_when_env_blank() {
|
||||
MockEnvironment env = new MockEnvironment().withProperty("GRAFANA_DB_PASSWORD", " ");
|
||||
FlywayConfig config = new FlywayConfig(null, env);
|
||||
|
||||
assertThatThrownBy(config::resolveGrafanaDbPassword)
|
||||
.isInstanceOf(IllegalStateException.class)
|
||||
.hasMessageContaining("GRAFANA_DB_PASSWORD is required");
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolveGrafanaDbPassword_returns_value_when_env_set() {
|
||||
MockEnvironment env = new MockEnvironment().withProperty("GRAFANA_DB_PASSWORD", "abc");
|
||||
FlywayConfig config = new FlywayConfig(null, env);
|
||||
|
||||
assertThat(config.resolveGrafanaDbPassword()).isEqualTo("abc");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package org.raddatz.familienarchiv.config;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
|
||||
import org.springframework.boot.jdbc.test.autoconfigure.AutoConfigureTestDatabase;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
// GRAFANA_DB_PASSWORD is supplied via the global test default in
|
||||
// src/test/resources/application.properties — FlywayConfig fails closed
|
||||
// when it is unset, so all tests that load the migration path need it.
|
||||
@DataJpaTest
|
||||
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
|
||||
@Import({PostgresContainerConfig.class, FlywayConfig.class})
|
||||
class GrafanaReaderRoleIntegrationTest {
|
||||
|
||||
@Autowired JdbcTemplate jdbc;
|
||||
|
||||
// --- positive grants (SELECT on the three explicitly granted tables) ---
|
||||
|
||||
@Test
|
||||
void grafana_reader_has_select_on_audit_log() {
|
||||
assertThat(hasPrivilege("audit_log", "SELECT")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void grafana_reader_has_select_on_documents() {
|
||||
assertThat(hasPrivilege("documents", "SELECT")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void grafana_reader_has_select_on_transcription_blocks() {
|
||||
assertThat(hasPrivilege("transcription_blocks", "SELECT")).isTrue();
|
||||
}
|
||||
|
||||
// --- write-deny on the granted tables: SELECT-only means SELECT-only.
|
||||
// A future migration that GRANTs INSERT/UPDATE/DELETE on any of these
|
||||
// would fail these tests, even though the original positive grants still
|
||||
// pass. Locks the boundary in both directions.
|
||||
|
||||
@Test
|
||||
void grafana_reader_has_no_INSERT_on_documents() {
|
||||
assertThat(hasPrivilege("documents", "INSERT")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void grafana_reader_has_no_UPDATE_on_audit_log() {
|
||||
assertThat(hasPrivilege("audit_log", "UPDATE")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void grafana_reader_has_no_DELETE_on_transcription_blocks() {
|
||||
assertThat(hasPrivilege("transcription_blocks", "DELETE")).isFalse();
|
||||
}
|
||||
|
||||
// --- negative grants: PII / sensitive tables MUST NOT be readable.
|
||||
// The parameterized form catches the "someone widened the grant to
|
||||
// ALL TABLES IN SCHEMA public" footgun — three specific positive grants
|
||||
// would still pass while this sweep turns red.
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {
|
||||
"app_users",
|
||||
"user_groups",
|
||||
"persons",
|
||||
"notifications",
|
||||
"document_comments",
|
||||
"document_annotations",
|
||||
"geschichten"
|
||||
})
|
||||
void grafana_reader_has_no_SELECT_on_protected_table(String table) {
|
||||
assertThat(hasPrivilege(table, "SELECT")).isFalse();
|
||||
}
|
||||
|
||||
private boolean hasPrivilege(String table, String privilege) {
|
||||
Boolean result = jdbc.queryForObject(
|
||||
"SELECT has_table_privilege('grafana_reader', ?, ?)",
|
||||
Boolean.class,
|
||||
table,
|
||||
privilege);
|
||||
return Boolean.TRUE.equals(result);
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,15 @@ class RateLimitInterceptorTest {
|
||||
verify(response).setStatus(HttpStatus.TOO_MANY_REQUESTS.value());
|
||||
}
|
||||
|
||||
@Test
|
||||
void blocked_response_includes_retry_after_header() throws Exception {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
interceptor.preHandle(request, response, null);
|
||||
}
|
||||
interceptor.preHandle(request, response, null);
|
||||
verify(response).setHeader("Retry-After", "60");
|
||||
}
|
||||
|
||||
@Test
|
||||
void different_ips_have_independent_limits() throws Exception {
|
||||
HttpServletRequest other = mock(HttpServletRequest.class);
|
||||
|
||||
@@ -27,7 +27,6 @@ import org.springframework.security.test.context.support.WithMockUser;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
|
||||
import org.raddatz.familienarchiv.document.DocumentSearchItem;
|
||||
import org.raddatz.familienarchiv.document.SearchMatchData;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
@@ -44,6 +43,7 @@ import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.patch;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
@@ -129,16 +129,13 @@ class DocumentControllerTest {
|
||||
@WithMockUser
|
||||
void search_responseBodyItemsContainMatchData() throws Exception {
|
||||
UUID docId = UUID.randomUUID();
|
||||
Document doc = Document.builder()
|
||||
.id(docId)
|
||||
.title("Brief an Anna")
|
||||
.originalFilename("brief.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.build();
|
||||
var matchData = new SearchMatchData(
|
||||
"Er schrieb einen langen Brief", List.of(), false, List.of(), List.of(), List.of(), null, List.of());
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of(new DocumentSearchItem(doc, matchData, 0, List.of()))));
|
||||
.thenReturn(DocumentSearchResult.of(List.of(new DocumentListItem(
|
||||
docId, "Brief an Anna", "brief.pdf", null, null, null,
|
||||
List.of(), List.of(), null, null, null, null,
|
||||
0, List.of(), matchData))));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search").param("q", "Brief"))
|
||||
.andExpect(status().isOk())
|
||||
@@ -147,6 +144,27 @@ class DocumentControllerTest {
|
||||
.value("Er schrieb einen langen Brief"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_returns_flat_item_with_id_and_without_sensitive_fields() throws Exception {
|
||||
UUID docId = UUID.randomUUID();
|
||||
var matchData = new SearchMatchData(null, List.of(), false, List.of(), List.of(), List.of(), null, List.of());
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of(new DocumentListItem(
|
||||
docId, "Brief an Anna", "brief.pdf", null, null, null,
|
||||
List.of(), List.of(), null, null, null, null,
|
||||
0, List.of(), matchData))));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search"))
|
||||
.andExpect(status().isOk())
|
||||
// flat id field present at top of item (not nested under $.items[0].document.id)
|
||||
.andExpect(jsonPath("$.items[0].id").value(docId.toString()))
|
||||
// sensitive storage fields must never appear in list response
|
||||
.andExpect(jsonPath("$.items[0].transcription").doesNotExist())
|
||||
.andExpect(jsonPath("$.items[0].filePath").doesNotExist())
|
||||
.andExpect(jsonPath("$.items[0].fileHash").doesNotExist());
|
||||
}
|
||||
|
||||
// ─── /api/documents/search pagination ─────────────────────────────────────
|
||||
|
||||
@Test
|
||||
@@ -1338,4 +1356,16 @@ class DocumentControllerTest {
|
||||
DocumentStatus.REVIEWED,
|
||||
org.raddatz.familienarchiv.tag.TagOperator.AND)));
|
||||
}
|
||||
|
||||
// ─── CSRF protection ──────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
@WithMockUser
|
||||
void post_without_csrf_token_returns_403_CSRF_TOKEN_MISSING() throws Exception {
|
||||
mockMvc.perform(post("/api/documents")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{}"))
|
||||
.andExpect(status().isForbidden())
|
||||
.andExpect(jsonPath("$.code").value(ErrorCode.CSRF_TOKEN_MISSING.name()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
package org.raddatz.familienarchiv.document;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||
import org.raddatz.familienarchiv.audit.AuditLogQueryService;
|
||||
import org.raddatz.familienarchiv.dashboard.DashboardService;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonRepository;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
import org.raddatz.familienarchiv.tag.TagRepository;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Verifies that lazy-loaded associations on {@link Document} are accessible after a service
|
||||
* method returns — i.e. no {@link org.hibernate.LazyInitializationException} is thrown outside
|
||||
* the Hibernate session that loaded the entity.
|
||||
*
|
||||
* <p><b>Known limitation:</b> calling {@code getDocumentById} (or any other service method) from
|
||||
* within an already-open transaction is not covered here. When an outer transaction is active,
|
||||
* the service's own {@code @Transactional} merges into it and Hibernate keeps the same session
|
||||
* open, so the lazy-init guard behaves differently than in a non-transactional caller. This is a
|
||||
* known constraint of the test setup, not a bug in the production code.
|
||||
*/
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE)
|
||||
@ActiveProfiles("test")
|
||||
@Import(PostgresContainerConfig.class)
|
||||
class DocumentLazyLoadingTest {
|
||||
|
||||
@MockitoBean
|
||||
S3Client s3Client;
|
||||
|
||||
@Autowired
|
||||
DocumentRepository documentRepository;
|
||||
|
||||
@Autowired
|
||||
PersonRepository personRepository;
|
||||
|
||||
@Autowired
|
||||
TagRepository tagRepository;
|
||||
|
||||
@Autowired
|
||||
DocumentService documentService;
|
||||
|
||||
@Autowired
|
||||
DashboardService dashboardService;
|
||||
|
||||
@MockitoBean
|
||||
AuditLogQueryService auditLogQueryService;
|
||||
|
||||
@AfterEach
|
||||
void cleanup() {
|
||||
documentRepository.deleteAll();
|
||||
tagRepository.deleteAll();
|
||||
personRepository.deleteAll();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getDocumentById_tagsAndReceiversAccessible_afterReturnFromService() {
|
||||
Person sender = savedPerson("Max", "LzSender");
|
||||
Person receiver = savedPerson("Anna", "LzReceiver");
|
||||
Tag tag = savedTag("LzTag");
|
||||
Document doc = savedDocument("LazyTest", "lazy_test.pdf", sender, Set.of(receiver), Set.of(tag));
|
||||
|
||||
Document result = documentService.getDocumentById(doc.getId());
|
||||
|
||||
// Only the collection access itself is in assertThatCode — guards against LazyInitializationException.
|
||||
// Value assertions live outside so failures surface as AssertionError, not as unexpected exception.
|
||||
assertThatCode(() -> {
|
||||
result.getTags().size();
|
||||
result.getReceivers().size();
|
||||
}).doesNotThrowAnyException();
|
||||
assertThat(result.getTags()).isNotEmpty();
|
||||
result.getTags().forEach(t -> assertThat(t.getName()).isNotNull());
|
||||
assertThat(result.getReceivers()).isNotEmpty();
|
||||
result.getReceivers().forEach(r -> assertThat(r.getLastName()).isNotNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getRecentActivity_collectionsAccessibleAfterReturn() {
|
||||
Person sender = savedPerson("Hans", "RaSender");
|
||||
Tag tag = savedTag("RaTag");
|
||||
for (int i = 0; i < 3; i++) {
|
||||
savedDocument("RaDoc " + i, "ra_doc" + i + ".pdf", sender, Set.of(), Set.of(tag));
|
||||
}
|
||||
|
||||
List<Document> results = documentService.getRecentActivity(3);
|
||||
|
||||
// Access lazy fields inside assertThatCode — guards against LazyInitializationException.
|
||||
// Value assertions live outside so failures surface as AssertionError, not as unexpected exception.
|
||||
assertThatCode(() -> {
|
||||
results.forEach(d -> d.getSender().getLastName());
|
||||
results.forEach(d -> d.getTags().size());
|
||||
}).doesNotThrowAnyException();
|
||||
results.forEach(d -> assertThat(d.getSender()).isNotNull());
|
||||
results.forEach(d -> assertThat(d.getSender().getLastName()).isNotNull());
|
||||
results.forEach(d -> assertThat(d.getTags()).isNotEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_receiverSort_doesNotThrowLazyInitializationException() {
|
||||
Person sender = savedPerson("Hans", "SrSender");
|
||||
Person receiver = savedPerson("Anna", "SrReceiver");
|
||||
Tag tag = savedTag("SrTag");
|
||||
savedDocument("SrDoc", "sr_doc.pdf", sender, Set.of(receiver), Set.of(tag));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.RECEIVER, "asc", null,
|
||||
PageRequest.of(0, 20));
|
||||
assertThat(result.totalElements()).isGreaterThan(0);
|
||||
assertThatCode(() ->
|
||||
result.items().forEach(i -> { if (i.sender() != null) i.sender().getLastName(); }))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_senderSort_doesNotThrowLazyInitializationException() {
|
||||
Person sender = savedPerson("Hans", "SsSender");
|
||||
Tag tag = savedTag("SsTag");
|
||||
savedDocument("SsDoc", "ss_doc.pdf", sender, Set.of(), Set.of(tag));
|
||||
|
||||
assertThatCode(() -> documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.SENDER, "asc", null,
|
||||
PageRequest.of(0, 20)))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void dashboardService_getResume_accessesReceiversViaGetDocumentById_withoutException() {
|
||||
Person sender = savedPerson("Max", "DsSender");
|
||||
Person receiver = savedPerson("Anna", "DsReceiver");
|
||||
Document doc = savedDocument("DashboardTest", "dashboard_test.pdf", sender, Set.of(receiver), Set.of());
|
||||
UUID fakeUserId = UUID.randomUUID();
|
||||
when(auditLogQueryService.findMostRecentDocumentForUser(any())).thenReturn(Optional.of(doc.getId()));
|
||||
when(auditLogQueryService.findRecentContributorsPerDocument(any())).thenReturn(java.util.Map.of());
|
||||
|
||||
assertThatCode(() -> dashboardService.getResume(fakeUserId))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
private Person savedPerson(String firstName, String lastName) {
|
||||
return personRepository.save(Person.builder().firstName(firstName).lastName(lastName).build());
|
||||
}
|
||||
|
||||
private Tag savedTag(String name) {
|
||||
return tagRepository.save(Tag.builder().name(name).build());
|
||||
}
|
||||
|
||||
private Document savedDocument(String title, String filename, Person sender,
|
||||
Set<Person> receivers, Set<Tag> tags) {
|
||||
return documentRepository.save(Document.builder()
|
||||
.title(title).originalFilename(filename)
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.sender(sender)
|
||||
.receivers(new HashSet<>(receivers))
|
||||
.tags(new HashSet<>(tags))
|
||||
.build());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package org.raddatz.familienarchiv.document;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||
import org.raddatz.familienarchiv.audit.AuditLogQueryService;
|
||||
import org.raddatz.familienarchiv.ocr.TrainingLabel;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||
|
||||
/**
|
||||
* AC #2: Document with trainingLabels does not cause LazyInitializationException in search.
|
||||
* AC #3: Detail API still returns trainingLabels after the Document.list graph change.
|
||||
*/
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE)
|
||||
@ActiveProfiles("test")
|
||||
@Import(PostgresContainerConfig.class)
|
||||
class DocumentListItemIntegrationTest {
|
||||
|
||||
@MockitoBean
|
||||
S3Client s3Client;
|
||||
|
||||
@MockitoBean
|
||||
AuditLogQueryService auditLogQueryService;
|
||||
|
||||
@Autowired
|
||||
DocumentRepository documentRepository;
|
||||
|
||||
@Autowired
|
||||
DocumentService documentService;
|
||||
|
||||
@AfterEach
|
||||
void cleanup() {
|
||||
documentRepository.deleteAll();
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_doesNotThrow_whenDocumentHasTrainingLabels() {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("Kurrent Brief")
|
||||
.originalFilename("kurrent.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.trainingLabels(new HashSet<>(Set.of(TrainingLabel.KURRENT_RECOGNITION)))
|
||||
.build());
|
||||
|
||||
assertThatCode(() -> documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(0, 50)))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_returns_list_item_without_sensitive_fields_when_document_has_training_labels() {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("Kurrent Brief")
|
||||
.originalFilename("kurrent2.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.trainingLabels(new HashSet<>(Set.of(TrainingLabel.KURRENT_RECOGNITION)))
|
||||
.build());
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(0, 50));
|
||||
|
||||
assertThat(result.totalElements()).isGreaterThan(0);
|
||||
DocumentListItem item = result.items().get(0);
|
||||
assertThat(item.id()).isNotNull();
|
||||
assertThat(item.title()).isEqualTo("Kurrent Brief");
|
||||
}
|
||||
|
||||
@Test
|
||||
void detail_stillReturnsTrainingLabels() {
|
||||
Document saved = documentRepository.save(Document.builder()
|
||||
.title("Detail Test")
|
||||
.originalFilename("detail_test.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.trainingLabels(new HashSet<>(Set.of(TrainingLabel.KURRENT_RECOGNITION)))
|
||||
.build());
|
||||
|
||||
// Document.full entity graph (used by getDocumentById) must still load trainingLabels
|
||||
Document loaded = documentService.getDocumentById(saved.getId());
|
||||
|
||||
assertThat(loaded.getTrainingLabels()).containsExactly(TrainingLabel.KURRENT_RECOGNITION);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,9 @@
|
||||
package org.raddatz.familienarchiv.document;
|
||||
|
||||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.persistence.EntityManagerFactory;
|
||||
import org.hibernate.SessionFactory;
|
||||
import org.hibernate.stat.Statistics;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||
import org.raddatz.familienarchiv.config.FlywayConfig;
|
||||
@@ -21,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.jdbc.test.autoconfigure.AutoConfigureTestDatabase;
|
||||
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.data.jpa.domain.Specification;
|
||||
|
||||
import org.springframework.data.domain.Page;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
@@ -55,6 +60,12 @@ class DocumentRepositoryTest {
|
||||
@Autowired
|
||||
private TranscriptionBlockRepository transcriptionBlockRepository;
|
||||
|
||||
@Autowired
|
||||
private EntityManagerFactory entityManagerFactory;
|
||||
|
||||
@Autowired
|
||||
private EntityManager entityManager;
|
||||
|
||||
// ─── save and findById ────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
@@ -490,6 +501,117 @@ class DocumentRepositoryTest {
|
||||
assertThat(ids).containsExactlyInAnyOrder(grandparent.getId(), parent2.getId(), child2.getId());
|
||||
}
|
||||
|
||||
// ─── query-count — entity-graph assertions ────────────────────────────────
|
||||
|
||||
@Test
|
||||
void findAll_withSpecAndPageable_loadsDocumentsInAtMostFiveStatements() {
|
||||
Person sender = personRepository.save(Person.builder().firstName("Hans").lastName("QcSender").build());
|
||||
Person receiver = personRepository.save(Person.builder().firstName("Anna").lastName("QcReceiver").build());
|
||||
Tag tag = tagRepository.save(Tag.builder().name("QcTag").build());
|
||||
for (int i = 0; i < 10; i++) {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("QcDoc " + i).originalFilename("qcdoc" + i + ".pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.sender(sender)
|
||||
.receivers(new HashSet<>(Set.of(receiver)))
|
||||
.tags(new HashSet<>(Set.of(tag)))
|
||||
.build());
|
||||
}
|
||||
entityManager.flush();
|
||||
entityManager.clear();
|
||||
Statistics stats = entityManagerFactory.unwrap(SessionFactory.class).getStatistics();
|
||||
stats.setStatisticsEnabled(true);
|
||||
stats.clear();
|
||||
|
||||
Specification<Document> allDocs = (root, query, cb) -> null;
|
||||
documentRepository.findAll(allDocs, PageRequest.of(0, 10));
|
||||
|
||||
assertThat(stats.getPrepareStatementCount())
|
||||
.as("@EntityGraph(Document.list) must load 10 docs in ≤5 statements, not N+1")
|
||||
.isLessThanOrEqualTo(5);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findById_loadsSenderReceiversAndTagsInAtMostTwoStatements() {
|
||||
Person sender = personRepository.save(Person.builder().firstName("Max").lastName("FbSender").build());
|
||||
Set<Person> receivers = new HashSet<>();
|
||||
for (int i = 0; i < 3; i++) {
|
||||
receivers.add(personRepository.save(
|
||||
Person.builder().firstName("R" + i).lastName("FbReceiver").build()));
|
||||
}
|
||||
Set<Tag> tags = new HashSet<>();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
tags.add(tagRepository.save(Tag.builder().name("FbTag" + i).build()));
|
||||
}
|
||||
Document doc = documentRepository.save(Document.builder()
|
||||
.title("FindByIdQc").originalFilename("findbyid_qc.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.sender(sender).receivers(receivers).tags(tags)
|
||||
.build());
|
||||
entityManager.flush();
|
||||
entityManager.clear();
|
||||
Statistics stats = entityManagerFactory.unwrap(SessionFactory.class).getStatistics();
|
||||
stats.setStatisticsEnabled(true);
|
||||
stats.clear();
|
||||
|
||||
documentRepository.findById(doc.getId());
|
||||
|
||||
assertThat(stats.getPrepareStatementCount())
|
||||
.as("@EntityGraph(Document.full) must load sender+receivers+tags in ≤2 statements, not 4")
|
||||
.isLessThanOrEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll_withPageable_loadsSenderWithoutNPlusOne() {
|
||||
Person sender = personRepository.save(Person.builder().firstName("Maria").lastName("RaSender").build());
|
||||
Tag tag = tagRepository.save(Tag.builder().name("RaTag2").build());
|
||||
for (int i = 0; i < 5; i++) {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("RaDoc2 " + i).originalFilename("radoc2_" + i + ".pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.sender(sender)
|
||||
.tags(new HashSet<>(Set.of(tag)))
|
||||
.build());
|
||||
}
|
||||
entityManager.flush();
|
||||
entityManager.clear();
|
||||
Statistics stats = entityManagerFactory.unwrap(SessionFactory.class).getStatistics();
|
||||
stats.setStatisticsEnabled(true);
|
||||
stats.clear();
|
||||
|
||||
documentRepository.findAll(PageRequest.of(0, 5, Sort.by(Sort.Direction.DESC, "updatedAt")));
|
||||
|
||||
assertThat(stats.getPrepareStatementCount())
|
||||
.as("@EntityGraph(Document.list) via findAll(Pageable) must not N+1 sender for 5 docs")
|
||||
.isLessThanOrEqualTo(5);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll_withSpecOnly_appliesEntityGraphInAtMostFiveStatements() {
|
||||
Person sender = personRepository.save(Person.builder().firstName("Otto").lastName("SoSender").build());
|
||||
Tag tag = tagRepository.save(Tag.builder().name("SoTag").build());
|
||||
for (int i = 0; i < 5; i++) {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("SoDoc " + i).originalFilename("sodoc_" + i + ".pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.sender(sender)
|
||||
.tags(new HashSet<>(Set.of(tag)))
|
||||
.build());
|
||||
}
|
||||
entityManager.flush();
|
||||
entityManager.clear();
|
||||
Statistics stats = entityManagerFactory.unwrap(SessionFactory.class).getStatistics();
|
||||
stats.setStatisticsEnabled(true);
|
||||
stats.clear();
|
||||
|
||||
Specification<Document> allDocs = (root, query, cb) -> null;
|
||||
documentRepository.findAll(allDocs);
|
||||
|
||||
assertThat(stats.getPrepareStatementCount())
|
||||
.as("@EntityGraph(Document.list) via findAll(Spec) must not N+1 sender for 5 docs")
|
||||
.isLessThanOrEqualTo(5);
|
||||
}
|
||||
|
||||
// ─── seeding helpers ─────────────────────────────────────────────────────
|
||||
|
||||
private Document uploaded(String title) {
|
||||
|
||||
@@ -125,10 +125,10 @@ class DocumentSearchPagedIntegrationTest {
|
||||
|
||||
// No document id should appear on both pages — slicing must be exclusive.
|
||||
var idsOnPage0 = page0.items().stream()
|
||||
.map(item -> item.document().getId())
|
||||
.map(item -> item.id())
|
||||
.toList();
|
||||
var idsOnPage1 = page1.items().stream()
|
||||
.map(item -> item.document().getId())
|
||||
.map(item -> item.id())
|
||||
.toList();
|
||||
for (UUID id : idsOnPage0) {
|
||||
assertThat(idsOnPage1).doesNotContain(id);
|
||||
|
||||
@@ -3,8 +3,6 @@ package org.raddatz.familienarchiv.document;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.audit.ActivityActorDTO;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.springframework.data.domain.PageRequest;
|
||||
|
||||
import java.util.List;
|
||||
@@ -14,14 +12,11 @@ import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class DocumentSearchResultTest {
|
||||
|
||||
private DocumentSearchItem item(UUID docId) {
|
||||
Document doc = Document.builder()
|
||||
.id(docId)
|
||||
.title("Test")
|
||||
.originalFilename("test.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.build();
|
||||
return new DocumentSearchItem(doc, SearchMatchData.empty(), 0, List.of());
|
||||
private DocumentListItem item(UUID docId) {
|
||||
return new DocumentListItem(
|
||||
docId, "Test", "test.pdf", null, null, null,
|
||||
List.of(), List.of(), null, null, null, null,
|
||||
0, List.of(), SearchMatchData.empty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -45,7 +40,7 @@ class DocumentSearchResultTest {
|
||||
|
||||
@Test
|
||||
void paged_factory_populates_paging_fields_from_pageable_and_total() {
|
||||
List<DocumentSearchItem> slice = List.of(item(UUID.randomUUID()), item(UUID.randomUUID()));
|
||||
List<DocumentListItem> slice = List.of(item(UUID.randomUUID()), item(UUID.randomUUID()));
|
||||
|
||||
DocumentSearchResult result = DocumentSearchResult.paged(slice, PageRequest.of(1, 50), 120L);
|
||||
|
||||
@@ -68,9 +63,10 @@ class DocumentSearchResultTest {
|
||||
void of_exposes_items_with_completion_and_contributors() {
|
||||
UUID id = UUID.randomUUID();
|
||||
ActivityActorDTO actor = new ActivityActorDTO("AB", "#f00", "Anna Braun");
|
||||
Document doc = Document.builder().id(id).title("T").originalFilename("t.pdf")
|
||||
.status(DocumentStatus.UPLOADED).build();
|
||||
DocumentSearchItem item = new DocumentSearchItem(doc, SearchMatchData.empty(), 75, List.of(actor));
|
||||
DocumentListItem item = new DocumentListItem(
|
||||
id, "T", "t.pdf", null, null, null,
|
||||
List.of(), List.of(), null, null, null, null,
|
||||
75, List.of(actor), SearchMatchData.empty());
|
||||
|
||||
DocumentSearchResult result = DocumentSearchResult.of(List.of(item));
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ class DocumentServiceSortTest {
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.DATE, "DESC", null, PAGE);
|
||||
|
||||
assertThat(result.items()).hasSize(2);
|
||||
assertThat(result.items().get(0).document().getId()).isEqualTo(id2); // newer first
|
||||
assertThat(result.items().get(0).id()).isEqualTo(id2); // newer first
|
||||
}
|
||||
|
||||
// ─── RELEVANCE sort — pure text (no filters) ──────────────────────────────
|
||||
@@ -104,7 +104,7 @@ class DocumentServiceSortTest {
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, PAGE);
|
||||
|
||||
assertThat(result.items().get(0).document().getId()).isEqualTo(id1);
|
||||
assertThat(result.items().get(0).id()).isEqualTo(id1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -121,7 +121,7 @@ class DocumentServiceSortTest {
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null, null, null, null, PAGE);
|
||||
|
||||
assertThat(result.items().get(0).document().getId()).isEqualTo(id1);
|
||||
assertThat(result.items().get(0).id()).isEqualTo(id1);
|
||||
}
|
||||
|
||||
// ─── RELEVANCE sort — overflow guard ─────────────────────────────────────
|
||||
@@ -156,7 +156,7 @@ class DocumentServiceSortTest {
|
||||
DocumentSort.RELEVANCE, null, null, PAGE);
|
||||
|
||||
assertThat(result.items()).hasSize(1);
|
||||
assertThat(result.items().get(0).document().getId()).isEqualTo(uuidId);
|
||||
assertThat(result.items().get(0).id()).isEqualTo(uuidId);
|
||||
}
|
||||
|
||||
// ─── RELEVANCE sort — text + active filter ────────────────────────────────
|
||||
|
||||
@@ -11,7 +11,7 @@ import org.raddatz.familienarchiv.audit.AuditLogQueryService;
|
||||
import org.raddatz.familienarchiv.audit.AuditService;
|
||||
import org.raddatz.familienarchiv.document.annotation.AnnotationService;
|
||||
import org.raddatz.familienarchiv.document.transcription.TranscriptionBlockQueryService;
|
||||
import org.raddatz.familienarchiv.document.DocumentSearchItem;
|
||||
import org.raddatz.familienarchiv.document.DocumentListItem;
|
||||
import org.raddatz.familienarchiv.document.DocumentSearchResult;
|
||||
import org.raddatz.familienarchiv.document.DocumentSort;
|
||||
import org.raddatz.familienarchiv.document.DocumentUpdateDTO;
|
||||
@@ -1444,7 +1444,7 @@ class DocumentServiceTest {
|
||||
assertThat(result.totalPages()).isEqualTo(3);
|
||||
assertThat(result.items()).hasSize(50);
|
||||
// Page 1 (offset 50) under ascending sender sort should start at L050
|
||||
assertThat(result.items().get(0).document().getSender().getLastName()).isEqualTo("L050");
|
||||
assertThat(result.items().get(0).sender().getLastName()).isEqualTo("L050");
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -1565,7 +1565,7 @@ class DocumentServiceTest {
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, UNPAGED);
|
||||
|
||||
assertThat(result.items()).hasSize(2);
|
||||
assertThat(result.items()).extracting(item -> item.document().getTitle()).containsExactly("Has Sender", "No Sender");
|
||||
assertThat(result.items()).extracting(DocumentListItem::title).containsExactly("Has Sender", "No Sender");
|
||||
}
|
||||
|
||||
// ─── searchDocuments — RECEIVER sort, empty receivers ───────────────────────
|
||||
@@ -1584,7 +1584,7 @@ class DocumentServiceTest {
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null, DocumentSort.RECEIVER, "asc", null, UNPAGED);
|
||||
|
||||
assertThat(result.items()).extracting(item -> item.document().getTitle())
|
||||
assertThat(result.items()).extracting(DocumentListItem::title)
|
||||
.containsExactly("Has Receiver", "No Receivers");
|
||||
}
|
||||
|
||||
@@ -1607,7 +1607,7 @@ class DocumentServiceTest {
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, UNPAGED);
|
||||
|
||||
// null lastName should sort to end (treated as empty), not before "smith" (as "null")
|
||||
assertThat(result.items()).extracting(item -> item.document().getTitle())
|
||||
assertThat(result.items()).extracting(DocumentListItem::title)
|
||||
.containsExactly("smith doc", "Null lastname doc");
|
||||
}
|
||||
|
||||
|
||||
@@ -135,7 +135,7 @@ class MassImportServiceTest {
|
||||
@Test
|
||||
void runImportAsync_throwsConflict_whenAlreadyRunning() {
|
||||
MassImportService.ImportStatus running = new MassImportService.ImportStatus(
|
||||
MassImportService.State.RUNNING, "IMPORT_RUNNING", "Running...", 0, LocalDateTime.now());
|
||||
MassImportService.State.RUNNING, "IMPORT_RUNNING", "Running...", 0, List.of(), LocalDateTime.now());
|
||||
ReflectionTestUtils.setField(service, "currentStatus", running);
|
||||
|
||||
assertThatThrownBy(() -> service.runImportAsync())
|
||||
@@ -154,9 +154,76 @@ class MassImportServiceTest {
|
||||
.build();
|
||||
when(documentService.findByOriginalFilename("doc001.pdf")).thenReturn(Optional.of(existing));
|
||||
|
||||
service.importSingleDocument(minimalCells("doc001.pdf"), Optional.empty(), "doc001.pdf", "doc001");
|
||||
Optional<MassImportService.SkipReason> result = service.importSingleDocument(minimalCells("doc001.pdf"), Optional.empty(), "doc001.pdf", "doc001");
|
||||
|
||||
verify(documentService, never()).save(any());
|
||||
assertThat(result).isPresent().contains(MassImportService.SkipReason.ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — already-exists guard fires before file I/O ─────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_skipsWithAlreadyExists_whenDocumentUploadedAndFileIsPresent(@TempDir Path tempDir) throws Exception {
|
||||
// Document already exists with status UPLOADED (not PLACEHOLDER).
|
||||
// A physical PDF file is also present on disk (valid magic bytes).
|
||||
// Expected: ALREADY_EXISTS is returned and no S3 upload is attempted —
|
||||
// the guard fires before any file I/O, so no partial processing occurs.
|
||||
Document existing = Document.builder()
|
||||
.id(UUID.randomUUID())
|
||||
.originalFilename("present.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.build();
|
||||
when(documentService.findByOriginalFilename("present.pdf")).thenReturn(Optional.of(existing));
|
||||
|
||||
Path physicalFile = tempDir.resolve("present.pdf");
|
||||
byte[] pdfHeader = {0x25, 0x50, 0x44, 0x46, 0x2D}; // %PDF-
|
||||
Files.write(physicalFile, pdfHeader);
|
||||
|
||||
Optional<MassImportService.SkipReason> result = service.importSingleDocument(
|
||||
minimalCells("present.pdf"), Optional.of(physicalFile.toFile()), "present.pdf", "present");
|
||||
|
||||
assertThat(result).isPresent().contains(MassImportService.SkipReason.ALREADY_EXISTS);
|
||||
verify(s3Client, never()).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
verify(documentService, never()).save(any());
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — S3 failure surfaced in skippedFiles ──────────
|
||||
|
||||
@Test
|
||||
void runImportAsync_addsS3UploadFailed_toSkippedFiles_whenS3Throws(@TempDir Path tempDir) throws Exception {
|
||||
byte[] pdfHeader = {0x25, 0x50, 0x44, 0x46, 0x2D}; // %PDF-
|
||||
Files.write(tempDir.resolve("upload_fail.pdf"), pdfHeader);
|
||||
buildMinimalImportXlsx(tempDir, "upload_fail.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("upload_fail.pdf")).thenReturn(Optional.empty());
|
||||
doThrow(new RuntimeException("S3 unavailable"))
|
||||
.when(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skipped()).isEqualTo(1);
|
||||
assertThat(service.getStatus().skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::filename, MassImportService.SkippedFile::reason)
|
||||
.containsExactly(org.assertj.core.groups.Tuple.tuple("upload_fail.pdf", MassImportService.SkipReason.S3_UPLOAD_FAILED));
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_addsAlreadyExists_toSkippedFiles_whenDocumentAlreadyUploaded(@TempDir Path tempDir) throws Exception {
|
||||
buildMinimalImportXlsx(tempDir, "existing.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
Document existing = Document.builder()
|
||||
.id(UUID.randomUUID())
|
||||
.originalFilename("existing.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.build();
|
||||
when(documentService.findByOriginalFilename("existing.pdf")).thenReturn(Optional.of(existing));
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skipped()).isEqualTo(1);
|
||||
assertThat(service.getStatus().skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::reason)
|
||||
.containsExactly(MassImportService.SkipReason.ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — create new document (metadata only) ───────────
|
||||
@@ -208,7 +275,7 @@ class MassImportServiceTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void importSingleDocument_returnsEarly_whenS3UploadFails(@TempDir Path tempDir) throws Exception {
|
||||
void importSingleDocument_returnsS3UploadFailed_whenS3UploadFails(@TempDir Path tempDir) throws Exception {
|
||||
Path tempFile = tempDir.resolve("fail.pdf");
|
||||
Files.write(tempFile, "data".getBytes());
|
||||
|
||||
@@ -216,10 +283,11 @@ class MassImportServiceTest {
|
||||
doThrow(new RuntimeException("S3 error"))
|
||||
.when(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
|
||||
service.importSingleDocument(
|
||||
Optional<MassImportService.SkipReason> result = service.importSingleDocument(
|
||||
minimalCells("fail.pdf"), Optional.of(tempFile.toFile()), "fail.pdf", "fail");
|
||||
|
||||
verify(documentService, never()).save(any());
|
||||
assertThat(result).isPresent().contains(MassImportService.SkipReason.S3_UPLOAD_FAILED);
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — sender handling ───────────────────────────────
|
||||
@@ -325,8 +393,8 @@ class MassImportServiceTest {
|
||||
@Test
|
||||
void processRows_returnsZero_whenOnlyHeaderRow() {
|
||||
List<List<String>> rows = List.of(List.of("header", "col1"));
|
||||
Integer result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
assertThat(result).isEqualTo(0);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
assertThat(result.processed()).isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -335,8 +403,8 @@ class MassImportServiceTest {
|
||||
List.of("header"),
|
||||
minimalCells("") // blank index
|
||||
);
|
||||
Integer result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
assertThat(result).isEqualTo(0);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
assertThat(result.processed()).isEqualTo(0);
|
||||
verify(documentService, never()).findByOriginalFilename(any());
|
||||
}
|
||||
|
||||
@@ -349,9 +417,9 @@ class MassImportServiceTest {
|
||||
List.of("header"),
|
||||
minimalCells("doc001") // no dot → appends ".pdf"
|
||||
);
|
||||
Integer result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
|
||||
assertThat(result).isEqualTo(1);
|
||||
assertThat(result.processed()).isEqualTo(1);
|
||||
verify(documentService).findByOriginalFilename("doc001.pdf");
|
||||
}
|
||||
|
||||
@@ -364,12 +432,116 @@ class MassImportServiceTest {
|
||||
List.of("header"),
|
||||
minimalCells("doc002.pdf") // has dot → used as-is
|
||||
);
|
||||
Integer result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
|
||||
assertThat(result).isEqualTo(1);
|
||||
assertThat(result.processed()).isEqualTo(1);
|
||||
verify(documentService).findByOriginalFilename("doc002.pdf");
|
||||
}
|
||||
|
||||
// ─── isValidImportFilename — security regression — do not remove ─────────
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameIsNull() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", (String) null);
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameIsBlank() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", " ");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsForwardSlash() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "etc/passwd");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsBackslash() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "..\\etc\\passwd");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsDotDot() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "doc..evil.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameIsDotDot() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "..");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameIsAbsolutePath() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "/etc/passwd");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsNullByte() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "file\0.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsTrue_whenFilenameIsPlainBasename() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "document.pdf");
|
||||
assertThat(result).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsUnicodeDivisionSlash() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "foo∕bar.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsFullwidthSlash() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "foo/bar.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsUnicodeReverseSolidus() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "foo⧵bar.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsTrue_whenFilenameHasLeadingDot() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", ".hidden.pdf");
|
||||
assertThat(result).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsTrue_whenFilenameHasSpaces() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "Brief an Oma.pdf");
|
||||
assertThat(result).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void processRows_skipsRowAndContinues_whenFilenameIsPathTraversal() {
|
||||
when(documentService.findByOriginalFilename("legitimate.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<List<String>> rows = List.of(
|
||||
List.of("header"),
|
||||
minimalCells("../evil"), // row 1: path traversal — should be skipped
|
||||
minimalCells("legitimate.pdf") // row 2: valid — should be processed
|
||||
);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
|
||||
assertThat(result.processed()).isEqualTo(1);
|
||||
assertThat(result.skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::reason)
|
||||
.containsExactly(MassImportService.SkipReason.INVALID_FILENAME_PATH_TRAVERSAL);
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — non-blank optional fields ────────────────────
|
||||
|
||||
@Test
|
||||
@@ -525,6 +697,82 @@ class MassImportServiceTest {
|
||||
assertThat(result).isEqualTo("hello");
|
||||
}
|
||||
|
||||
// ─── PDF magic byte validation regression ─────────────────────────────────
|
||||
|
||||
@Test
|
||||
void runImportAsync_uploadsValidPdf_andSkipsFakeOne(@TempDir Path tempDir) throws Exception {
|
||||
setupOneValidOneFakeImport(tempDir);
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
verify(s3Client, times(1)).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_setsSkippedCount_toOne_whenOneFakeFile(@TempDir Path tempDir) throws Exception {
|
||||
setupOneValidOneFakeImport(tempDir);
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skipped()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_includesRejectedFilename_inSkippedFiles(@TempDir Path tempDir) throws Exception {
|
||||
setupOneValidOneFakeImport(tempDir);
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::filename)
|
||||
.contains("fake.pdf");
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_skipsFile_whenShorterThanFourBytes(@TempDir Path tempDir) throws Exception {
|
||||
Files.write(tempDir.resolve("tiny.pdf"), new byte[]{0x25, 0x50, 0x44}); // only 3 bytes
|
||||
buildMinimalImportXlsx(tempDir, "tiny.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
lenient().when(documentService.findByOriginalFilename(any())).thenReturn(Optional.empty());
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skipped()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_skipsFile_whenMagicBytesCheckThrowsIOException(@TempDir Path tempDir) throws Exception {
|
||||
Files.writeString(tempDir.resolve("unreadable.pdf"), "some content");
|
||||
buildMinimalImportXlsx(tempDir, "unreadable.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
lenient().when(documentService.findByOriginalFilename(any())).thenReturn(Optional.empty());
|
||||
|
||||
MassImportService spyService = spy(service);
|
||||
doThrow(new java.io.IOException("simulated read error")).when(spyService).openFileStream(any(File.class));
|
||||
|
||||
spyService.runImportAsync();
|
||||
|
||||
assertThat(spyService.getStatus().skipped()).isEqualTo(1);
|
||||
assertThat(spyService.getStatus().skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::reason)
|
||||
.containsExactly(MassImportService.SkipReason.FILE_READ_ERROR);
|
||||
}
|
||||
|
||||
// ─── findFileRecursive — symlink escape security regression — do not remove ─
|
||||
|
||||
@Test
|
||||
void findFileRecursive_throwsDomainException_whenSymlinkEscapesImportDir(
|
||||
@TempDir Path importDirPath, @TempDir Path outsideDir) throws Exception {
|
||||
Path outsideFile = outsideDir.resolve("secret.pdf");
|
||||
Files.writeString(outsideFile, "sensitive content");
|
||||
Files.createSymbolicLink(importDirPath.resolve("secret.pdf"), outsideFile);
|
||||
|
||||
ReflectionTestUtils.setField(service, "importDir", importDirPath.toString());
|
||||
|
||||
assertThatThrownBy(() -> ReflectionTestUtils.invokeMethod(service, "findFileRecursive", "secret.pdf"))
|
||||
.isInstanceOf(DomainException.class);
|
||||
}
|
||||
|
||||
// ─── readOds — XXE security regression ───────────────────────────────────
|
||||
|
||||
// Security regression — do not remove.
|
||||
@@ -621,4 +869,28 @@ class MassImportServiceTest {
|
||||
}
|
||||
return destination.toFile();
|
||||
}
|
||||
|
||||
private void setupOneValidOneFakeImport(Path tempDir) throws Exception {
|
||||
byte[] pdfHeader = {0x25, 0x50, 0x44, 0x46, 0x2D}; // %PDF-
|
||||
Files.write(tempDir.resolve("real.pdf"), pdfHeader);
|
||||
Files.writeString(tempDir.resolve("fake.pdf"), "not a pdf");
|
||||
buildMinimalImportXlsx(tempDir, "real.pdf", "fake.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename(any())).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
}
|
||||
|
||||
private void buildMinimalImportXlsx(Path dir, String... filenames) throws Exception {
|
||||
Path xlsx = dir.resolve("import.xlsx");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
org.apache.poi.ss.usermodel.Sheet sheet = wb.createSheet("Sheet1");
|
||||
sheet.createRow(0).createCell(0).setCellValue("Index");
|
||||
for (int i = 0; i < filenames.length; i++) {
|
||||
sheet.createRow(i + 1).createCell(0).setCellValue(filenames[i]);
|
||||
}
|
||||
try (OutputStream out = Files.newOutputStream(xlsx)) {
|
||||
wb.write(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ class AdminControllerTest {
|
||||
@WithMockUser(authorities = "ADMIN")
|
||||
void importStatus_returns200_withStatusCode_whenAdmin() throws Exception {
|
||||
MassImportService.ImportStatus status = new MassImportService.ImportStatus(
|
||||
MassImportService.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, null);
|
||||
MassImportService.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
when(massImportService.getStatus()).thenReturn(status);
|
||||
|
||||
mockMvc.perform(get("/api/admin/import-status"))
|
||||
@@ -61,7 +61,7 @@ class AdminControllerTest {
|
||||
@WithMockUser(authorities = "ADMIN")
|
||||
void importStatus_messageField_notPresentInApiResponse() throws Exception {
|
||||
MassImportService.ImportStatus status = new MassImportService.ImportStatus(
|
||||
MassImportService.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, null);
|
||||
MassImportService.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
when(massImportService.getStatus()).thenReturn(status);
|
||||
|
||||
mockMvc.perform(get("/api/admin/import-status"))
|
||||
|
||||
@@ -20,6 +20,8 @@ import org.springframework.test.web.servlet.MockMvc;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
|
||||
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
|
||||
@@ -178,7 +180,7 @@ class UserControllerTest {
|
||||
.content("{\"currentPassword\":\"old\",\"newPassword\":\"new123!\"}"))
|
||||
.andExpect(status().isNoContent());
|
||||
|
||||
org.mockito.Mockito.verify(authService).revokeOtherSessions(any(), org.mockito.ArgumentMatchers.eq("user@example.com"));
|
||||
verify(authService).revokeOtherSessions(any(), eq("user@example.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -189,6 +191,16 @@ class UserControllerTest {
|
||||
.andExpect(status().isUnauthorized());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(username = "user@example.com")
|
||||
void changePassword_without_csrf_returns_403_CSRF_TOKEN_MISSING() throws Exception {
|
||||
mockMvc.perform(post("/api/users/me/password")
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.content("{\"currentPassword\":\"old\",\"newPassword\":\"new123!\"}"))
|
||||
.andExpect(status().isForbidden())
|
||||
.andExpect(jsonPath("$.code").value("CSRF_TOKEN_MISSING"));
|
||||
}
|
||||
|
||||
// ─── POST /api/users/{id}/force-logout ────────────────────────────────────
|
||||
|
||||
@Test
|
||||
@@ -230,4 +242,12 @@ class UserControllerTest {
|
||||
mockMvc.perform(post("/api/users/" + targetId + "/force-logout").with(csrf()))
|
||||
.andExpect(status().isNotFound());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(username = "admin@example.com", authorities = "ADMIN_USER")
|
||||
void forceLogout_without_csrf_returns_403_CSRF_TOKEN_MISSING() throws Exception {
|
||||
mockMvc.perform(post("/api/users/" + UUID.randomUUID() + "/force-logout"))
|
||||
.andExpect(status().isForbidden())
|
||||
.andExpect(jsonPath("$.code").value("CSRF_TOKEN_MISSING"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,2 +1,8 @@
|
||||
logging.level.root=WARN
|
||||
logging.level.org.raddatz=INFO
|
||||
|
||||
# Default test value so FlywayConfig's fail-closed check passes without each
|
||||
# test having to set GRAFANA_DB_PASSWORD explicitly. The actual value is
|
||||
# irrelevant in tests — Flyway only uses it to set the grafana_reader role's
|
||||
# password, which no test connects with.
|
||||
GRAFANA_DB_PASSWORD=test-grafana-reader-password
|
||||
|
||||
@@ -147,6 +147,9 @@ services:
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-changeme}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL:-http://localhost:3003}
|
||||
# Read-only password for the grafana_reader PostgreSQL role; interpolated
|
||||
# into the provisioned PostgreSQL datasource (see datasources.yml).
|
||||
GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD}
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
@@ -165,6 +168,7 @@ services:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- obs-net
|
||||
- archiv-net # PO Overview dashboard queries archive-db via the grafana_reader role
|
||||
|
||||
# --- Error Tracking: GlitchTip ---
|
||||
|
||||
|
||||
@@ -227,6 +227,9 @@ services:
|
||||
SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/archiv
|
||||
SPRING_DATASOURCE_USERNAME: archiv
|
||||
SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
# Consumed by Flyway V68 via the ${grafanaDbPassword} placeholder to set
|
||||
# the read-only grafana_reader role's password.
|
||||
GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD}
|
||||
# Application uses the bucket-scoped service account, not MinIO root.
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_ACCESS_KEY: archiv-app
|
||||
@@ -252,6 +255,8 @@ services:
|
||||
OTEL_METRICS_EXPORTER: none
|
||||
MANAGEMENT_METRICS_TAGS_APPLICATION: Familienarchiv
|
||||
MANAGEMENT_TRACING_SAMPLING_PROBABILITY: ${MANAGEMENT_TRACING_SAMPLING_PROBABILITY:-0.1}
|
||||
SENTRY_DSN: ${SENTRY_DSN:-}
|
||||
LOGGING_STRUCTURED_FORMAT_CONSOLE: ecs
|
||||
networks:
|
||||
- archiv-net
|
||||
healthcheck:
|
||||
@@ -266,6 +271,10 @@ services:
|
||||
build:
|
||||
context: ./frontend
|
||||
target: production
|
||||
args:
|
||||
# Vite build-time variable — baked into the JS bundle at build time.
|
||||
# Empty default so deploys succeed before the secret is configured.
|
||||
VITE_SENTRY_DSN: ${VITE_SENTRY_DSN:-}
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
backend:
|
||||
@@ -276,6 +285,9 @@ services:
|
||||
# SSR fetches go inside the docker network; clients hit https://${APP_DOMAIN}
|
||||
API_INTERNAL_URL: http://backend:8080
|
||||
ORIGIN: https://${APP_DOMAIN}
|
||||
# Enforce upload size limit in the adapter-node layer (fixes GHSA-2crg-3p73-43xp bypass).
|
||||
# Must be ≤ client_max_body_size in the Caddy reverse proxy to avoid 413 mismatches.
|
||||
BODY_SIZE_LIMIT: 50M
|
||||
networks:
|
||||
- archiv-net
|
||||
healthcheck:
|
||||
|
||||
@@ -163,6 +163,9 @@ services:
|
||||
SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${POSTGRES_DB}
|
||||
SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER}
|
||||
SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
# Consumed by Flyway V68 via the ${grafanaDbPassword} placeholder to set
|
||||
# the read-only grafana_reader role's password.
|
||||
GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD}
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_ACCESS_KEY: ${MINIO_ROOT_USER}
|
||||
S3_SECRET_KEY: ${MINIO_ROOT_PASSWORD}
|
||||
@@ -228,6 +231,9 @@ services:
|
||||
API_INTERNAL_URL: http://backend:8080
|
||||
# Vite dev proxy forwards /api from browser to the backend container
|
||||
API_PROXY_TARGET: http://backend:8080
|
||||
# Upload size limit for adapter-node (production target). Not enforced by Vite dev server
|
||||
# but kept here to match docker-compose.prod.yml and prevent config drift.
|
||||
BODY_SIZE_LIMIT: 50M
|
||||
ports:
|
||||
- "${PORT_FRONTEND}:5173"
|
||||
networks:
|
||||
|
||||
@@ -63,7 +63,7 @@ Members of the cross-cutting layer have no entity of their own, no user-facing C
|
||||
| `audit` | Append-only event store (`audit_log`) for all domain mutations. Feeds the activity feed and Family Pulse dashboard. | Consumed by 5+ domains; no user-facing CRUD of its own |
|
||||
| `config` | Infrastructure bean definitions: `MinioConfig`, `AsyncConfig`, `WebConfig` | Framework infra; no business logic |
|
||||
| `dashboard` | Stats aggregation for the admin dashboard and Family Pulse widget | Aggregates from 3+ domains; no owned entities |
|
||||
| `exception` | `DomainException`, `ErrorCode` enum, `GlobalExceptionHandler` | Framework infra; consumed by every controller and service. Adding a new `ErrorCode` requires matching updates in `frontend/src/lib/shared/errors.ts` and all three `messages/*.json` locale files. |
|
||||
| `exception` | `DomainException`, `ErrorCode` enum, `GlobalExceptionHandler` | Framework infra; consumed by every controller and service. Adding a new `ErrorCode` requires matching updates in `frontend/src/lib/shared/errors.ts` and all three `messages/*.json` locale files. Current security-related codes: `CSRF_TOKEN_MISSING` (403 on mutating request without valid `X-XSRF-TOKEN` header), `TOO_MANY_LOGIN_ATTEMPTS` (429 when login rate limit exceeded). |
|
||||
| `filestorage` | `FileService` — MinIO/S3 upload, download, presigned-URL generation | Generic service; consumed by `document` and `ocr` |
|
||||
| `importing` | `MassImportService` — async ODS/Excel batch import | Orchestrates across `person`, `tag`, `document` |
|
||||
| `security` | `SecurityConfig`, `Permission` enum, `@RequirePermission` annotation, `PermissionAspect` (AOP) | Framework infra; enforced globally across all controllers |
|
||||
@@ -117,7 +117,7 @@ Controllers never call repositories directly. Services never reach into another
|
||||
### Permission system
|
||||
Permissions are enforced via `@RequirePermission(Permission.X)` on controller methods, checked at runtime by `PermissionAspect` (Spring AOP). The `Permission` enum defines the available capabilities (`READ_ALL`, `WRITE_ALL`, `ADMIN`, `ADMIN_USER`, `ADMIN_TAG`, `ADMIN_PERMISSION`, `ANNOTATE_ALL`, `BLOG_WRITE`). This is not Spring Security's `@PreAuthorize` — do not mix the two mechanisms.
|
||||
|
||||
Sessions use a Base64-encoded Basic Auth token stored in an `httpOnly`, `SameSite=strict` cookie (`auth_token`, maxAge=86400 s). CSRF protection is disabled because this cookie configuration structurally prevents cross-origin credential theft. See [docs/security-guide.md](security-guide.md) for the full security reference.
|
||||
Sessions use a Spring Session JDBC-backed cookie (`fa_session`, `httpOnly`, `SameSite=strict`, maxAge=86400 s). CSRF protection uses the double-submit cookie pattern: Spring Security sets an `XSRF-TOKEN` cookie (readable by JS); SvelteKit's `handleFetch` injects the value as `X-XSRF-TOKEN` on every mutating request; a missing or mismatched token returns `403 CSRF_TOKEN_MISSING`. See [ADR-022](adr/022-csrf-session-revocation-rate-limiting.md) and [docs/security-guide.md](security-guide.md) for the full security reference.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -152,6 +152,7 @@ All vars are set in `.env` at the repo root (copy from `.env.example`). The back
|
||||
| `PORT_GRAFANA` | Host port for the Grafana UI (bound to `127.0.0.1` only) | `3003` | — | — |
|
||||
| `POSTGRES_HOST` | PostgreSQL hostname for GlitchTip's db-init job and workers. Override when only the staging stack is running and `archive-db` is not resolvable by that name. | `archive-db` | — | — |
|
||||
| `GRAFANA_ADMIN_PASSWORD` | Grafana `admin` user password | `changeme` | YES (prod) | YES |
|
||||
| `GRAFANA_DB_PASSWORD` | Password for the read-only `grafana_reader` PostgreSQL role used by the PO Overview dashboard (issue #651). Consumed by Flyway V68 and the Grafana PostgreSQL datasource. Generate with `openssl rand -hex 32`. | — | YES (prod) | YES |
|
||||
| `PORT_GLITCHTIP` | Host port for the GlitchTip UI (bound to `127.0.0.1` only) | `3002` | — | — |
|
||||
| `GLITCHTIP_DOMAIN` | Public-facing base URL for GlitchTip (used in email links and CORS) | `http://localhost:3002` | YES (prod) | — |
|
||||
| `GLITCHTIP_SECRET_KEY` | Django secret key for GlitchTip — generate with `python3 -c "import secrets; print(secrets.token_hex(32))"` | — | YES | YES |
|
||||
@@ -256,6 +257,7 @@ git.raddatz.cloud A <server IP>
|
||||
| `MAIL_USERNAME` | release.yml | SMTP user |
|
||||
| `MAIL_PASSWORD` | release.yml | SMTP password |
|
||||
| `GRAFANA_ADMIN_PASSWORD` | both | Grafana `admin` login — generate a strong password |
|
||||
| `GRAFANA_DB_PASSWORD` | both | Read-only `grafana_reader` role password — `openssl rand -hex 32` |
|
||||
| `GLITCHTIP_SECRET_KEY` | both | Django secret key — `openssl rand -hex 32` |
|
||||
| `SENTRY_DSN` | both | GlitchTip project DSN — set after first-run (§4); leave empty to keep Sentry disabled |
|
||||
| `VITE_SENTRY_DSN` | both | GlitchTip frontend project DSN — set after first-run (§4); leave empty to keep Sentry disabled |
|
||||
@@ -357,6 +359,7 @@ Both files are passed explicitly via `--env-file` to the compose command, so the
|
||||
| Gitea secret | Notes |
|
||||
|---|---|
|
||||
| `GRAFANA_ADMIN_PASSWORD` | Strong unique password; shared by nightly and release |
|
||||
| `GRAFANA_DB_PASSWORD` | `openssl rand -hex 32`; shared by nightly and release — read-only DB role for the PO Overview dashboard |
|
||||
| `GLITCHTIP_SECRET_KEY` | `openssl rand -hex 32`; shared by nightly and release |
|
||||
| `STAGING_POSTGRES_PASSWORD` / `PROD_POSTGRES_PASSWORD` | Must match the running PostgreSQL container |
|
||||
|
||||
@@ -427,6 +430,31 @@ docker exec obs-loki wget -qO- \
|
||||
|
||||
Prometheus port `9090` and Grafana port `3003` (default; configurable via `PORT_GRAFANA`) are bound to `127.0.0.1` on the host. No other observability ports are host-bound.
|
||||
|
||||
##### Rotate the `grafana_reader` DB password
|
||||
|
||||
The PO Overview dashboard reads `audit_log`, `documents`, and `transcription_blocks` through the SELECT-only `grafana_reader` PostgreSQL role (issue #651, ADR-024). The role's password is owned by `R__grafana_reader_password.sql` — a Flyway *repeatable* migration that re-runs whenever the resolved `${grafanaDbPassword}` placeholder changes. That makes rotation a two-restart operation, no manual `psql` required.
|
||||
|
||||
```bash
|
||||
# 1. Generate a new value
|
||||
openssl rand -hex 32
|
||||
|
||||
# 2. Update both sides:
|
||||
# - Gitea secret GRAFANA_DB_PASSWORD (nightly + release workflows pick it up)
|
||||
# - Local .env on the server / dev machine
|
||||
|
||||
# 3. Restart the backend. Flyway sees that R__'s resolved checksum changed and
|
||||
# re-applies it, issuing ALTER ROLE grafana_reader WITH PASSWORD '<new>'.
|
||||
docker compose restart backend
|
||||
|
||||
# 4. Restart obs-grafana so the provisioned datasource picks up the new env value.
|
||||
docker compose -f docker-compose.observability.yml restart obs-grafana
|
||||
|
||||
# 5. Verify the dashboard loads — PO Overview's Postgres panels should populate
|
||||
# instead of "Data source error".
|
||||
```
|
||||
|
||||
If `GRAFANA_DB_PASSWORD` is unset, the backend **refuses to start** (`IllegalStateException`). That is deliberate — see `FlywayConfig.resolveGrafanaDbPassword()` and the rationale in ADR-024.
|
||||
|
||||
#### GlitchTip
|
||||
|
||||
| Item | Value |
|
||||
|
||||
@@ -57,6 +57,10 @@ _See also [Annotation](#annotation-documentannotation)._
|
||||
|
||||
**Mass import** — an asynchronous batch process (`MassImportService`) that reads an Excel or ODS file and creates `Person`s, `Tag`s, and `PLACEHOLDER` `Document`s in one shot. Only one import can run at a time (`IMPORT_ALREADY_RUNNING` error if attempted concurrently).
|
||||
|
||||
**SkippedFile** (`MassImportService.SkippedFile`) — a file that was presented for import but not processed, recorded with a `filename` and a `reason` code. Possible reasons: `INVALID_PDF_SIGNATURE` (magic-byte validation failed), `S3_UPLOAD_FAILED` (file upload to MinIO/S3 threw an exception), `FILE_READ_ERROR` (the file could not be opened for reading), or `ALREADY_EXISTS` (a document with the same filename already exists in the archive with a status other than `PLACEHOLDER`).
|
||||
|
||||
**skipped count** — the total number of `SkippedFile` entries accumulated during a single import run (`ImportStatus.skipped()`). Shown in the amber warning section of the Import Status Card in the admin UI; a value of zero suppresses the section entirely.
|
||||
|
||||
**Transcription queue** — the set of `Document`s and `TranscriptionBlock`s awaiting work, computed on-the-fly from `Document`/`Block` status. Three views: segmentation queue, transcription queue, ready-to-read queue. NOT a persistent entity — no `transcription_queues` table exists.
|
||||
_See also [DocumentStatus lifecycle](#documentstatus-lifecycle)._
|
||||
|
||||
@@ -76,6 +80,14 @@ _See also [DocumentStatus lifecycle](#documentstatus-lifecycle)._
|
||||
|
||||
**Sütterlin** — A specific standardized style of Kurrent taught in German schools from 1915 to 1941.
|
||||
|
||||
**Illegible word** — a word whose recognition confidence falls below the configured threshold; replaced with the literal token `[unleserlich]` in the rendered block text and counted in the `ocr_illegible_words_total` Prometheus counter.
|
||||
|
||||
**Models-ready gauge** — the `ocr_models_ready` Prometheus gauge, flipped from `0` to `1` once the FastAPI lifespan startup has finished loading the Kraken model and the spell-checker. Used both for the `/health` endpoint and as the supervised signal for the `ocr_models_ready < 1 for 2m` alert.
|
||||
|
||||
**Recognition model accuracy** — the accuracy reported by `ketos train` for the recognition (text-line) model, exposed as `ocr_model_accuracy{kind="recognition"}`. Sourced from `_parse_best_checkpoint` on the highest-scoring checkpoint after training.
|
||||
|
||||
**Segmentation model accuracy** — the accuracy reported by `ketos segtrain` for the baseline layout analysis (`blla`) model, exposed as `ocr_model_accuracy{kind="segmentation"}`. Distinct from recognition accuracy because the two models are trained and improved independently.
|
||||
|
||||
---
|
||||
|
||||
## Other Domain Terms
|
||||
|
||||
@@ -118,11 +118,14 @@ To find a trace for a specific request in staging/production, either increase th
|
||||
|
||||
## Metrics (Prometheus → Grafana)
|
||||
|
||||
Prometheus scrapes the backend management endpoint every 15 s:
|
||||
Prometheus scrapes two targets every 15 s:
|
||||
|
||||
```
|
||||
Target: backend:8081/actuator/prometheus
|
||||
Labels: job="spring-boot", application="Familienarchiv"
|
||||
|
||||
Target: ocr:8000/metrics
|
||||
Labels: job="ocr-service"
|
||||
```
|
||||
|
||||
All Spring Boot metrics carry the `application="Familienarchiv"` tag, which is how the Grafana Spring Boot Observability dashboard (ID 17175) filters to this service.
|
||||
@@ -146,6 +149,70 @@ jvm_memory_used_bytes{area="heap", application="Familienarchiv"}
|
||||
hikaricp_connections_active
|
||||
```
|
||||
|
||||
### OCR-service custom metrics
|
||||
|
||||
Exposed at `ocr:8000/metrics` by `prometheus-fastapi-instrumentator`. The
|
||||
`http_*` metrics describe the FastAPI request layer; the `ocr_*` series are
|
||||
domain-specific. **Never label these with PII or document content** — labels
|
||||
have unbounded cardinality risk and are visible to anyone with Grafana access.
|
||||
|
||||
| Metric | Type | Labels | Unit | What it tracks |
|
||||
|---|---|---|---|---|
|
||||
| `ocr_jobs_total` | Counter | `engine` (`surya`/`kraken`), `script_type` | jobs | OCR jobs that started after a successful PDF download |
|
||||
| `ocr_pages_total` | Counter | `engine` | pages | Successfully OCR'd pages in the streaming generator |
|
||||
| `ocr_skipped_pages_total` | Counter | — | pages | Pages skipped because the engine raised on them |
|
||||
| `ocr_words_total` | Counter | — | words | Recognized words summed across every block |
|
||||
| `ocr_illegible_words_total` | Counter | — | words | Words below the confidence threshold (rendered as `[unleserlich]`) |
|
||||
| `ocr_processing_seconds` | Histogram | `engine` | seconds | Per-page (stream) or per-document (`/ocr`) engine time, excluding preprocessing |
|
||||
| `ocr_training_runs_total` | Counter | `kind` (`recognition`/`segmentation`), `outcome` (`success`/`error`) | runs | Completed training runs |
|
||||
| `ocr_model_accuracy` | Gauge | `kind` | ratio (0–1) | Latest accuracy reported by a successful training run |
|
||||
| `ocr_models_ready` | Gauge | — | 0\|1 | 1 once the lifespan startup has finished loading models |
|
||||
|
||||
Canonical example queries (the same ones referenced in issue #652):
|
||||
|
||||
```promql
|
||||
# OCR throughput by engine
|
||||
sum by (engine) (rate(ocr_pages_total[5m]))
|
||||
|
||||
# Share of words rendered as [unleserlich]
|
||||
sum(rate(ocr_illegible_words_total[5m]))
|
||||
/ sum(rate(ocr_words_total[5m]))
|
||||
|
||||
# p95 page processing time per engine
|
||||
histogram_quantile(0.95, sum by (engine, le) (
|
||||
rate(ocr_processing_seconds_bucket[5m])
|
||||
))
|
||||
|
||||
# Training error rate
|
||||
sum(rate(ocr_training_runs_total{outcome="error"}[1h]))
|
||||
/ sum(rate(ocr_training_runs_total[1h]))
|
||||
|
||||
# Latest recognition vs segmentation accuracy
|
||||
ocr_model_accuracy
|
||||
```
|
||||
|
||||
### Internal-only endpoints
|
||||
|
||||
`/metrics` is exposed by the OCR service over plain HTTP without
|
||||
authentication. The container is reachable only on the internal Docker
|
||||
network — Caddy never proxies to it directly. If the service is ever
|
||||
exposed (e.g. a `ports:` mapping is added), block the endpoint at the
|
||||
reverse proxy:
|
||||
|
||||
```caddy
|
||||
ocr.example.com {
|
||||
@internal_only path /metrics /health
|
||||
respond @internal_only 404
|
||||
reverse_proxy ocr:8000
|
||||
}
|
||||
```
|
||||
|
||||
The `MetricsPathFilter` in `ocr-service/main.py` suppresses uvicorn's
|
||||
**stdout** access log lines for `/metrics` and `/health` so the container
|
||||
console stays focused on real OCR traffic. Promtail/Loki still receive
|
||||
access lines from any other source. Treat the filter as console
|
||||
noise-control, not an audit-suppression mechanism.
|
||||
|
||||
## Errors (GlitchTip)
|
||||
|
||||
GlitchTip receives errors from both the backend (via Sentry Java SDK) and the frontend (via Sentry JavaScript SDK). It groups events by fingerprint, tracks first/last seen times, and links to the release that introduced the error.
|
||||
|
||||
115
docs/adr/022-csrf-session-revocation-rate-limiting.md
Normal file
115
docs/adr/022-csrf-session-revocation-rate-limiting.md
Normal file
@@ -0,0 +1,115 @@
|
||||
# ADR-022 — CSRF Protection, Session Revocation, and Login Rate Limiting
|
||||
|
||||
**Date:** 2026-05-18
|
||||
**Status:** Accepted
|
||||
**Issue:** #524
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
ADR-020 established stateful authentication via Spring Session JDBC. Three
|
||||
follow-on security concerns were left open:
|
||||
|
||||
1. **CSRF.** State-changing API calls from the SvelteKit frontend use session
|
||||
cookies. Without CSRF protection an attacker can forge cross-origin requests
|
||||
that carry the victim's session cookie.
|
||||
|
||||
2. **Session revocation.** A user who changes or resets their password may still
|
||||
have other active sessions (other browsers, shared devices). Those sessions
|
||||
should be invalidated so the credential change takes full effect immediately.
|
||||
|
||||
3. **Login rate limiting.** The login endpoint accepts arbitrary email/password
|
||||
pairs. Without throttling it is vulnerable to brute-force and credential-
|
||||
stuffing attacks.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. CSRF — double-submit cookie pattern
|
||||
|
||||
`SecurityConfig` enables `CookieCsrfTokenRepository.withHttpOnlyFalse()`:
|
||||
|
||||
- The backend sets an `XSRF-TOKEN` cookie (readable by JavaScript) on every
|
||||
response.
|
||||
- All state-changing requests (`POST`, `PUT`, `PATCH`, `DELETE`) must include
|
||||
an `X-XSRF-TOKEN` request header whose value matches the cookie.
|
||||
- `CsrfTokenRequestAttributeHandler` is used (non-XOR mode) — correct for
|
||||
SPAs where token deferred loading would otherwise corrupt values.
|
||||
- SvelteKit's `handleFetch` hook injects the header and mirrors the cookie for
|
||||
every mutating API call.
|
||||
- CSRF validation failures return HTTP 403 with JSON body
|
||||
`{"code": "CSRF_TOKEN_MISSING"}` via a custom `AccessDeniedHandler`.
|
||||
|
||||
Login (`POST /api/auth/login`), forgot-password, and reset-password are
|
||||
**not** CSRF-exempt — the XSRF-TOKEN cookie is set on the first GET to the
|
||||
login page, so the double-submit requirement is satisfiable from the browser.
|
||||
|
||||
### 2. Session revocation
|
||||
|
||||
`AuthService` gains two methods backed by `JdbcIndexedSessionRepository`:
|
||||
|
||||
- `revokeOtherSessions(currentSessionId, principal)` — deletes all sessions
|
||||
for a principal **except** the caller's current session. Called on password
|
||||
change so the user stays logged in on the current device.
|
||||
- `revokeAllSessions(principal)` — deletes every session for a principal.
|
||||
Called on password reset (unauthenticated flow) so no prior sessions survive.
|
||||
|
||||
Both methods are no-ops when `sessionRepository` is `null` (unit-test
|
||||
contexts that do not load Spring Session).
|
||||
|
||||
### 3. Login rate limiting — in-memory token bucket
|
||||
|
||||
`LoginRateLimiter` (Bucket4j + Caffeine) enforces two independent limits:
|
||||
|
||||
| Bucket | Limit | Window | Key |
|
||||
|--------|-------|--------|-----|
|
||||
| Per IP + email | 10 attempts | 15 min | `ip:email` |
|
||||
| Per IP (all emails) | 20 attempts | 15 min | `ip` |
|
||||
|
||||
On each login attempt both buckets are checked **sequentially**:
|
||||
1. Consume from the `ip:email` bucket first.
|
||||
2. If the IP-level bucket is exhausted, **refund** the `ip:email` token.
|
||||
|
||||
The refund prevents IP-level blocking from silently consuming per-email quota:
|
||||
without it, 20 blocked attempts for `target@example.com` from a single IP
|
||||
(caused by another email exhausting the IP bucket) would drain all 10 of
|
||||
`target@`'s tokens.
|
||||
|
||||
On a successful login both buckets are invalidated for that `(ip, email)` pair
|
||||
so a legitimately authenticated user regains the full window immediately.
|
||||
|
||||
Rate-limit violations are audited as `LOGIN_RATE_LIMITED` events.
|
||||
|
||||
The cache is **node-local** (in-memory). In a multi-replica deployment the
|
||||
effective rate limit is multiplied by the replica count. This is acceptable for
|
||||
the current single-VPS production setup and is noted with a comment in the
|
||||
source.
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
- **CSRF:** All SvelteKit API calls must supply `X-XSRF-TOKEN`. Bare `curl`
|
||||
calls or non-browser clients must obtain and pass the token manually.
|
||||
Integration tests use `.with(csrf())` from `spring-security-test`.
|
||||
- **Session revocation:** Requires `JdbcIndexedSessionRepository` to be wired
|
||||
(Spring Session JDBC dependency). Unit tests inject `null` and verify the
|
||||
no-op path.
|
||||
- **Rate limiting:** False positives are possible if many users share a NAT/VPN
|
||||
IP. The per-IP limit (20) is intentionally loose to reduce collateral
|
||||
blocking; the per-IP+email limit (10) is the primary defence.
|
||||
- `ObjectMapper` in the CSRF `AccessDeniedHandler` uses a static instance
|
||||
because `@WebMvcTest` slices exclude `JacksonAutoConfiguration`. The response
|
||||
only serialises a fixed String key (`"code"`) so naming strategy and custom
|
||||
modules are irrelevant.
|
||||
- IP extraction uses `HttpServletRequest.getRemoteAddr()`. In deployments behind
|
||||
a reverse proxy the `X-Forwarded-For` header is not trusted — doing so would
|
||||
let clients spoof their IP and trivially bypass the per-IP limit. Trusting
|
||||
proxy headers requires separate work (e.g. Spring's `ForwardedHeaderFilter`
|
||||
with an allowlist of trusted proxy addresses).
|
||||
- IPv6 and IPv4-mapped addresses (e.g. `::ffff:1.2.3.4`) are not normalised to
|
||||
a canonical form. An attacker with access to multiple IPv6 addresses could
|
||||
rotate addresses to bypass the per-IP bucket. This is a known limitation of
|
||||
address-based rate limiting and is acceptable for the current deployment.
|
||||
110
docs/adr/022-eager-to-lazy-fetch-strategy.md
Normal file
110
docs/adr/022-eager-to-lazy-fetch-strategy.md
Normal file
@@ -0,0 +1,110 @@
|
||||
# ADR-022 — EAGER→LAZY Fetch Strategy for Document Collections
|
||||
|
||||
**Date:** 2026-05-18
|
||||
**Status:** Accepted
|
||||
**Issue:** #467
|
||||
**PR:** #622
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
A pre-production query audit of 24 HTTP requests to the document list and detail endpoints
|
||||
produced **2,733 SQL statements** — primarily N+1 queries caused by `FetchType.EAGER` on
|
||||
`Document.receivers`, `Document.tags`, `Document.trainingLabels`, and `Document.sender`.
|
||||
|
||||
With EAGER fetch, every `Document` loaded by any repository method immediately triggers
|
||||
additional `SELECT` statements for each associated collection, regardless of whether the
|
||||
caller needs those associations. For a list of 100 documents, this means up to 400 extra
|
||||
queries for `receivers` alone.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
Switch all four associations to `FetchType.LAZY` and use a two-tier strategy to load exactly
|
||||
what each code path needs:
|
||||
|
||||
**Tier 1 — Named entity graphs on `Document` + `@EntityGraph` overrides on `DocumentRepository`:**
|
||||
|
||||
- `Document.full` — loads `sender`, `receivers`, `tags` — used by `findById` (detail view)
|
||||
- `Document.list` — loads `sender`, `tags` — used by `findAll(Spec, Pageable)`,
|
||||
`findAll(Spec)`, and `findAll(Pageable)` (list/search/dashboard paths)
|
||||
|
||||
Each repository method that is called from a hot code path has an `@EntityGraph` override
|
||||
that declares exactly which associations to JOIN-fetch, collapsing N+1 into 1–2 queries.
|
||||
|
||||
**Tier 2 — `@BatchSize(50)` fallback on all four associations:**
|
||||
|
||||
For any lazy access path not covered by an entity graph (e.g., a future ad-hoc query or an
|
||||
in-memory sort that touches `trainingLabels`), Hibernate batches the secondary `SELECT` to
|
||||
at most one statement per 50 entities instead of one per entity.
|
||||
|
||||
**Session lifetime for post-return lazy access:**
|
||||
|
||||
`getDocumentById` and `getRecentActivity` return entities to callers that may access lazy
|
||||
associations after the repository call returns. Both methods are annotated
|
||||
`@Transactional(readOnly = true)` to keep the Hibernate session open until the service method
|
||||
returns, making those post-return accesses safe.
|
||||
|
||||
This is an intentional exception to the project convention that read methods are not annotated
|
||||
(see `CLAUDE.md §Services`). The convention remains correct for all other read methods; this
|
||||
exception applies only to methods that serve lazy-initialized associations to their callers.
|
||||
|
||||
---
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### `@BatchSize`-only (no entity graphs)
|
||||
|
||||
`@BatchSize(50)` on all associations would eliminate the worst N+1 cases (100 documents → 2
|
||||
batch queries instead of 100 individual queries) without requiring repository overrides. Simpler
|
||||
to maintain — no named graph definitions, no per-method overrides.
|
||||
|
||||
Rejected because batch loading is best-effort: it depends on what Hibernate happens to find in
|
||||
the first-level cache and produces a variable number of statements. Entity graphs produce a
|
||||
deterministic, verifiable statement count that can be asserted in tests. The query-count test
|
||||
suite (`DocumentRepositoryTest`) validates the exact statement bounds on every CI run.
|
||||
|
||||
### Single unified entity graph (`Document.full` everywhere)
|
||||
|
||||
Loading `receivers` on every list query is wasteful — the document list view only needs
|
||||
`sender` and `tags`. `receivers` is a `@ManyToMany` collection that, when JOIN-fetched together
|
||||
with `tags`, forces Hibernate to split into two queries anyway (to avoid Cartesian product).
|
||||
Using a single graph on list paths would load data the UI does not display.
|
||||
|
||||
Rejected in favour of two graphs with distinct scopes: `Document.list` for list paths
|
||||
(sender + tags), `Document.full` for detail paths (sender + receivers + tags).
|
||||
|
||||
### `@Transactional` on the Spring Data repository methods
|
||||
|
||||
Spring Data allows `@Transactional` on repository interfaces directly. This would keep the
|
||||
session open for all calls to those methods without touching the service layer.
|
||||
|
||||
Rejected because the transaction boundary belongs at the service layer — repositories should
|
||||
not own transaction lifecycle. The service methods are the natural scope for "keep the session
|
||||
open long enough for the caller to use the result."
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
- **Query count reduced from ~2,733 to ≤10 statements per 24 HTTP requests** — verified by
|
||||
`DocumentRepositoryTest` query-count assertions and `DocumentLazyLoadingTest` smoke tests.
|
||||
- **Read methods that return lazily-initialized entities must carry `@Transactional(readOnly = true)`.**
|
||||
Any future service method that loads a `Document` and returns it to a caller that accesses
|
||||
lazy associations must follow this pattern. Removing the annotation causes
|
||||
`LazyInitializationException` in production.
|
||||
- **New lazy code paths need an entity graph or `@BatchSize` review.** Any new
|
||||
`DocumentRepository` method added to a hot code path should be assessed for N+1 risk and
|
||||
given an `@EntityGraph` override if warranted.
|
||||
- **`@JsonIgnoreProperties({"hibernateLazyInitializer", "handler"})` required on serialized lazy-proxy entities.**
|
||||
`Person` and `Tag` carry this annotation to prevent Jackson from attempting to serialize
|
||||
Hibernate proxy internals when the association is not initialized. Any new entity that is
|
||||
used as a lazy association and serialized directly (without a DTO) needs the same annotation.
|
||||
- **Named graph strings in `Document.java` and `DocumentRepository.java` must stay in sync.**
|
||||
The `@NamedEntityGraph(name = "Document.full")` / `@NamedEntityGraph(name = "Document.list")`
|
||||
definitions on `Document` are referenced by string in every `@EntityGraph(value = "...")` on
|
||||
`DocumentRepository`. If the names diverge (e.g. a graph is renamed in one place but not the
|
||||
other), Spring Data throws at application startup. Always update both files together when
|
||||
renaming or restructuring a named graph.
|
||||
@@ -0,0 +1,94 @@
|
||||
# ADR-023: Prometheus Instrumentator and Metrics Registry Injection
|
||||
|
||||
## Status
|
||||
|
||||
Accepted
|
||||
|
||||
## Context
|
||||
|
||||
Until issue #652 the OCR service exposed no `/metrics` endpoint. The
|
||||
observability stack already scrapes the Spring Boot backend's actuator
|
||||
endpoint, but it had nothing to scrape on the Python side. Without HTTP-
|
||||
and domain-level metrics from `ocr-service` we cannot answer questions
|
||||
like "what is the share of words rendered as `[unleserlich]`" or
|
||||
"is the training error rate above its budget" from Grafana.
|
||||
|
||||
Two implementation requirements influenced the design:
|
||||
|
||||
1. **Counter / gauge isolation in tests.** `prometheus_client` collectors
|
||||
are module-level singletons keyed by name on the global `REGISTRY`.
|
||||
Re-importing or naively re-instantiating them raises a duplicated-
|
||||
collector error and cross-test state leaks (a `.inc()` in test A is
|
||||
still readable by test B). A test harness needs a way to swap the
|
||||
active container for a fresh per-test instance.
|
||||
|
||||
2. **Minimal blast radius on the request path.** We did not want to
|
||||
hand-instrument every endpoint with FastAPI middleware. The
|
||||
`prometheus-fastapi-instrumentator` library already provides
|
||||
`http_requests_total`, `http_request_duration_seconds`, and the
|
||||
`/metrics` exposition route, all idiomatic Prometheus names.
|
||||
|
||||
## Decision
|
||||
|
||||
- Add `prometheus-fastapi-instrumentator==7.0.0` and pin its transitive
|
||||
dependency `prometheus-client==0.25.0` explicitly in
|
||||
`ocr-service/requirements.txt`.
|
||||
- Mount the instrumentator once at module load:
|
||||
`Instrumentator(excluded_handlers=["/health", "/metrics"]).instrument(app).expose(app)`.
|
||||
This adds `/metrics` and an HTTP-level dashboard surface without
|
||||
changing any endpoint code.
|
||||
- Define every domain metric (`ocr_jobs_total`, `ocr_pages_total`,
|
||||
`ocr_processing_seconds`, …) inside a `build_metrics(registry)`
|
||||
factory in `ocr-service/metrics.py` that returns a frozen `OcrMetrics`
|
||||
dataclass. Production code binds the container to the default
|
||||
`REGISTRY` once: `metrics: OcrMetrics = build_metrics(REGISTRY)`.
|
||||
- Tests use a `fresh_metrics` fixture that builds a new
|
||||
`CollectorRegistry()` per test and monkeypatches `main.metrics` with
|
||||
a container bound to it. The endpoint code keeps reading
|
||||
`metrics.<name>` without knowing whether it is talking to the global
|
||||
registry or a per-test one.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- One reusable factory captures the metric definitions; future metrics
|
||||
go in one place.
|
||||
- Tests run with full counter isolation. Cross-test state leakage is
|
||||
impossible because each test sees its own dataclass instance.
|
||||
- The instrumentator gives us `http_*` metrics for free, including a
|
||||
Grafana-ready histogram that pairs with the Spring Boot one.
|
||||
|
||||
**Negative**
|
||||
|
||||
- One extra level of indirection: any test that asserts on metric
|
||||
values must remember to monkeypatch `main.metrics`, not the registry
|
||||
directly. Rebinding through the registry is harmless but useless —
|
||||
the dataclass holds references to the original collectors.
|
||||
- `prometheus-client` is now pinned. Upgrading it requires an explicit
|
||||
bump and re-checking the instrumentator's compatibility range.
|
||||
- `/metrics` is exposed unauthenticated and relies on the Docker
|
||||
internal network for confidentiality. See
|
||||
[docs/OBSERVABILITY.md §Internal-only endpoints](../OBSERVABILITY.md)
|
||||
for the Caddy snippet that must be added if the service ever gets a
|
||||
host-side port mapping.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Hand-roll the `/metrics` endpoint.** Rejected: would have meant
|
||||
duplicating what `prometheus-fastapi-instrumentator` ships, plus
|
||||
middleware for the HTTP histograms.
|
||||
- **Skip the factory; pass `registry` as a function argument
|
||||
everywhere.** Rejected: clutters every endpoint signature and breaks
|
||||
the symmetry with the Spring Boot side, which also relies on a
|
||||
process-global Micrometer registry.
|
||||
- **Use a `pytest` autouse fixture that resets `REGISTRY` between
|
||||
tests.** Rejected: `prometheus_client` does not expose a clean
|
||||
"unregister all" hook, and we would be relying on private APIs.
|
||||
|
||||
## References
|
||||
|
||||
- Issue: [#652](https://git.raddatz.cloud/marcel/familienarchiv/issues/652)
|
||||
- Library: <https://github.com/trallnag/prometheus-fastapi-instrumentator>
|
||||
- Code: `ocr-service/metrics.py`, `ocr-service/main.py`,
|
||||
`ocr-service/test_metrics.py`
|
||||
123
docs/adr/024-grafana-reads-archive-db-via-bridged-network.md
Normal file
123
docs/adr/024-grafana-reads-archive-db-via-bridged-network.md
Normal file
@@ -0,0 +1,123 @@
|
||||
# ADR-024: Grafana reads archive-db via a bridged network and a SELECT-only role
|
||||
|
||||
## Status
|
||||
|
||||
Accepted
|
||||
|
||||
## Context
|
||||
|
||||
Issue #651 (the PO Overview Grafana dashboard) needs aggregates over three
|
||||
tables in the main application database — `audit_log`, `documents`, and
|
||||
`transcription_blocks` — to answer the operator's four weekly questions: is
|
||||
everything working, are people using it, is the archive making progress, is
|
||||
OCR working well.
|
||||
|
||||
Until now, `obs-grafana` and the rest of the observability stack lived on
|
||||
their own Docker network (`obs-net`) and never touched `archiv-net`, where
|
||||
`archive-db` runs. The two were intentionally isolated: a compromise of any
|
||||
observability container could not pivot to the application database.
|
||||
|
||||
The PO Overview's archive-progress and user-activity panels need rolling
|
||||
7-day SQL aggregates that cannot be served by Prometheus or Loki. That
|
||||
forces a connection from `obs-grafana` to `archive-db` for the first time.
|
||||
|
||||
Two implementation requirements shaped the design:
|
||||
|
||||
1. **Least privilege on the database side.** The Spring Boot application
|
||||
role (`archiv`) has full read/write on every table. Letting Grafana
|
||||
connect with that role would mean a Grafana compromise becomes an
|
||||
application compromise. The dashboard only needs SELECT on three
|
||||
tables; the role must reflect that and nothing more.
|
||||
|
||||
2. **Operational simplicity of secret rotation.** The role's password is
|
||||
shared between the migration that sets it and the Grafana datasource
|
||||
that uses it. A first version of this work put the password in a
|
||||
versioned Flyway migration (V68), which Flyway only applies once —
|
||||
leaving rotation as an out-of-band `psql ALTER ROLE` step that no
|
||||
runbook documented. The shape must support rotation without manual
|
||||
SQL.
|
||||
|
||||
## Decision
|
||||
|
||||
- Provision a dedicated PostgreSQL role `grafana_reader` with `LOGIN` plus
|
||||
`GRANT SELECT` on `audit_log`, `documents`, `transcription_blocks` only.
|
||||
No INSERT/UPDATE/DELETE on any table, no access to any other table —
|
||||
enforced by the database, locked in by both positive and parameterized
|
||||
negative tests in `GrafanaReaderRoleIntegrationTest`.
|
||||
- Split the role's lifecycle across two migrations:
|
||||
- `V68__add_grafana_reader_role.sql` — versioned, immutable, idempotent.
|
||||
Creates the role and applies the grants. Runs exactly once per
|
||||
database, like every other versioned migration.
|
||||
- `R__grafana_reader_password.sql` — Flyway *repeatable* migration that
|
||||
issues `ALTER ROLE grafana_reader WITH PASSWORD '${grafanaDbPassword}'`.
|
||||
Flyway computes the checksum on the resolved content, so any change
|
||||
to `GRAFANA_DB_PASSWORD` flips the checksum and re-applies the
|
||||
migration on the next boot. Rotation becomes "bump env var, restart
|
||||
backend, restart obs-grafana" — see the runbook in
|
||||
`docs/DEPLOYMENT.md §4 → Rotate the grafana_reader DB password`.
|
||||
- Resolve the password through Spring's `Environment` rather than a raw
|
||||
`System.getenv()` call, so tests inject via `application.properties`
|
||||
and the resolver is unit-testable with `MockEnvironment`. Fail closed
|
||||
with `IllegalStateException` when the variable is unset — no fallback
|
||||
string. Same shape as `UserDataInitializer`'s refusal to seed default
|
||||
admin credentials outside dev/test/e2e.
|
||||
- Join `obs-grafana` to `archiv-net` in addition to `obs-net`. Only the
|
||||
Grafana container crosses the boundary; Loki, Tempo, Prometheus,
|
||||
GlitchTip, and the worker containers remain `obs-net`-only.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- Database-level least privilege: a Grafana compromise gains SELECT on
|
||||
three tables. Cannot write, cannot read PII tables like `app_users`,
|
||||
`persons`, `notifications`, `document_comments`, `geschichten`. The
|
||||
parameterized PII negative sweep in `GrafanaReaderRoleIntegrationTest`
|
||||
is the regression gate; new sensitive tables get added to that list.
|
||||
- Rotation is documented, idempotent, and survives operator turnover.
|
||||
No "the password set on day 1 is the password forever" failure mode.
|
||||
- Tests pin down both sides of the boundary: positive grants must hold,
|
||||
write-deny must hold, and the PII negative list must stay empty.
|
||||
|
||||
**Negative / trade-offs**
|
||||
|
||||
- `obs-net` is no longer fully isolated from `archiv-net`. A Grafana RCE
|
||||
(e.g. via a future Grafana CVE) gains a TCP path to `archive-db` —
|
||||
contained, but not impossible. The least-privilege role is the
|
||||
mitigation; we accept that mitigation as sufficient for a single
|
||||
bridged container.
|
||||
- The backend must hold `GRAFANA_DB_PASSWORD` in its environment forever,
|
||||
so Flyway can resolve the placeholder on every boot. A backend RCE
|
||||
therefore also leaks the Grafana datasource password. Acceptable
|
||||
because that password's blast radius is itself bounded by the
|
||||
least-privilege grants on `grafana_reader`.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Prometheus PostgreSQL exporter, no direct connection.** Loses ad-hoc
|
||||
SQL aggregates — the dashboard would need every metric pre-defined as
|
||||
an exporter query, with a redeploy to add a new one. The PO Overview
|
||||
is the type of dashboard that grows panels over time; pre-defining
|
||||
every aggregate is the wrong shape.
|
||||
- **Read replica or logical-replication slot dedicated to Grafana.**
|
||||
Real operational cost (extra Postgres instance, replication monitoring,
|
||||
storage doubled) disproportionate to a weekly PO glance.
|
||||
- **Versioned migration with `flyway repair` for rotation.** Rejected:
|
||||
conflates schema lifecycle with credential lifecycle, requires manual
|
||||
intervention to rotate, and the repair command's semantics are
|
||||
surprising to operators unfamiliar with Flyway internals.
|
||||
- **Hardcoded fallback password when env var is unset.** Rejected as a
|
||||
security blocker: publishes a known credential for a role with read
|
||||
access to user activity and full letter text. The fail-closed
|
||||
behavior is the explicit defense.
|
||||
|
||||
## References
|
||||
|
||||
- Issue #651 — PO Overview Grafana dashboard
|
||||
- `backend/src/main/resources/db/migration/V68__add_grafana_reader_role.sql`
|
||||
- `backend/src/main/resources/db/migration/R__grafana_reader_password.sql`
|
||||
- `backend/src/main/java/org/raddatz/familienarchiv/config/FlywayConfig.java`
|
||||
- `backend/src/test/java/org/raddatz/familienarchiv/config/GrafanaReaderRoleIntegrationTest.java`
|
||||
- `infra/observability/grafana/provisioning/datasources/datasources.yml`
|
||||
- `docker-compose.observability.yml` — `archiv-net` bridge on `obs-grafana`
|
||||
- `docs/DEPLOYMENT.md §4` — rotation runbook
|
||||
@@ -43,9 +43,12 @@ Rel(ocr, storage, "Fetches PDF via presigned URL", "HTTP / S3 presigned")
|
||||
Rel(mc, storage, "Bootstraps bucket + service account on startup", "MinIO Client CLI")
|
||||
Rel(promtail, loki, "Pushes log streams", "HTTP/Loki push API")
|
||||
Rel(backend, tempo, "Sends distributed traces via OTLP", "HTTP / OTLP / port 4318 (archiv-net)")
|
||||
Rel(prometheus, backend, "Scrapes JVM + HTTP metrics", "HTTP 8081 /actuator/prometheus")
|
||||
Rel(prometheus, ocr, "Scrapes OCR + http_* metrics", "HTTP 8000 /metrics")
|
||||
Rel(grafana, prometheus, "Queries metrics", "HTTP 9090")
|
||||
Rel(grafana, loki, "Queries logs", "HTTP 3100")
|
||||
Rel(grafana, tempo, "Queries traces", "HTTP 3200")
|
||||
Rel(grafana, db, "Read-only dashboard queries via grafana_reader role", "PostgreSQL / archiv-net")
|
||||
Rel(glitchtip, db, "Stores error events in glitchtip DB", "PostgreSQL / archiv-net")
|
||||
Rel(obs_glitchtip_worker, obs_redis, "Processes Celery tasks", "Redis / obs-net")
|
||||
|
||||
|
||||
@@ -9,18 +9,23 @@ ContainerDb(db, "PostgreSQL", "PostgreSQL 16")
|
||||
System_Boundary(backend, "API Backend (Spring Boot)") {
|
||||
Component(authCtrl, "AuthSessionController", "@RestController org.raddatz.familienarchiv.auth", "POST /api/auth/login validates credentials, rotates the session ID via SessionAuthenticationStrategy (CWE-384 defense), attaches the SecurityContext to the new session. POST /api/auth/logout invalidates the session unconditionally, then best-effort audits.")
|
||||
Component(authSvc, "AuthService", "@Service org.raddatz.familienarchiv.auth", "Delegates credential validation to AuthenticationManager (DaoAuthenticationProvider — timing-equalised via dummy BCrypt on misses). Emits LOGIN_SUCCESS / LOGIN_FAILED / LOGOUT audit entries without ever logging the password attempt.")
|
||||
Component(secFilter, "Security Filter Chain", "Spring Security", "Permits /api/auth/login, /api/auth/forgot-password, /api/auth/reset-password, /api/auth/invite/**, /api/auth/register; everything else requires an authenticated session. Returns 401 (not 302) on missing/expired session. CSRF is disabled pending #524.")
|
||||
Component(sessionRepo, "Spring Session JDBC", "spring-boot-starter-session-jdbc", "Persists sessions in spring_session / spring_session_attributes (Flyway V67). 8-hour idle timeout. Cookie name fa_session, SameSite=Strict, HttpOnly, Secure behind Caddy. Indexes the session by Principal name for revocation in #524.")
|
||||
Component(secFilter, "Security Filter Chain", "Spring Security", "Permits /api/auth/login, /api/auth/forgot-password, /api/auth/reset-password, /api/auth/invite/**, /api/auth/register; everything else requires an authenticated session. Returns 401 (not 302) on missing/expired session. CSRF enabled: double-submit cookie pattern (CookieCsrfTokenRepository.withHttpOnlyFalse + CsrfTokenRequestAttributeHandler). Custom AccessDeniedHandler returns JSON {\"code\":\"CSRF_TOKEN_MISSING\"}.")
|
||||
Component(sessionRepo, "Spring Session JDBC", "spring-boot-starter-session-jdbc", "Persists sessions in spring_session / spring_session_attributes (Flyway V67). 8-hour idle timeout. Cookie name fa_session, SameSite=Strict, HttpOnly, Secure behind Caddy. Indexes the session by Principal name for revocation.")
|
||||
Component(permAspect, "PermissionAspect", "Spring AOP", "Intercepts methods annotated with @RequirePermission. Checks the authenticated user's granted authorities against the required permission. Throws 401/403 if denied.")
|
||||
Component(secConf, "SecurityConfig", "Spring @Configuration", "Wires the filter chain, BCryptPasswordEncoder, DaoAuthenticationProvider, AuthenticationManager, and the ChangeSessionIdAuthenticationStrategy bean used by AuthSessionController.")
|
||||
Component(userDetails, "CustomUserDetailsService", "Spring Security UserDetailsService", "Loads AppUser by email from DB. Converts group permissions to Spring GrantedAuthority objects.")
|
||||
Component(rateLimiter, "LoginRateLimiter", "@Component org.raddatz.familienarchiv.auth", "Dual Bucket4j/Caffeine in-memory rate limiting: per ip:email bucket and per ip bucket. checkAndConsume() throws TOO_MANY_LOGIN_ATTEMPTS (429) when either bucket is exhausted. invalidateOnSuccess() resets both buckets on successful login. Buckets expire after idle windowMinutes.")
|
||||
Component(rateLimitProps, "RateLimitProperties", "@ConfigurationProperties(\"rate-limit.login\") org.raddatz.familienarchiv.auth", "Externalized config for login rate limiting: maxAttemptsPerIpEmail (default 10), maxAttemptsPerIp (default 20), windowMinutes (default 15). Bound from application.yaml rate-limit.login block.")
|
||||
}
|
||||
|
||||
Rel(frontend, authCtrl, "POST /api/auth/login + /logout", "HTTPS, JSON")
|
||||
Rel(frontend, secFilter, "All other API calls", "HTTPS + fa_session cookie")
|
||||
Rel(frontend, secFilter, "All other API calls", "HTTPS + fa_session cookie + X-XSRF-TOKEN header")
|
||||
Rel(authCtrl, authSvc, "Validate creds + audit")
|
||||
Rel(authCtrl, sessionRepo, "getSession() / invalidate()")
|
||||
Rel(authSvc, userDetails, "Authenticates via AuthenticationManager")
|
||||
Rel(authSvc, rateLimiter, "checkAndConsume() / invalidateOnSuccess()")
|
||||
Rel(authSvc, sessionRepo, "revokeOtherSessions() / revokeAllSessions()")
|
||||
Rel(rateLimiter, rateLimitProps, "Reads config")
|
||||
Rel(secFilter, sessionRepo, "Resolves session by fa_session cookie")
|
||||
Rel(secFilter, permAspect, "Authenticated requests reach guarded service methods")
|
||||
Rel(secConf, userDetails, "Wires as UserDetailsService")
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
@startuml
|
||||
title Authentication Flow (Spring Session JDBC, behind Caddy reverse proxy)
|
||||
note over Browser, DB
|
||||
Phase 1 of the auth rewrite (ADR-020 / #523).
|
||||
Replaces the Basic-credentials-in-cookie model
|
||||
with an opaque server-side session id (fa_session).
|
||||
Phase 2 of the auth rewrite (ADR-020, ADR-022 / #523, #524).
|
||||
Adds CSRF double-submit cookies, login rate limiting, and
|
||||
session revocation on password change/reset.
|
||||
end note
|
||||
|
||||
actor User
|
||||
@@ -11,9 +11,10 @@ participant Browser
|
||||
participant "Caddy (TLS termination)" as Caddy
|
||||
participant "Frontend (SvelteKit)" as Frontend
|
||||
participant "Backend (Spring Boot)" as Backend
|
||||
participant "LoginRateLimiter\n(Caffeine+Bucket4j)" as RateLimiter
|
||||
participant "spring_session\n(PostgreSQL)" as DB
|
||||
|
||||
== Login ==
|
||||
== Login (with rate limiting + CSRF bootstrap) ==
|
||||
User -> Browser: Enter email + password
|
||||
Browser -> Caddy: HTTPS POST /?/login (form action)
|
||||
note right of Caddy
|
||||
@@ -30,19 +31,46 @@ note right of Backend
|
||||
→ request.getScheme() = "https"
|
||||
→ Secure cookie flag set automatically.
|
||||
end note
|
||||
Backend -> Backend: AuthenticationManager\nauthenticate(email, password)
|
||||
Backend -> DB: SELECT user WHERE email=?
|
||||
DB --> Backend: AppUser + groups + permissions
|
||||
Backend -> Backend: BCrypt.matches(password, hash)\n(timing-safe: dummy hash on miss)
|
||||
Backend -> Backend: getSession(true).setAttribute(\n SPRING_SECURITY_CONTEXT, ctx)
|
||||
Backend -> DB: INSERT spring_session\n+ spring_session_attributes
|
||||
Backend -> Backend: AuditService.log(LOGIN_SUCCESS,\n {userId, ip, ua})
|
||||
Backend --> Frontend: 200 OK — AppUser\nSet-Cookie: fa_session=<opaque>;\n Path=/; HttpOnly; SameSite=Strict; Secure
|
||||
Frontend -> Frontend: Parse Set-Cookie, re-emit fa_session\n(matches backend attrs)
|
||||
Frontend --> Caddy: 303 → /\nSet-Cookie: fa_session=<opaque>
|
||||
Caddy --> Browser: HTTPS 303 + Set-Cookie
|
||||
Backend -> RateLimiter: checkAndConsume(ip, email)\n[10/15min per ip+email; 20/15min per ip]
|
||||
alt Rate limit exceeded
|
||||
RateLimiter --> Backend: throw DomainException(TOO_MANY_LOGIN_ATTEMPTS)
|
||||
Backend -> Backend: AuditService.log(LOGIN_RATE_LIMITED, {ip, email})
|
||||
Backend --> Frontend: 429 Too Many Requests\n{"code":"TOO_MANY_LOGIN_ATTEMPTS"}
|
||||
Frontend --> Browser: Show rate-limit error
|
||||
else Under limit
|
||||
Backend -> Backend: AuthenticationManager\nauthenticate(email, password)
|
||||
Backend -> DB: SELECT user WHERE email=?
|
||||
DB --> Backend: AppUser + groups + permissions
|
||||
Backend -> Backend: BCrypt.matches(password, hash)\n(timing-safe: dummy hash on miss)
|
||||
Backend -> Backend: getSession(true).setAttribute(\n SPRING_SECURITY_CONTEXT, ctx)
|
||||
Backend -> DB: INSERT spring_session\n+ spring_session_attributes
|
||||
Backend -> RateLimiter: invalidateOnSuccess(ip, email)
|
||||
Backend -> Backend: AuditService.log(LOGIN_SUCCESS,\n {userId, ip, ua})
|
||||
Backend --> Frontend: 200 OK — AppUser\nSet-Cookie: fa_session=<opaque>;\n Path=/; HttpOnly; SameSite=Strict; Secure\nSet-Cookie: XSRF-TOKEN=<token>;\n Path=/; SameSite=Strict; Secure
|
||||
Frontend -> Frontend: Parse Set-Cookie, re-emit fa_session\n(matches backend attrs)
|
||||
Frontend --> Caddy: 303 → /\nSet-Cookie: fa_session=<opaque>
|
||||
Caddy --> Browser: HTTPS 303 + Set-Cookie
|
||||
end
|
||||
|
||||
== Authenticated request ==
|
||||
== Authenticated mutating request (CSRF double-submit) ==
|
||||
note over Browser, Backend
|
||||
handleFetch in hooks.client.ts reads the XSRF-TOKEN cookie
|
||||
and injects X-XSRF-TOKEN header on every POST/PUT/PATCH/DELETE.
|
||||
end note
|
||||
Browser -> Caddy: HTTPS POST /api/...\nCookie: fa_session=<opaque>; XSRF-TOKEN=<token>\nX-XSRF-TOKEN: <token>
|
||||
Caddy -> Backend: HTTP POST /api/...\n+ Cookie + X-XSRF-TOKEN
|
||||
alt X-XSRF-TOKEN missing or mismatched
|
||||
Backend --> Caddy: 403 Forbidden\n{"code":"CSRF_TOKEN_MISSING"}
|
||||
Caddy --> Browser: HTTPS 403
|
||||
else CSRF valid
|
||||
Backend -> DB: SELECT * FROM spring_session WHERE SESSION_ID = ?
|
||||
DB --> Backend: session row
|
||||
Backend -> Backend: Process request
|
||||
Backend --> Caddy: 2xx response + refreshed XSRF-TOKEN cookie
|
||||
Caddy --> Browser: HTTPS 2xx
|
||||
end
|
||||
|
||||
== Authenticated read request ==
|
||||
Browser -> Caddy: HTTPS GET /\nCookie: fa_session=<opaque>
|
||||
Caddy -> Frontend: HTTP GET / + Cookie + X-Forwarded-Proto: https
|
||||
Frontend -> Frontend: hooks.server.ts reads fa_session
|
||||
@@ -61,6 +89,28 @@ else Session expired (idle > 8h) or unknown
|
||||
Caddy --> Browser: HTTPS 302
|
||||
end
|
||||
|
||||
== Password change (revoke other sessions) ==
|
||||
Browser -> Backend: POST /api/users/me/password\n{currentPassword, newPassword}\n+ X-XSRF-TOKEN
|
||||
Backend -> Backend: Verify currentPassword
|
||||
Backend -> DB: UPDATE app_users SET password_hash = ?
|
||||
Backend -> DB: DELETE spring_session WHERE principal = ?\n AND session_id != <current>
|
||||
note right of Backend
|
||||
revokeOtherSessions: caller stays logged in,
|
||||
all other devices are signed out.
|
||||
end note
|
||||
Backend --> Browser: 204 No Content
|
||||
|
||||
== Password reset (revoke all sessions) ==
|
||||
Browser -> Backend: POST /api/auth/reset-password\n{token, newPassword}
|
||||
Backend -> Backend: Verify reset token
|
||||
Backend -> DB: UPDATE app_users SET password_hash = ?
|
||||
Backend -> DB: DELETE spring_session WHERE principal = ?
|
||||
note right of Backend
|
||||
revokeAllSessions: unauthenticated caller has
|
||||
no session to preserve — all sessions wiped.
|
||||
end note
|
||||
Backend --> Browser: 204 No Content
|
||||
|
||||
== Logout ==
|
||||
Browser -> Caddy: HTTPS POST /logout
|
||||
Caddy -> Frontend: HTTP POST /logout\nCookie: fa_session=<opaque>
|
||||
|
||||
313
docs/import-migration/01-findings-spreadsheet-analysis.md
Normal file
313
docs/import-migration/01-findings-spreadsheet-analysis.md
Normal file
@@ -0,0 +1,313 @@
|
||||
# Spreadsheet Analysis — Findings (2026-05-25)
|
||||
|
||||
Analysis of the **real raw archive** spreadsheets against the current `MassImportService`
|
||||
(`backend/.../importing/MassImportService.java`). Goal: import ~7,600 letter rows + a
|
||||
163-person register, with PDFs to follow.
|
||||
|
||||
Every issue has an ID (`IMP-NN`), severity, evidence, and a proposed approach.
|
||||
|
||||
---
|
||||
|
||||
## 0. Context: how the importer reads a row today
|
||||
|
||||
`MassImportService` reads **sheet index 0** and maps columns by configurable indices
|
||||
(`app.import.col.*`, defaults in the source):
|
||||
|
||||
| Property | Default col | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `colIndex` | 0 | Index (→ filename `<index>.pdf`) |
|
||||
| `colBox` | 1 | Box |
|
||||
| `colFolder` | 2 | Mappe |
|
||||
| `colSender` | 3 | Sender (raw) |
|
||||
| `colReceivers` | 5 | Receivers (raw) |
|
||||
| `colDate` | 7 | Date |
|
||||
| `colLocation` | 9 | Location |
|
||||
| `colTags` | 10 | Tag (single) |
|
||||
| `colSummary` | 11 | Summary |
|
||||
| `colTranscription` | 13 | Transcription |
|
||||
|
||||
These defaults match the **ODS** file exactly (`Index, Box, Mappe, Von, BriefeschreiberIn,
|
||||
An, EmpfängerIn, Datum, Datum Originalformat, Ort, Schlagwort, Inhalt, Zeitlicher Kontext,
|
||||
Transkript` = 14 cols). The ODS was the development target. The new xlsx is a different beast.
|
||||
|
||||
Per-row pipeline: skip if Index blank → derive filename from Index → validate filename →
|
||||
look for file on disk (recursive; metadata-only if absent) → check PDF magic bytes →
|
||||
`importSingleDocument` (upsert by `originalFilename`, dedupe non-placeholders as
|
||||
`ALREADY_EXISTS`). Date parsing is **ISO-only** (`LocalDate.parse`).
|
||||
|
||||
---
|
||||
|
||||
## IMP-01 — New xlsx column layout ≠ importer defaults 🔴 BLOCKER
|
||||
|
||||
The new `…aktuell…xlsx` (sheet `Familienarchiv`, 7,943 rows × 12 cols) has a **denser,
|
||||
different** layout. There is an extra `Datei` column at index 1, and the normalized
|
||||
`Von`/`An`/ISO-`Datum` columns from the ODS **do not exist**.
|
||||
|
||||
| col | New xlsx header | Importer default expects | Result with defaults |
|
||||
| --- | --- | --- | --- |
|
||||
| 0 | Index | Index | ✅ ok |
|
||||
| 1 | **Datei** (path) | Box | ❌ Box ← `..\__scan\W-0001.pdf` |
|
||||
| 2 | Box | Mappe | ❌ Mappe ← `V` |
|
||||
| 3 | Mappe | Sender | ❌ Sender ← `1` |
|
||||
| 4 | BriefeschreiberIn (sender) | — (unused) | ❌ sender ignored |
|
||||
| 5 | EmpfängerIn (receiver) | Receivers | ✅ coincidentally ok |
|
||||
| 6 | Datum des Briefes | — (unused) | ❌ date ignored |
|
||||
| 7 | Ort (location) | Date | ❌ Date ← `Rotterdam` → null |
|
||||
| 8 | Schlagwort (tag) | — (unused) | ❌ tag ignored |
|
||||
| 9 | Inhalt (summary) | Location | ❌ Location ← summary text |
|
||||
| 10 | — | Tag | ❌ empty |
|
||||
| 11 | — | Summary | ❌ empty |
|
||||
| 13 | — | Transcription | ❌ column doesn't exist |
|
||||
|
||||
**Impact:** importing as-is produces almost entirely garbage metadata.
|
||||
|
||||
**Proposed approach (decide with Marcel):**
|
||||
- (a) Re-map via the existing `app.import.col.*` properties — fast, no code. New mapping:
|
||||
`index=0, box=2, folder=3, sender=4, receivers=5, date=6, location=7, tags=8, summary=9`,
|
||||
and there is **no** transcription column (point it past the end or add a "missing column"
|
||||
convention). Caveat: tags land in `colTags` but the real per-letter keywords are in
|
||||
`Inhalt` (col 9) — see IMP-08 note on tags vs summary.
|
||||
- (b) Make the importer **header-driven** (map by header name, not index) so it survives
|
||||
layout drift across files. More robust, needs a code change (→ Gitea issue).
|
||||
|
||||
Recommendation: (b) is the durable fix given we have ≥3 different layouts already.
|
||||
|
||||
---
|
||||
|
||||
## IMP-02 — 90% of dates are free-text the parser can't read 🔴 BLOCKER
|
||||
|
||||
The dates are written **as in the letter**. `parseDate()` only does `LocalDate.parse()`
|
||||
(ISO `yyyy-MM-dd`), so anything non-ISO becomes `null`.
|
||||
|
||||
Of **7,319** rows with a date value (col 6):
|
||||
|
||||
| kind | count | parses today? |
|
||||
| --- | --- | --- |
|
||||
| Real Excel date cells (→ ISO via POI) | 748 | ✅ |
|
||||
| Free-text date strings | 6,571 | ❌ → null |
|
||||
|
||||
→ **90% of dated rows lose their date.** (623 rows have no date at all.)
|
||||
|
||||
Observed free-text formats (counts approximate, from col 6):
|
||||
|
||||
| Format | Count | Examples |
|
||||
| --- | --- | --- |
|
||||
| `D.M.YY` | 1,338 | `11.10.08`, `13.5.09` |
|
||||
| `D.RomanMonth.YY/YYYY` | ~1,527 | `22.III.18`, `19.XII.1954`, `1.III.27` |
|
||||
| `D.Month YYYY` | 950 | `6.März 1888`, `9.März 1888` (note: **no space** after the dot) |
|
||||
| `D.M.YYYY` | 358 | `15.2.1888`, `7.3.1888` |
|
||||
| Approximate / unknown | 146 | `?`, `13.7.18?`, `17.Nov (?) 1887`, `13.Januar ? 1907` |
|
||||
| `Month YYYY` / season / holiday | 41+27 | `Mai 1895`, `Herbst 1913`, `Pfingsten 1922`, `Ostern 1890` |
|
||||
| `YYYY` only | 17 | `1905`, `1949` |
|
||||
| `D.M.` no year | 10 | `8.9.`, `14.3.` |
|
||||
| Ranges | 5+ | `8.1.1916 - 15.3.1916`, `1881/82`, `1945/46?` |
|
||||
| Abbrev/English months, no space | many | `29.Sept.1891`, `10.Oct.95`, `9.December1889`, `18.Dez.1916` |
|
||||
| Slash separator | ~315 | `2/2. 18`, `17/6. 1916`, `10/4. 1917` |
|
||||
| English `Month D. YYYY` | several | `April 12. 1922`, `Oct.5. 1916`, `Mai 23. 1917` |
|
||||
| Trailing notes | 5+ | `26.4.1888, 2. Brief`, `31.8.1888,2.Brief` |
|
||||
| 3-digit year (typo) | 107 | `30.1.889` (→ 1889), `4.3.1023` (in person file → 1923) |
|
||||
| Day-range within month | several | `7./8. Sept.1923` |
|
||||
|
||||
**Proposed approach:** build a tolerant German/historical date parser (→ Gitea issue, it's
|
||||
a code change). Requirements:
|
||||
- Numeric `D.M.YY[YY]` and `D/M. YY[YY]` (slash = dot).
|
||||
- Roman-numeral months (`I`–`XII`).
|
||||
- German + English month names, full + abbreviated, with/without separating space
|
||||
(`März`, `Sept.`, `Dez`, `December`, `Oct.`).
|
||||
- 2-digit and 3-digit year normalization (`08`→1908? needs a century rule; `889`→1889).
|
||||
- Partial dates → store what's known. The schema only has a single `documentDate
|
||||
LocalDate`; **decide** whether to (i) store first-of-month/year, (ii) add a
|
||||
`datePrecision` enum + `dateOriginal` text column, or (iii) keep raw text in a new
|
||||
`documentDateRaw` field and leave `documentate` null when imprecise. Recommendation:
|
||||
preserve the **original string** always (new column) + best-effort parsed date +
|
||||
precision flag, so nothing is lost and the UI can show "ca. 1916".
|
||||
- Unparseable/approximate (`?`, `Herbst 1913`) → keep raw, leave parsed date null, **do
|
||||
not drop the row**.
|
||||
|
||||
**Cross-check:** even after IMP-01 is fixed so the date column is read, IMP-02 still bites.
|
||||
Both must be solved before a real import.
|
||||
|
||||
---
|
||||
|
||||
## IMP-03 — New xlsx has no normalized/ISO date or name columns 🔴 BLOCKER
|
||||
|
||||
The ODS had helper columns the importer relied on: `Von`/`An` (normalized names) and
|
||||
`Datum` (ISO) alongside `Datum Originalformat`. The new xlsx has **only the raw**
|
||||
`BriefeschreiberIn` / `EmpfängerIn` / `Datum des Briefes`. So:
|
||||
- Names must be parsed from raw strings (PersonNameParser already does receivers; **sender
|
||||
is taken raw, never split** — fine for senders, which are single, but no normalization).
|
||||
- Dates must be parsed from raw (IMP-02).
|
||||
|
||||
This is the root reason IMP-01/02 exist: the new file is the *uncurated* source, not the
|
||||
hand-normalized ODS. Tie any importer redesign to this reality — we will not get clean
|
||||
helper columns in the 7k-row file.
|
||||
|
||||
---
|
||||
|
||||
## IMP-04 — Person register not imported at all 🟠 MAJOR
|
||||
|
||||
`Personendatei 2.xlsx` → sheet `Tabelle1`, **163 people**, columns:
|
||||
`Generation, Familienname, Vorname, geb als (maiden), Geburtsdatum, Geburtsort,
|
||||
Todesdatum, Sterbeort, verheiratet mit, Bemerkung`.
|
||||
|
||||
Today `MassImportService` has **no person-register import**. Persons are only
|
||||
auto-created as bare aliases from the document sender/receiver strings
|
||||
(`personService.findOrCreateByAlias`). All this rich genealogical data is unused:
|
||||
- birth/death dates + places,
|
||||
- maiden names (the key to dedup — see IMP-05),
|
||||
- `verheiratet mit` (marriage links → `PersonRelationship` domain),
|
||||
- `Bemerkung` relationship hints (`"Schwester v Marie Cram"`, `"Nichte von Herbert"`),
|
||||
- `Generation` (G 1–G 4),
|
||||
- nicknames in quotes (`"Tante Lolly"`).
|
||||
|
||||
Data-quality notes in this file too: multi-value `Vorname` (`Charlotte,Meta,Jacobi`);
|
||||
mixed Excel-date vs text dates; typos (`4.3.1023`); missing-day dates (`.12.1955`);
|
||||
trailing spaces (`30.8.1862 `).
|
||||
|
||||
**Proposed approach:** a separate **Person import** (→ Gitea issue). Order matters: import
|
||||
persons *first* so documents can link to real people instead of creating alias stubs.
|
||||
Use `geb als` + `verheiratet mit` to pre-build the alias/relationship graph.
|
||||
|
||||
---
|
||||
|
||||
## IMP-05 — Name variations create duplicate Persons 🟠 MAJOR
|
||||
|
||||
The same person appears under several surface forms across the document sheet:
|
||||
- `Eugenie Müller` (151) vs `Eugenie de Gruyter` (452) — maiden vs married.
|
||||
- `Clara Cram` (sender 1,284) vs `Clara de Gruyter` (455) vs `Clara de Gruyter sen.` (66).
|
||||
- `Walter de Gruyter` (589) vs bare `Walter` (78).
|
||||
|
||||
`findOrCreateByAlias` keys on the raw string, so each variant becomes (or matches) a
|
||||
distinct alias and likely a **distinct Person**. Result: fragmented person records,
|
||||
broken Briefwechsel pairing, wrong stats.
|
||||
|
||||
**Proposed approach:** drive dedup from the register's `geb als` column (IMP-04) —
|
||||
`Eugenie de Gruyter geb Müller` tells us the two strings are one person. Build an alias
|
||||
map (married ↔ maiden ↔ nickname) before/while importing documents. This is partly data
|
||||
(an alias mapping table/sheet) and partly code (consume it). Likely a Gitea issue once the
|
||||
mapping format is decided.
|
||||
|
||||
945 distinct sender strings / 274 distinct receiver strings — expect a long-tail of
|
||||
variants to reconcile. Don't try to be perfect on the first pass; get the high-frequency
|
||||
names right.
|
||||
|
||||
---
|
||||
|
||||
## IMP-06 — 93 data rows with blank Index are silently dropped 🟠 MAJOR
|
||||
|
||||
`processRows` does `if (index.isBlank()) continue;`. **93 rows** have a blank Index but
|
||||
carry other data (sender/receiver/date/etc.). These are silently skipped — they don't even
|
||||
appear in the `skippedFiles` report (that list only covers rows that *had* an index but
|
||||
failed file checks).
|
||||
|
||||
**Proposed approach:** before import, triage these 93 rows — are they continuation rows,
|
||||
section markers, or genuine letters missing an ID? At minimum, surface a count/warning so
|
||||
nothing vanishes unnoticed. Possibly a small importer change to report blank-index skips.
|
||||
|
||||
---
|
||||
|
||||
## IMP-07 — 43 duplicate Index values 🟡 MINOR
|
||||
|
||||
43 Index values repeat (e.g. `W-0388`, `Eu-0332`, `C-0234`, `C-0235`, `C-0236`, `J-0175`).
|
||||
Since the filename is derived from Index, the importer's upsert keys both rows on the same
|
||||
`originalFilename`: the second occurrence is treated as `ALREADY_EXISTS` (if the first
|
||||
isn't a placeholder) and **its metadata is lost**, or it overwrites a placeholder.
|
||||
|
||||
**Proposed approach:** list the 43 duplicates, check whether they're true duplicates or
|
||||
two distinct letters that share an ID by mistake. Fix in the source data, or extend the ID
|
||||
scheme. Data task first; software only if the ID scheme must change.
|
||||
|
||||
---
|
||||
|
||||
## IMP-08 — Section/title rows interleaved with data 🟡 MINOR
|
||||
|
||||
Row 2 of the sheet is a section header sitting only in the sender column
|
||||
(`Brautbriefe von Walter der Gruyter an Eugenie Müller`) with a blank Index — caught by the
|
||||
blank-Index skip (overlaps IMP-06). There may be more such banners scattered through 7,943
|
||||
rows. Also relevant: the per-letter **keywords live in `Inhalt` (col 9)** as comma-joined
|
||||
values (`Tilburg,Verwandschaft`, `poetisch,Reise nach Breda`), while `Schlagwort` (col 8)
|
||||
holds a single broad tag (`Brautbriefe`). The importer only takes **one** tag column —
|
||||
decide which column feeds tags vs summary, and whether to split comma-lists into multiple
|
||||
tags.
|
||||
|
||||
**Proposed approach:** scan for rows where Index is blank but other cells are set (already
|
||||
have the count: relates to the 93 in IMP-06). Confirm tag vs summary column choice with
|
||||
Marcel.
|
||||
|
||||
---
|
||||
|
||||
## IMP-09 — Index ↔ Datei filename mismatches 🟡 MINOR
|
||||
|
||||
The `Datei` column (col 1) holds explicit relative paths (`..\__scan\W-0001.pdf`) but they
|
||||
don't always agree with the Index. Example: row 20 has Index `W-0010x` but Datei
|
||||
`..\__scan\W-0011x.pdf`. The importer derives the filename from **Index**, so it will look
|
||||
for `W-0010x.pdf` and may miss the actual scan. (Note: the `Datei` paths themselves are
|
||||
Windows-style with `\` and `..` and would be **rejected** by `isValidImportFilename` if anyone
|
||||
tried to use that column directly — 7,623 rows use backslashes, 7,455 contain `..`.)
|
||||
|
||||
**Proposed approach:** when the PDFs arrive, reconcile Index-derived names against actual
|
||||
filenames; produce a mismatch report. Keep deriving from Index (stable IDs) but flag
|
||||
disagreements. Mostly a data/QA task.
|
||||
|
||||
---
|
||||
|
||||
## IMP-10 — `x`-suffix rows (letter backsides / enclosures) 🟡 MINOR
|
||||
|
||||
**42 rows** have an `x`-suffixed Index (`W-0001x`, `W-0002x`, …). They're sparse — typically
|
||||
only Index + Datei + sender + receiver, no box/folder/date. They appear to be the reverse
|
||||
side or an enclosure of the preceding letter. The importer treats each as an independent
|
||||
Document, and the `metadataComplete` heuristic flags them complete as soon as a sender is
|
||||
present (date/box/folder all missing).
|
||||
|
||||
**Proposed approach:** decide whether `x` rows should be (a) separate documents, (b) extra
|
||||
pages/files attached to their parent, or (c) skipped. Affects both the data model and the
|
||||
`metadataComplete` heuristic. Discuss with Marcel.
|
||||
|
||||
---
|
||||
|
||||
## IMP-11 — Multi-receiver separators include bare `u` / `u.` 🟡 MINOR
|
||||
|
||||
`PersonNameParser.parseReceivers` already handles ` und `, ` u `, `//`, `geb.`,
|
||||
parenthesised shared surnames, and `Familie` filtering — good. But the real data also uses
|
||||
the abbreviation in forms the top-receivers list shows are common:
|
||||
`Eugenie u Walter de Gruyter` (230), `Herbert u Clara` (94), `Juan u Marie Cram` (75),
|
||||
and space-joined pairs like `Ella Anita` (79) that may be two people.
|
||||
Raw separator tally on receivers: ` und ` ×70, `,` ×11, `;` ×2, `/` ×1 — plus the many ` u `
|
||||
cases above. Senders are **not** parsed at all (taken raw), which is fine unless a sender
|
||||
cell ever holds two names.
|
||||
|
||||
**Proposed approach:** add `MassImportServiceTest` cases for the real-world strings above;
|
||||
extend the parser only where it actually fails. `Ella Anita`-style space-joined pairs are
|
||||
ambiguous — likely leave as one person unless the register says otherwise (ties to IMP-05).
|
||||
|
||||
---
|
||||
|
||||
## IMP-12 — Importer reads only the first sheet, no validation 🟡 MINOR
|
||||
|
||||
`readXlsx` does `workbook.getSheetAt(0)`. For the new xlsx that's `Familienarchiv` (✅), but
|
||||
the file also contains `Inhaltsverzeichnis grob`, `Inhaltsverzeichnis WdG`, `Tabelle4`.
|
||||
There is no header validation: if the wrong file/sheet is dropped in `/import`, the importer
|
||||
will happily map columns positionally and import nonsense. Also `findSpreadsheetFile()` picks
|
||||
the **first** spreadsheet found in `/import` — with three spreadsheets present there today,
|
||||
which one wins is filesystem-order-dependent.
|
||||
|
||||
**Proposed approach:** (a) validate the header row against expected names before importing;
|
||||
(b) make the target sheet/file explicit (config or header match) rather than "first found".
|
||||
Ties into the header-driven mapping in IMP-01(b).
|
||||
|
||||
---
|
||||
|
||||
## Summary of recommended sequencing
|
||||
|
||||
1. **Decide the importer mapping strategy** (IMP-01): positional re-config vs header-driven.
|
||||
Header-driven is the durable choice and unblocks IMP-03/12.
|
||||
2. **Build the tolerant date parser** (IMP-02) with original-string preservation + precision.
|
||||
3. **Import the Person register first** (IMP-04) and build the alias/marriage graph,
|
||||
which feeds person dedup (IMP-05).
|
||||
4. **Then import documents**, with reporting for blank-index (IMP-06), duplicates (IMP-07),
|
||||
and section rows (IMP-08).
|
||||
5. **Reconcile files** when the ~7,000 PDFs arrive (IMP-09), and decide `x`-row semantics
|
||||
(IMP-10).
|
||||
|
||||
Code-change items (→ Gitea issues when we get there): IMP-01(b), IMP-02, IMP-04, IMP-05
|
||||
(consume side), IMP-06 reporting, IMP-12. Pure-data items stay in this folder.
|
||||
386
docs/import-migration/02-normalization-spec.md
Normal file
386
docs/import-migration/02-normalization-spec.md
Normal file
@@ -0,0 +1,386 @@
|
||||
# Spec — Import Normalizer
|
||||
|
||||
> Authored in the voice of **"Elicit"**, requirements engineer (see
|
||||
> `.claude/personas/req_engineer.md`). This is a requirements artifact: it states
|
||||
> *what* the normalizer must do and *how we'll know it's done*, in problem/behaviour
|
||||
> language. Technology choices already made during brainstorming (Python, openpyxl,
|
||||
> overrides-and-rerun) are recorded as **constraints**, not re-litigated here.
|
||||
|
||||
- **Status:** Draft for review
|
||||
- **Date:** 2026-05-25
|
||||
- **Related:** [`01-findings-spreadsheet-analysis.md`](./01-findings-spreadsheet-analysis.md) (issues `IMP-01..12`), [`README.md`](./README.md)
|
||||
- **Scope boundary:** This spec covers the **offline normalizer** that turns the raw
|
||||
spreadsheets into a clean, canonical dataset + review artifacts. Wiring the canonical
|
||||
contract into the Java `MassImportService` and the `Document`/`Person` model is **Phase 2**
|
||||
and gets its own spec. This spec only *defines the contract* Phase 2 must satisfy.
|
||||
|
||||
---
|
||||
|
||||
## 1. Project Brief
|
||||
|
||||
**Vision.** Turn the family's human-curated, free-form archive spreadsheets into a clean,
|
||||
canonical dataset that imports deterministically — without hand-editing thousands of rows
|
||||
and without losing the historical nuance of how things were originally written.
|
||||
|
||||
**Problem.** The real archive (`…aktuell…xlsx`, 7,943 rows) and the person register
|
||||
(`Personendatei 2.xlsx`, 163 people) were authored for humans to read, not machines to
|
||||
import. Dates are written as they appeared in each letter (≈90% unparseable by the current
|
||||
importer), the column layout differs from what the importer expects, and the same person
|
||||
appears under many names. Importing as-is produces garbage (see `IMP-01..12`).
|
||||
|
||||
**Goal (measurable).**
|
||||
- G1 — After the automated pass, **≤ 5%** of dated rows remain `UNKNOWN`; after the
|
||||
overrides-iteration loop, **≤ 0.5%**.
|
||||
- G2 — **100%** of source rows are represented in the canonical output or in a review file —
|
||||
*zero silent drops*.
|
||||
- G3 — **100%** of original values (raw date string, raw name string, source row number)
|
||||
are preserved.
|
||||
- G4 — A full run over the current inputs completes in **< 60 s** on the dev laptop and is
|
||||
**content-deterministic** when re-run with unchanged inputs+overrides: identical canonical
|
||||
cell matrices and identical review-file contents. (Workbook metadata is pinned; literal xlsx
|
||||
byte-identity is not guaranteed because the zip container stores entry metadata.)
|
||||
|
||||
**Primary actor.** Marcel — solo owner & data steward (tech comfort 4/5). Also: a future
|
||||
agent re-running the pipeline; and the `MassImportService` as the downstream consumer.
|
||||
|
||||
**Non-Goals (explicitly out of scope).**
|
||||
- NG1 — Changing `MassImportService` or the DB schema (that is Phase 2).
|
||||
- NG2 — Uploading/attaching the ~7,000 PDFs (they arrive later; import matches by `index`).
|
||||
- NG3 — A GUI. The interface is spreadsheets in, CSVs out, an overrides file hand-edited.
|
||||
- NG4 — Perfect genealogical reconstruction. We resolve confidently-matchable people; the
|
||||
long tail stays as provisional persons.
|
||||
- NG5 — OCR/transcription content (the new xlsx has no transcription column).
|
||||
|
||||
**Key assumptions.** (A1) Sheet `Familienarchiv` is the document source of truth.
|
||||
(A2) Archive date range is **1873–1957** (drives the 2-digit-year century rule).
|
||||
(A3) `index` is the stable document key and the basis for future PDF matching.
|
||||
(A4) `Schlagwort` is a broad tag; `Inhalt` is a short summary/topic.
|
||||
|
||||
**Risks.** (R1) 2-digit/partial dates are genuinely ambiguous → mitigated by precision flag
|
||||
+ overrides. (R2) Name matching false-positives merge distinct people → mitigated by
|
||||
conservative matching + review before merge. (R3) Source spreadsheet may be re-exported with
|
||||
layout drift → mitigated by header-name-based mapping, not fixed indices.
|
||||
|
||||
---
|
||||
|
||||
## 2. Personas
|
||||
|
||||
**Marcel — Data Steward.** Role: solo owner of Familienarchiv. Context: holds the complete
|
||||
raw archive; PDFs follow. Tech comfort: 4/5 (semi-technical, reads CSV/spreadsheets fluently,
|
||||
not keen to hand-edit 7,600 rows). Primary goal: a clean, importable dataset he trusts.
|
||||
Frustrations: dates in ~20 formats; one ancestor under 4 name variants. **JTBD:** *"When I
|
||||
have raw, human-curated archive spreadsheets, I want to transform them into a clean importable
|
||||
dataset without losing how things were originally written, so I can load the archive and keep
|
||||
correcting edge cases as they surface."*
|
||||
|
||||
**The Returning Agent.** Role: a future assistant session resuming the work. Goal: re-run the
|
||||
pipeline deterministically and understand exactly what still needs human input. **JTBD:**
|
||||
*"When I pick this up cold, I want one command and a clear residue report, so I can continue
|
||||
without re-deriving context."*
|
||||
|
||||
---
|
||||
|
||||
## 3. Constraints & Decisions Already Made
|
||||
|
||||
These were settled during brainstorming and are fixed inputs to the requirements below.
|
||||
|
||||
| # | Decision | Rationale |
|
||||
| --- | --- | --- |
|
||||
| C1 | **New canonical layout** with explicit headers (not the old positional ODS shape). | Fits the new data; importer becomes header-driven in Phase 2. |
|
||||
| C2 | Dates stored as **parsed (nullable) + raw + precision**. | Historical archive; never lose the original; enable "ca. 1916". |
|
||||
| C3 | **Include person resolution** (register + alias/marriage map → canonical persons) in this effort. | Maiden-name dedup needs the register. |
|
||||
| C4 | **Overrides-file + re-run** loop for residue. | Deterministic, diffable, repeatable. |
|
||||
| C5 | Implementation: **Python 3.12 + openpyxl**, standalone tool at `tools/import-normalizer/`. | Fast iteration; no Spring rebuild / coverage gate on transform code. |
|
||||
| C6 | Century rule for archive **1873–1957**: 2-digit `00–57`→`19YY`, `73–99`→`18YY`, `58–72`→**flag**; 3-digit `DDD`→`1DDD`; never 20xx. | Stated by Marcel. Boundaries live in config. |
|
||||
| C7 | `Schlagwort`→tag, `Inhalt`→summary. | Matches importer's existing semantics. |
|
||||
| C8 | Non-register correspondents become **provisional persons**. | ~945 distinct sender strings vs 163 register people. |
|
||||
|
||||
---
|
||||
|
||||
## 4. Functional Requirements
|
||||
|
||||
Each requirement has a stable ID. User stories use Connextra + Given-When-Then; system rules
|
||||
use EARS. Traceability to findings in §8.
|
||||
|
||||
### 4.1 Ingest & layout (`FR-INGEST`, `FR-MAP`)
|
||||
|
||||
**US-MAP-01** — *As the data steward, I want each source column mapped to a named canonical
|
||||
field regardless of its position, so a re-exported spreadsheet with shifted columns still
|
||||
imports correctly.*
|
||||
- AC1 — Given the `Familienarchiv` sheet, when the normalizer reads the header row, then it
|
||||
maps columns by **header name** (not fixed index) to the canonical fields.
|
||||
- AC2 — Given a header the normalizer does not recognise, when it runs, then it records the
|
||||
unknown header in `review/summary.txt` and continues (does not crash).
|
||||
- AC3 — Given a required source header is **absent**, when it runs, then it aborts with a
|
||||
clear message naming the missing header (fail loud, before producing partial output).
|
||||
|
||||
- **REQ-INGEST-01** — The normalizer shall read only the `Familienarchiv` sheet of the
|
||||
document workbook and the `Tabelle1` sheet of the person workbook.
|
||||
- **REQ-MAP-01** — Header matching shall be case-insensitive and tolerant of internal
|
||||
multiple spaces (e.g. `"Datum des Briefes"`).
|
||||
|
||||
### 4.2 Row triage (`FR-TRIAGE`) — resolves IMP-06, IMP-07, IMP-08
|
||||
|
||||
**US-TRIAGE-01** — *As the data steward, I want rows that have data but no index surfaced
|
||||
rather than dropped, so I never lose a letter silently.*
|
||||
- AC1 — Given a row whose `index` is blank but which has any other non-empty cell, when the
|
||||
normalizer runs, then that row is written to `review/blank-index-rows.csv` with its source
|
||||
row number and is **not** emitted as a canonical document.
|
||||
- AC2 — Given a fully empty row, when it runs, then the row is skipped and counted (not
|
||||
reported as an anomaly).
|
||||
|
||||
- **REQ-TRIAGE-01** — If two or more rows resolve to the same `index`, then the normalizer
|
||||
shall emit all of them to `review/duplicate-index.csv` and mark each canonical row
|
||||
`needs_review = duplicate_index` (it shall **not** silently drop either).
|
||||
- **REQ-TRIAGE-02** — Where a row is identified as a section/banner row (blank index, text
|
||||
only in a name column), the normalizer shall classify it as such in the blank-index report.
|
||||
- **REQ-TRIAGE-03** — Rows whose `index` ends in `x` (a transcription/back-side of the base
|
||||
letter, not yet independently mappable) shall be **skipped** — not emitted as a canonical
|
||||
document — and written to `review/skipped-x-suffix.csv` with their source row and base index
|
||||
(`index` minus the trailing `x`), so they can be linked in a later pass. (Resolves IMP-10.)
|
||||
|
||||
### 4.3 Date normalization (`FR-DATE`) — resolves IMP-02, IMP-03
|
||||
|
||||
**US-DATE-01** — *As the data steward, I want every date interpreted as precisely as the
|
||||
source allows, with the original always kept, so I can sort the archive and still see what the
|
||||
letter actually said.*
|
||||
- AC1 — Given a parseable date, when normalized, then `date_iso` holds the best-effort ISO
|
||||
date, `date_raw` holds the verbatim source string, and `date_precision` ∈
|
||||
`{DAY, MONTH, SEASON, YEAR, RANGE, APPROX, UNKNOWN}`.
|
||||
- AC2 — Given an unparseable date, when normalized, then `date_iso` is empty,
|
||||
`date_precision = UNKNOWN`, `date_raw` is preserved, and the value appears in
|
||||
`review/unparsed-dates.csv`.
|
||||
- AC3 — Given the same `date_raw` appears in `overrides/dates.csv`, when normalized, then the
|
||||
override's `(iso, precision)` wins over the automatic parse.
|
||||
|
||||
- **REQ-DATE-01** — The parser shall accept, at minimum, these forms (see §10 examples):
|
||||
Excel/ISO; `D.M.YYYY`/`D.M.YY`; `D/M. YY[YY]` (slash treated as dot); Roman-numeral months
|
||||
`I–XII`; German + English month names, full and abbreviated, with or without a separating
|
||||
space; `Month YYYY`; season/holiday + year; bare `YYYY`; and start-anchored ranges.
|
||||
- **REQ-DATE-02** — Precision shall be assigned by what is known: full day → `DAY`; month+year
|
||||
→ `MONTH` (day = 1); a **named feast/holiday + year** → resolved to its **actual calendar
|
||||
date for that year** → `DAY`; a **season + year** → representative mid-season month (day = 1)
|
||||
→ `SEASON`; year only → `YEAR` (month = Jan, day = 1); a range → start date + `RANGE`; a
|
||||
value carrying an uncertainty marker (`?`, `um`, `ca`, `circa`) → `APPROX` with best-effort date.
|
||||
- **REQ-DATE-03** — Two-digit and three-digit years shall be expanded per **C6**; a 2-digit
|
||||
year in `58–72` shall yield `UNKNOWN` + a review entry rather than a guess.
|
||||
- **REQ-DATE-04** — Trailing editorial notes (e.g. `", 2. Brief"`) shall be stripped before
|
||||
parsing and preserved (kept within `date_raw`; not invented into the date).
|
||||
- **REQ-DATE-05** — The parser shall be pure and side-effect-free so it can be unit-tested in
|
||||
isolation (see NFR-TEST-01).
|
||||
- **REQ-DATE-06** — **Movable feasts are never mapped to a fixed month**; they shall be
|
||||
computed per year from Easter (Gauss/Butcher computus): Karfreitag = Easter−2, Ostern =
|
||||
Easter Sunday, Himmelfahrt = Easter+39, Pfingst(sonntag) = Easter+49, Pfingstmontag =
|
||||
Easter+50, Fronleichnam = Easter+60, 1.–4. Advent = the 4th…1st Sunday before 25 Dec. Fixed
|
||||
feasts use a lookup table (Neujahr=01-01, Heiligabend=12-24, Weihnachten=12-25,
|
||||
Silvester=12-31, …). Seasons map to representative months: Frühling/Frühjahr=Apr, Sommer=Jul,
|
||||
Herbst=Oct, Winter=Jan. The feast/season tables and Easter algorithm live in `config.py`
|
||||
(NFR-MAINT-01).
|
||||
|
||||
### 4.4 Person resolution & dedup (`FR-PERS`, `FR-DEDUP`) — resolves IMP-04, IMP-05, IMP-11
|
||||
|
||||
**US-PERS-01** — *As the data steward, I want the genealogical register turned into canonical
|
||||
people with all their known facts, so documents can link to real persons.*
|
||||
- AC1 — Given a register row, when parsed, then a canonical person is produced with
|
||||
`person_id`, name parts, `maiden_name`, birth/death (parsed + raw + place), spouse,
|
||||
generation, nickname, notes — applying the same date rules as §4.3 to birth/death dates.
|
||||
- AC2 — Given multi-value given names (`"Charlotte,Meta,Jacobi"`), when parsed, then the
|
||||
primary given name is the first; the remainder are retained as additional names/aliases.
|
||||
|
||||
**US-PERS-02** — *As the data steward, I want each sender/receiver string matched to a
|
||||
canonical person where possible and never dropped otherwise, so the correspondence graph is
|
||||
complete.*
|
||||
- AC1 — Given a sender/receiver string, when resolved, then it maps to a register
|
||||
`person_id` via the alias index (exact → normalized/casefold → conservative fuzzy).
|
||||
- AC2 — Given no confident match, when resolved, then a **provisional person** is created from
|
||||
the cleaned string, linked, and listed in `review/unmatched-names.csv` (occurrence count +
|
||||
example source rows).
|
||||
- AC3 — Given the string appears in `overrides/names.csv`, when resolved, then it maps to the
|
||||
specified `person_id` (override wins).
|
||||
- AC4 — Given a multi-person receiver cell (`"Eugenie u Walter de Gruyter"`, `"Herbert u
|
||||
Clara"`, `"…//…"`, `"Hedi und Tutu (Gruber)"`), when resolved, then it is split into
|
||||
individual people, each resolved independently; ambiguous space-joined pairs
|
||||
(`"Ella Anita"`) are emitted to `review/ambiguous-receivers.csv` rather than guessed.
|
||||
|
||||
- **REQ-DEDUP-01** — The alias index shall be derived from the register: canonical
|
||||
"First Last", maiden form (`geb als`), spouse-surname married form, nickname, and
|
||||
first-name-only **only when unambiguous** across the register.
|
||||
- **REQ-DEDUP-02** — The normalizer shall not merge two distinct strings into one person on
|
||||
fuzzy similarity alone above a configured threshold without the match being reported; merges
|
||||
must be auditable.
|
||||
- **REQ-PERS-01** — Sender cells shall be parsed for multi-person content using the same rules
|
||||
as receiver cells (today the importer parses only receivers — IMP-11).
|
||||
|
||||
### 4.5 Overrides & idempotency (`FR-OVR`) — supports the iteration loop
|
||||
|
||||
- **REQ-OVR-01** — When the normalizer runs, then it shall load `overrides/dates.csv` and
|
||||
`overrides/names.csv` if present and apply them; absence of either file shall not be an error.
|
||||
- **REQ-OVR-02** — While overrides are unchanged and inputs are unchanged, re-running shall
|
||||
produce **byte-identical** canonical outputs and review files (NFR-IDEM-01).
|
||||
- **REQ-OVR-03** — Each override application shall be counted in `review/summary.txt` (how many
|
||||
dates/names were resolved by override vs automatically).
|
||||
|
||||
### 4.6 Canonical output & provenance (`FR-OUT`, `FR-PROV`) — resolves IMP-01, IMP-09, IMP-12
|
||||
|
||||
- **REQ-OUT-01** — The normalizer shall write `out/canonical-documents.xlsx` and
|
||||
`out/canonical-persons.xlsx` with the headered schemas in §6.
|
||||
- **REQ-PROV-01** — Every canonical document row shall carry `source_row` (1-based row number
|
||||
in the source sheet) so any value can be traced back to the original.
|
||||
- **REQ-PROV-02** — Every canonical row shall carry a `needs_review` field listing zero or more
|
||||
flags (`duplicate_index`, `unparsed_date`, `unmatched_sender`, `unmatched_receiver`,
|
||||
`index_file_mismatch`, …) so the import and the UI can foreground uncertain data.
|
||||
- **REQ-OUT-02** — Where the source `Datei` path disagrees with the index-derived filename
|
||||
(IMP-09), the normalizer shall record the discrepancy in `review/index-file-mismatch.csv`
|
||||
and flag the row; it shall **not** alter the `index` (the stable key).
|
||||
|
||||
---
|
||||
|
||||
## 5. Non-Functional Requirements
|
||||
|
||||
| ID | Category | Requirement (measurable) |
|
||||
| --- | --- | --- |
|
||||
| NFR-DATA-01 | Data integrity | 100% of source rows are accounted for in output **or** a review file; 100% of original date/name strings preserved verbatim. |
|
||||
| NFR-IDEM-01 | Determinism | Identical inputs + overrides ⇒ identical *logical* output across runs/machines: identical canonical cell matrices and review-file contents. Workbook `created`/`modified` metadata is pinned to a constant; ordering of all generated rows/aliases is stable (no set-iteration leakage). xlsx byte-identity is explicitly not required — determinism is asserted on content. |
|
||||
| NFR-PERF-01 | Performance | Full run over 7,943 doc rows + 163 person rows completes in < 60 s on the dev laptop. |
|
||||
| NFR-ACCUR-01 | Date accuracy | After automated pass, `UNKNOWN` dates ≤ 5% of dated rows; after overrides iteration, ≤ 0.5%. |
|
||||
| NFR-ACCUR-02 | Name coverage | Every sender/receiver occurrence yields a linked person (register or provisional); 0 dropped. |
|
||||
| NFR-I18N-01 | Encoding | UTF-8 end-to-end; German diacritics and ß round-trip with no mojibake in any output. |
|
||||
| NFR-TEST-01 | Testability | `dates.py` and `persons.py` have pytest tests covering every format/alias category in §10 with real examples from the archive. |
|
||||
| NFR-MAINT-01 | Maintainability | Column-name map, century boundaries, season→month map, and fuzzy threshold live in `config.py`, not inline in logic. |
|
||||
| NFR-OBSERV-01 | Observability | `review/summary.txt` reports per-run stats: rows in, documents out, dates by precision, names matched vs provisional, overrides applied, anomalies by type. |
|
||||
| NFR-SAFETY-01 | Source safety | Source workbooks are opened read-only and never written. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Data Dictionary (canonical contract)
|
||||
|
||||
This is the contract Phase 2 (the importer) must consume. Field-level, format-level — not a
|
||||
DB schema.
|
||||
|
||||
### 6.1 `canonical-documents.xlsx`
|
||||
|
||||
| Field | Required | Format / values | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `index` | yes | string | Stable key; basis for PDF matching. |
|
||||
| `box` | no | string | from `Box`. |
|
||||
| `folder` | no | string | from `Mappe`. |
|
||||
| `sender_person_id` | no | person_id | resolved; empty if no sender. |
|
||||
| `sender_name` | no | string | canonical display name (or cleaned raw if provisional). |
|
||||
| `receiver_person_ids` | no | `id\|id\|…` | pipe-separated. |
|
||||
| `receiver_names` | no | `name\|name\|…` | pipe-separated, aligned with ids. |
|
||||
| `date_iso` | no | `YYYY-MM-DD` | best-effort; empty if `UNKNOWN`. |
|
||||
| `date_raw` | no | string | verbatim source date. |
|
||||
| `date_precision` | yes | enum | `DAY\|MONTH\|SEASON\|YEAR\|RANGE\|APPROX\|UNKNOWN`. |
|
||||
| `location` | no | string | from `Ort`. |
|
||||
| `tags` | no | `tag\|tag` | from `Schlagwort`. |
|
||||
| `summary` | no | string | from `Inhalt`. |
|
||||
| `source_row` | yes | int | provenance (NFR-DATA-01). |
|
||||
| `needs_review` | yes | `flag\|flag` or empty | review flags (REQ-PROV-02). |
|
||||
|
||||
### 6.2 `canonical-persons.xlsx`
|
||||
|
||||
| Field | Required | Format | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `person_id` | yes | slug | stable id (e.g. `de-gruyter-eugenie`); collisions suffixed. |
|
||||
| `last_name` | yes | string | from `Familienname`. |
|
||||
| `first_name` | no | string | primary given name. |
|
||||
| `maiden_name` | no | string | from `geb als` — drives dedup. |
|
||||
| `title` | no | string | e.g. honorifics if present. |
|
||||
| `nickname` | no | string | from quoted `Bemerkung`/spouse field. |
|
||||
| `birth_date` / `birth_date_raw` / `birth_place` | no | ISO / string / string | §4.3 rules. |
|
||||
| `death_date` / `death_date_raw` / `death_place` | no | ISO / string / string | §4.3 rules. |
|
||||
| `spouse` | no | person_id or name | from `verheiratet mit`. |
|
||||
| `generation` | no | string | `G 1`..`G 4`. |
|
||||
| `notes` | no | string | from `Bemerkung`. |
|
||||
| `aliases` | no | `a\|b\|c` | every surface form that maps here. |
|
||||
| `provisional` | yes | bool | true if created from a document string, not the register. |
|
||||
|
||||
---
|
||||
|
||||
## 7. Prioritized Backlog (MoSCoW)
|
||||
|
||||
| ID | Item | MoSCoW | Effort | Depends on |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| B1 | Project scaffolding + read both workbooks (`FR-INGEST`, header map `FR-MAP`) | Must | S | — |
|
||||
| B2 | Row triage + blank/duplicate/empty reports (`FR-TRIAGE`) | Must | S | B1 |
|
||||
| B3 | Date parser + precision + century rule + Easter/feast computus + season map + tests (`FR-DATE`) | Must | L | B1 |
|
||||
| B4 | Person register parser → canonical persons (`FR-PERS` US-PERS-01) | Must | M | B1 |
|
||||
| B5 | Alias index + name resolution + multi-person split (`FR-DEDUP`, US-PERS-02) | Must | L | B4 |
|
||||
| B6 | Overrides load + apply + idempotency (`FR-OVR`) | Must | S | B3,B5 |
|
||||
| B7 | Canonical writers + provenance + review summary (`FR-OUT`, `FR-PROV`) | Must | M | B2,B3,B5 |
|
||||
| B8 | Index↔Datei mismatch report (`REQ-OUT-02`) | Should | XS | B1 |
|
||||
| B9 | Ambiguous-receiver review path (US-PERS-02 AC4) | Should | S | B5 |
|
||||
| B10 | Comma-split `Inhalt` into extra tags | Could | XS | B7 |
|
||||
| B11 | Phase-2 importer wiring (separate spec) | Won't (this spec) | — | B7 |
|
||||
|
||||
---
|
||||
|
||||
## 8. Traceability — Findings → Requirements
|
||||
|
||||
| Finding | Severity | Addressed by |
|
||||
| --- | --- | --- |
|
||||
| IMP-01 layout mismatch | blocker | C1, FR-MAP, REQ-OUT-01 |
|
||||
| IMP-02 free-text dates | blocker | FR-DATE (all), C2, C6 |
|
||||
| IMP-03 no ISO/normalized cols | blocker | FR-DATE, FR-PERS |
|
||||
| IMP-04 register unimported | major | C3, US-PERS-01, §6.2 |
|
||||
| IMP-05 name variants → dupes | major | C3, FR-DEDUP |
|
||||
| IMP-06 blank-index dropped | major | US-TRIAGE-01 |
|
||||
| IMP-07 duplicate indices | minor | REQ-TRIAGE-01 |
|
||||
| IMP-08 section rows / tags vs summary | minor | REQ-TRIAGE-02, C7 |
|
||||
| IMP-09 index↔file mismatch | minor | REQ-OUT-02, B8 |
|
||||
| IMP-10 `x`-suffix rows | minor | REQ-TRIAGE-03 (skip + log this pass) |
|
||||
| IMP-11 sender not split / ` u ` sep | minor | REQ-PERS-01, US-PERS-02 AC4 |
|
||||
| IMP-12 first-sheet, no validation | minor | REQ-INGEST-01, FR-MAP AC2/AC3 |
|
||||
|
||||
---
|
||||
|
||||
## 9. Open Questions / TBD Register
|
||||
|
||||
| ID | Question | Why it matters | Ref | Resolution |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| OQ-01 ✅ | Season/holiday → date. | Accuracy of ~70 SEASON/feast rows. | REQ-DATE-06 | **Resolved (2026-05-25):** movable feasts (Ostern, Pfingsten, Himmelfahrt, Advent, …) **computed per year from Easter — never a fixed month**; fixed feasts looked up (Weihnachten=12-25, Neujahr=01-01, …); seasons = mid-season month (Frühling=Apr, Sommer=Jul, Herbst=Oct, Winter=Jan). |
|
||||
| OQ-02 ✅ | Date ranges: start only, or start+end? | Sorting/display of ~315 range values. | REQ-DATE-02 | **Confirmed:** store **start** in `date_iso`, precision `RANGE`, full text in `date_raw`. |
|
||||
| OQ-03 ✅ | `person_id` format. | Stability across re-runs; diffability. | §6 | **Confirmed:** readable slug `lastname-firstname`, numeric suffix on collision. |
|
||||
| OQ-04 ✅ | `x`-suffix row handling. | 42 rows. | REQ-TRIAGE-03 | **Resolved (2026-05-25):** `x` rows are transcriptions of the base letter but not yet mappable → **skip this pass**, log to `review/skipped-x-suffix.csv` for later linking. |
|
||||
| OQ-05 ✅ | Importer output format. | Phase-2 reader. | B11 | **Confirmed:** `.xlsx` (openpyxl-native, headered). |
|
||||
| OQ-06 ✅ | Fuzzy-match policy. | False-positive person merges (R2). | REQ-DEDUP-02 | **Confirmed:** conservative — report all fuzzy matches; no silent merge. |
|
||||
|
||||
*All open questions resolved as of 2026-05-25. New ambiguities discovered during build go here.*
|
||||
|
||||
---
|
||||
|
||||
## 10. Glossary & Worked Examples
|
||||
|
||||
**Precision** — how exactly a date is known (`DAY` … `UNKNOWN`). **Provisional person** — a
|
||||
person created from a document name string with no register match. **Alias index** — map from
|
||||
every known surface form of a name to a canonical `person_id`. **Override** — a
|
||||
human-supplied correction applied deterministically on each run.
|
||||
|
||||
**Date examples → expected outcome:**
|
||||
|
||||
| `date_raw` | `date_iso` | `date_precision` |
|
||||
| --- | --- | --- |
|
||||
| `15.2.1888` | 1888-02-15 | DAY |
|
||||
| `6.März 1888` | 1888-03-06 | DAY |
|
||||
| `22.III.18` | 1918-03-22 | DAY |
|
||||
| `13.5.09` | 1909-05-13 | DAY |
|
||||
| `10.Oct.95` | 1895-10-10 | DAY |
|
||||
| `17/6. 1916` | 1916-06-17 | DAY |
|
||||
| `Mai 1895` | 1895-05-01 | MONTH |
|
||||
| `Pfingsten 1922` | 1922-06-04 | DAY (computed: Easter 1922 = Apr 16, +49 days) |
|
||||
| `Herbst 1913` | 1913-10-01 | SEASON |
|
||||
| `1905` | 1905-01-01 | YEAR |
|
||||
| `8.1.1916 - 15.3.1916` | 1916-01-08 | RANGE |
|
||||
| `17.Nov (?) 1887` | 1887-11-17 | APPROX |
|
||||
| `?` | *(empty)* | UNKNOWN |
|
||||
|
||||
**Name examples → expected outcome:**
|
||||
|
||||
| raw cell | resolves to |
|
||||
| --- | --- |
|
||||
| `Eugenie Müller` (+ register `geb Müller`) | `de-gruyter-eugenie` (matched via maiden alias) |
|
||||
| `Eugenie de Gruyter` | `de-gruyter-eugenie` |
|
||||
| `Herbert u Clara` | `cram-herbert` + `cram-clara` (split, surname distributed) |
|
||||
| `Hedi und Tutu (Gruber)` | `gruber-hedi` + `gruber-tutu` |
|
||||
| `Ella Anita` | → `review/ambiguous-receivers.csv` (not auto-split) |
|
||||
| `Hans Wittkopf` (not in register) | provisional `wittkopf-hans` |
|
||||
2281
docs/import-migration/03-normalizer-implementation-plan.md
Normal file
2281
docs/import-migration/03-normalizer-implementation-plan.md
Normal file
File diff suppressed because it is too large
Load Diff
502
docs/import-migration/04-unresolved-names-plan.md
Normal file
502
docs/import-migration/04-unresolved-names-plan.md
Normal file
@@ -0,0 +1,502 @@
|
||||
# Unresolved-Name Classification Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add a focused `review/unresolved-names.csv` that isolates sender/receiver strings whose *name itself* is problematic (unknown/illegible, single-token, relational-only, collective/group, prose-in-name-column, or a genuine two-given-name pair), and fix the ambiguous-pair heuristic so a plain `First Surname` external person (e.g. `Mieze Schefold`) is no longer falsely flagged.
|
||||
|
||||
**Architecture:** A pure `classify_name(raw, given_names)` function in `persons.py` returns a `NameClass`. `ResolutionContext` classifies every *unmatched* name and records the non-`RESOLVABLE` ones in `self.unresolved`. A runtime-built given-name set (register first names + a small config supplement) lets the classifier distinguish a two-given-name pair (`Ella Anita` → two people) from a first+surname single person (`Mieze Schefold`). The orchestrator writes the aggregated report and per-category stats, replacing the noisy `ambiguous-receivers.csv`.
|
||||
|
||||
**Tech Stack:** Python 3.12, openpyxl, pytest — extends the existing `tools/import-normalizer/`.
|
||||
|
||||
**Context:** This builds on the completed normalizer (PR #663). Run all tests with CWD = the tool dir, e.g. `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_X.py -v`. Reuse the existing venv at `tools/import-normalizer/.venv` (do NOT recreate it). Commit on the current branch `docs/import-migration` (never main, never push). Each commit message ends with a trailing `Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>` line.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
tools/import-normalizer/
|
||||
├── config.py # + RELATIONAL_TERMS, COLLECTIVE_TERMS, UNKNOWN_NAME_MARKERS, PROSE_MAX_LEN, EXTRA_GIVEN_NAMES
|
||||
├── persons.py # + NameClass, classify_name(), build_given_names(); ResolutionContext gains given_names + self.unresolved
|
||||
├── normalize.py # writes unresolved-names.csv (replaces ambiguous-receivers.csv) + per-category stats
|
||||
├── README.md # + unresolved-names.csv row in the review-file table
|
||||
└── tests/
|
||||
├── test_config.py # + name-table presence test
|
||||
├── test_persons.py # + classify_name + build_given_names tests
|
||||
├── test_documents.py # ambiguous test → unresolved test (+ resolvable-pair test)
|
||||
└── test_normalize.py # integration asserts unresolved-names.csv
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Config — name-classification tables
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/config.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_config.py`
|
||||
|
||||
- [ ] **Step 1: Add the failing test** to `tests/test_config.py`
|
||||
|
||||
```python
|
||||
def test_name_classification_tables():
|
||||
assert "tante" in config.RELATIONAL_TERMS
|
||||
assert "familie" in config.COLLECTIVE_TERMS
|
||||
assert "unbekannt" in config.UNKNOWN_NAME_MARKERS
|
||||
assert config.PROSE_MAX_LEN >= 30
|
||||
assert "anita" in config.EXTRA_GIVEN_NAMES
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_config.py::test_name_classification_tables -v && cd -`
|
||||
Expected: FAIL — `AttributeError: module 'config' has no attribute 'RELATIONAL_TERMS'`.
|
||||
|
||||
- [ ] **Step 3: Implement** — append to `config.py` (after the existing tables, before/after `KNOWN_LAST_NAMES` — anywhere at module level)
|
||||
|
||||
```python
|
||||
# --- Name classification (unresolved-name review) ---
|
||||
# Relational reference terms — a sender/receiver named by relation, not a proper name.
|
||||
RELATIONAL_TERMS = {
|
||||
"tante", "onkel", "mutter", "vater", "oma", "opa", "großmutter", "grossmutter",
|
||||
"großvater", "grossvater", "schwester", "bruder", "cousin", "cousine", "kusine",
|
||||
"neffe", "nichte", "tochter", "sohn", "schwager", "schwägerin", "schwiegermutter",
|
||||
"schwiegervater", "enkel", "enkelin", "vetter", "base", "witwe", "witwer",
|
||||
}
|
||||
# Collective/group terms — not a single person. Matched against alpha-only word tokens
|
||||
# (so "Fam.Cram" -> ["fam","cram"] matches "fam"), NOT as substrings/prefixes.
|
||||
COLLECTIVE_TERMS = {
|
||||
"familie", "fam", "kinder", "eltern", "geschwister", "großeltern",
|
||||
"grosseltern", "alle", "diverse", "div", "gebrüder", "gebr",
|
||||
}
|
||||
# Markers of an unknown/illegible name (the literal "?" is handled separately in code).
|
||||
# All long enough to be safe as SUBSTRING matches — do NOT add short tokens like "nn"
|
||||
# (it occurs inside real names: Hanni, Johanna, Anna).
|
||||
UNKNOWN_NAME_MARKERS = {"unbekannt", "unbek", "unleserlich", "unklar", "unsicher"}
|
||||
# A name-column value longer than this (chars) is treated as prose/description, not a name.
|
||||
PROSE_MAX_LEN = 40
|
||||
# Common given names that may appear in two-given-name pairs (e.g. "Ella Anita") but are not
|
||||
# in the family register. Only used to detect AMBIGUOUS_PAIR — extend as review surfaces more.
|
||||
EXTRA_GIVEN_NAMES = {
|
||||
"ella", "anita", "kurt", "georg", "hanni", "mieze", "ellen", "leni", "klara",
|
||||
"margret", "gustava", "emmy", "minna", "sophie", "helga", "raymonde", "augusta",
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_config.py -v && cd -`
|
||||
Expected: PASS (all config tests).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/config.py tools/import-normalizer/tests/test_config.py
|
||||
git commit -m "feat(normalizer): config tables for name classification"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `classify_name` + `NameClass`
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/persons.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_persons.py`
|
||||
|
||||
- [ ] **Step 1: Add failing tests** to `tests/test_persons.py`
|
||||
|
||||
```python
|
||||
from persons import NameClass
|
||||
|
||||
GIVEN = {"ella", "anita", "kurt", "georg", "clara", "eugenie"}
|
||||
|
||||
def test_classify_unknown():
|
||||
assert persons.classify_name("?", GIVEN) is NameClass.UNKNOWN
|
||||
assert persons.classify_name("A. Kredell?", GIVEN) is NameClass.UNKNOWN
|
||||
assert persons.classify_name("unbekannt", GIVEN) is NameClass.UNKNOWN
|
||||
|
||||
def test_classify_prose():
|
||||
assert persons.classify_name("Adressenliste v Clara Cram zur Kondolenz", GIVEN) is NameClass.PROSE
|
||||
assert persons.classify_name("Clara de Gruyter(*1871)", GIVEN) is NameClass.PROSE # digit
|
||||
assert persons.classify_name('"Cramiade" Gedicht', GIVEN) is NameClass.PROSE # quote
|
||||
|
||||
def test_classify_collective():
|
||||
assert persons.classify_name("Familie", GIVEN) is NameClass.COLLECTIVE
|
||||
assert persons.classify_name("Fam.Cram", GIVEN) is NameClass.COLLECTIVE
|
||||
assert persons.classify_name("Eltern Cram", GIVEN) is NameClass.COLLECTIVE
|
||||
assert persons.classify_name("seine Kinder", GIVEN) is NameClass.COLLECTIVE
|
||||
|
||||
def test_classify_relational():
|
||||
assert persons.classify_name("Cousine Emmy Haniel", GIVEN) is NameClass.RELATIONAL
|
||||
assert persons.classify_name("Schwester Hanni", GIVEN) is NameClass.RELATIONAL
|
||||
|
||||
def test_classify_single_token():
|
||||
assert persons.classify_name("Agnes", GIVEN) is NameClass.SINGLE_TOKEN
|
||||
assert persons.classify_name("A.B.", GIVEN) is NameClass.SINGLE_TOKEN
|
||||
|
||||
def test_classify_ambiguous_pair():
|
||||
assert persons.classify_name("Ella Anita", GIVEN) is NameClass.AMBIGUOUS_PAIR
|
||||
assert persons.classify_name("Kurt Georg", GIVEN) is NameClass.AMBIGUOUS_PAIR
|
||||
|
||||
def test_classify_resolvable_single_person():
|
||||
# first + surname (surname not a given name) -> one real person, NOT ambiguous
|
||||
assert persons.classify_name("Mieze Schefold", GIVEN) is NameClass.RESOLVABLE
|
||||
assert persons.classify_name("Adolf Butenandt", GIVEN) is NameClass.RESOLVABLE
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_persons.py -k classify -v && cd -`
|
||||
Expected: FAIL — `NameClass` / `classify_name` not defined.
|
||||
|
||||
- [ ] **Step 3: Implement** — add to `persons.py`. Add `from enum import StrEnum` to the imports if not present, then add:
|
||||
|
||||
```python
|
||||
class NameClass(StrEnum):
|
||||
RESOLVABLE = "resolvable"
|
||||
UNKNOWN = "unknown"
|
||||
SINGLE_TOKEN = "single_token"
|
||||
RELATIONAL = "relational"
|
||||
COLLECTIVE = "collective"
|
||||
PROSE = "prose"
|
||||
AMBIGUOUS_PAIR = "ambiguous_pair"
|
||||
|
||||
|
||||
_QUOTE_CHARS = "\"'“”„‚‘’"
|
||||
|
||||
|
||||
def classify_name(raw: str, given_names: set[str]) -> NameClass:
|
||||
"""Classify a (post-split) sender/receiver string by why it may be unresolvable.
|
||||
|
||||
Precedence (first match wins): UNKNOWN -> PROSE -> COLLECTIVE -> RELATIONAL ->
|
||||
SINGLE_TOKEN -> AMBIGUOUS_PAIR -> RESOLVABLE.
|
||||
"""
|
||||
s = raw.strip()
|
||||
if not s:
|
||||
return NameClass.RESOLVABLE
|
||||
low = s.lower()
|
||||
tokens = s.split()
|
||||
# alpha-only word tokens: "Fam.Cram" -> ["fam","cram"], so collective/relational terms
|
||||
# are matched as whole words (no substring/prefix false positives like "Allerton").
|
||||
alpha_words = re.findall(r"[a-zäöüß]+", low)
|
||||
if "?" in s or any(m in low for m in config.UNKNOWN_NAME_MARKERS):
|
||||
return NameClass.UNKNOWN
|
||||
if (len(s) > config.PROSE_MAX_LEN or any(c.isdigit() for c in s)
|
||||
or any(q in s for q in _QUOTE_CHARS) or len(tokens) > 3):
|
||||
return NameClass.PROSE
|
||||
if any(w in config.COLLECTIVE_TERMS for w in alpha_words):
|
||||
return NameClass.COLLECTIVE
|
||||
if any(w in config.RELATIONAL_TERMS for w in alpha_words):
|
||||
return NameClass.RELATIONAL
|
||||
if len(tokens) == 1:
|
||||
return NameClass.SINGLE_TOKEN
|
||||
if len(tokens) == 2 and all(_norm(t) in given_names for t in tokens):
|
||||
return NameClass.AMBIGUOUS_PAIR
|
||||
return NameClass.RESOLVABLE
|
||||
|
||||
|
||||
# Known limitation: a 4+-token name with no digits/quotes (e.g. "Anna von der Heide") is
|
||||
# classified PROSE. Such multi-particle names are rare here and usually resolve via the
|
||||
# register; if they surface in review, lower-priority than the real prose entries.
|
||||
```
|
||||
|
||||
> Note: `_norm` already exists in `persons.py` (added in the alias-index task) and strips accents + lowercases. `classify_name` uses it so given-name matching is accent-insensitive.
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_persons.py -v && cd -`
|
||||
Expected: PASS (all persons tests, including the 7 new classify tests).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/persons.py tools/import-normalizer/tests/test_persons.py
|
||||
git commit -m "feat(normalizer): classify_name + NameClass"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `build_given_names`
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/persons.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_persons.py`
|
||||
|
||||
- [ ] **Step 1: Add failing test** to `tests/test_persons.py`
|
||||
|
||||
```python
|
||||
def test_build_given_names():
|
||||
people = persons.parse_register([
|
||||
{"last_name": "de Gruyter", "first_name": "Eugenie"},
|
||||
{"last_name": "Cram", "first_name": "Charlotte,Meta"}, # comma -> primary + extra given
|
||||
])
|
||||
g = persons.build_given_names(people, {"Anita"})
|
||||
assert "eugenie" in g
|
||||
assert "charlotte" in g and "meta" in g # primary + extra given names
|
||||
assert "anita" in g # from the extra set, normalized
|
||||
assert "schefold" not in g
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_persons.py::test_build_given_names -v && cd -`
|
||||
Expected: FAIL — `build_given_names` not defined.
|
||||
|
||||
- [ ] **Step 3: Implement** — add to `persons.py`
|
||||
|
||||
```python
|
||||
def build_given_names(register: list[Person], extra: set[str]) -> set[str]:
|
||||
"""Set of normalized given names from the register (first + extra given) plus a supplement.
|
||||
|
||||
Used by classify_name to tell a two-given-name pair (two people) from a first+surname.
|
||||
"""
|
||||
names: set[str] = set()
|
||||
for p in register:
|
||||
if p.first_name:
|
||||
names.add(_norm(p.first_name))
|
||||
for g in p.extra_given_names:
|
||||
names.add(_norm(g))
|
||||
for e in extra:
|
||||
names.add(_norm(e))
|
||||
return names
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_persons.py -v && cd -`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/persons.py tools/import-normalizer/tests/test_persons.py
|
||||
git commit -m "feat(normalizer): build_given_names from register + supplement"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Integrate — ResolutionContext records unresolved; orchestrator writes the report
|
||||
|
||||
This task touches `persons.py`, `normalize.py`, and two test files together so the whole suite stays green in one commit (removing `ctx.ambiguous` requires updating its only consumer, `normalize.py`, in the same change).
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/persons.py` (ResolutionContext)
|
||||
- Modify: `tools/import-normalizer/normalize.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_documents.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_normalize.py`
|
||||
|
||||
- [ ] **Step 1: Update the failing tests first**
|
||||
|
||||
In `tests/test_documents.py`, **replace** the existing `test_ambiguous_space_pair_flagged_not_split` function entirely with these two functions:
|
||||
|
||||
```python
|
||||
def test_ambiguous_pair_recorded_in_unresolved():
|
||||
people = persons.parse_register([{"last_name": "de Gruyter", "first_name": "Walter"}])
|
||||
ctx = persons.ResolutionContext(persons.AliasIndex(people), name_overrides={},
|
||||
given_names={"ella", "anita"})
|
||||
raw = documents.RawRow(source_row=7, index="C-0200", sender="", receivers="Ella Anita")
|
||||
doc = documents.to_canonical(raw, ctx, date_overrides={})
|
||||
assert len(doc.receiver_person_ids) == 1 # not split — one provisional
|
||||
assert any(name == "Ella Anita" and cat == "ambiguous_pair" for name, cat, _ in ctx.unresolved)
|
||||
|
||||
def test_resolvable_first_surname_pair_not_unresolved():
|
||||
ctx = persons.ResolutionContext(persons.AliasIndex([]), name_overrides={},
|
||||
given_names={"ella", "anita"})
|
||||
ctx.resolve_one("Mieze Schefold", source_row=1) # surname is not a given name
|
||||
assert ctx.unresolved == [] # RESOLVABLE -> not recorded
|
||||
```
|
||||
|
||||
In `tests/test_normalize.py`, in the `_doc_wb` fixture, change the `C-0001` row's receiver from empty to `"?"` so the run produces an unresolved entry. Find the line that appends the `C-0001` row and set its `EmpfängerIn` cell to `"?"`. For example the row currently reads:
|
||||
|
||||
```python
|
||||
ws.append(["C-0001", "", "", "", "Hans Wittkopf", "", "Freitag 1919", "", "", ""])
|
||||
```
|
||||
|
||||
change the 6th cell (EmpfängerIn) from `""` to `"?"`:
|
||||
|
||||
```python
|
||||
ws.append(["C-0001", "", "", "", "Hans Wittkopf", "?", "Freitag 1919", "", "", ""])
|
||||
```
|
||||
|
||||
Then add these assertions inside `test_run_end_to_end`, right after the existing `assert (review_dir / "unparsed-dates.csv").exists()` line:
|
||||
|
||||
```python
|
||||
assert (out_dir / "canonical-documents.xlsx").exists() # (keep existing asserts above)
|
||||
assert (review_dir / "unresolved-names.csv").exists()
|
||||
unresolved_text = (review_dir / "unresolved-names.csv").read_text(encoding="utf-8")
|
||||
assert "unknown" in unresolved_text and "?" in unresolved_text # the "?" receiver
|
||||
assert not (review_dir / "ambiguous-receivers.csv").exists() # replaced
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify they fail**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_documents.py tests/test_normalize.py -v && cd -`
|
||||
Expected: FAIL — `ResolutionContext` has no `given_names`/`unresolved`; `unresolved-names.csv` not written.
|
||||
|
||||
- [ ] **Step 3a: Implement — `ResolutionContext` in `persons.py`**
|
||||
|
||||
Replace the `ResolutionContext.__init__` body's two lines (`self.ambiguous` and add `given_names`) and the relevant methods. The new `__init__`:
|
||||
|
||||
```python
|
||||
def __init__(self, alias_index: AliasIndex, name_overrides: dict[str, str],
|
||||
given_names: set[str] | None = None):
|
||||
self.index = alias_index
|
||||
self.name_overrides = name_overrides
|
||||
self.given_names = given_names or set()
|
||||
self.provisional: dict[str, Person] = {}
|
||||
self.unmatched: dict[str, list] = {}
|
||||
self.unresolved: list[tuple] = [] # (raw_name, category, source_row) for non-RESOLVABLE names
|
||||
self._raw_to_pid: dict[str, str] = {}
|
||||
self.override_hits = 0
|
||||
```
|
||||
|
||||
In `resolve_one`, the provisional branch must classify the name. Replace this existing block:
|
||||
|
||||
```python
|
||||
# provisional person (unmatched) — never reuse a register id
|
||||
self.unmatched.setdefault(name, []).append(source_row)
|
||||
if name in self._raw_to_pid:
|
||||
return self._raw_to_pid[name], name, False
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```python
|
||||
# provisional person (unmatched) — never reuse a register id
|
||||
self.unmatched.setdefault(name, []).append(source_row)
|
||||
category = classify_name(name, self.given_names)
|
||||
if category is not NameClass.RESOLVABLE:
|
||||
self.unresolved.append((name, str(category), source_row))
|
||||
if name in self._raw_to_pid:
|
||||
return self._raw_to_pid[name], name, False
|
||||
```
|
||||
|
||||
Replace the entire `resolve_receivers` method (the ambiguous detection now lives in `resolve_one` via `classify_name`):
|
||||
|
||||
```python
|
||||
def resolve_receivers(self, raw: str, source_row: int):
|
||||
return [self.resolve_one(part, source_row) for part in split_receivers(raw)]
|
||||
```
|
||||
|
||||
- [ ] **Step 3b: Implement — `normalize.py`**
|
||||
|
||||
Find the line that builds the context:
|
||||
|
||||
```python
|
||||
ctx = persons.ResolutionContext(alias_index, name_overrides)
|
||||
```
|
||||
|
||||
replace it with (build the given-name set from the register + config supplement):
|
||||
|
||||
```python
|
||||
given_names = persons.build_given_names(register, config.EXTRA_GIVEN_NAMES)
|
||||
ctx = persons.ResolutionContext(alias_index, name_overrides, given_names=given_names)
|
||||
```
|
||||
|
||||
Replace the `ambiguous-receivers.csv` write line:
|
||||
|
||||
```python
|
||||
writers.write_review_csv(review_dir / "ambiguous-receivers.csv", ["raw", "part", "source_row"], ctx.ambiguous)
|
||||
```
|
||||
|
||||
with an aggregated unresolved-names report:
|
||||
|
||||
```python
|
||||
unresolved_agg: dict[tuple, list] = {}
|
||||
for name, category, row in ctx.unresolved:
|
||||
unresolved_agg.setdefault((category, name), []).append(row)
|
||||
unresolved_rows = sorted(
|
||||
([cat, name, len(rows), " ".join(map(str, sorted(rows)[:5]))]
|
||||
for (cat, name), rows in unresolved_agg.items()),
|
||||
key=lambda r: (r[0], -r[2], r[1]))
|
||||
writers.write_review_csv(review_dir / "unresolved-names.csv",
|
||||
["category", "raw", "count", "example_rows"], unresolved_rows)
|
||||
```
|
||||
|
||||
In the `stats` dict, replace the `"ambiguous_receivers"` line:
|
||||
|
||||
```python
|
||||
"ambiguous_receivers": len(ctx.ambiguous),
|
||||
```
|
||||
|
||||
with a per-category breakdown:
|
||||
|
||||
```python
|
||||
"unresolved_name_occurrences": len(ctx.unresolved),
|
||||
"unresolved_unknown": sum(1 for _, c, _ in ctx.unresolved if c == "unknown"),
|
||||
"unresolved_single_token": sum(1 for _, c, _ in ctx.unresolved if c == "single_token"),
|
||||
"unresolved_relational": sum(1 for _, c, _ in ctx.unresolved if c == "relational"),
|
||||
"unresolved_collective": sum(1 for _, c, _ in ctx.unresolved if c == "collective"),
|
||||
"unresolved_prose": sum(1 for _, c, _ in ctx.unresolved if c == "prose"),
|
||||
"unresolved_ambiguous_pair": sum(1 for _, c, _ in ctx.unresolved if c == "ambiguous_pair"),
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the whole suite to verify green**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/ -q && cd -`
|
||||
Expected: PASS (all tests, no `ambiguous` references remain).
|
||||
|
||||
Also grep to confirm no dangling references:
|
||||
Run: `grep -rn "ctx.ambiguous\|ambiguous-receivers\|ambiguous_receivers\|self.ambiguous" tools/import-normalizer/*.py`
|
||||
Expected: no matches.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/persons.py tools/import-normalizer/normalize.py tools/import-normalizer/tests/test_documents.py tools/import-normalizer/tests/test_normalize.py
|
||||
git commit -m "feat(normalizer): unresolved-names report + fix ambiguous-pair over-flagging"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: README — document the new report
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/README.md`
|
||||
|
||||
- [ ] **Step 1: Update the review-file table** in `README.md`. Replace the `ambiguous-receivers.csv` row with an `unresolved-names.csv` row. Find the table row referencing `ambiguous-receivers.csv` and replace it with:
|
||||
|
||||
```markdown
|
||||
| `unresolved-names.csv` | Names whose value is itself problematic, grouped by `category`: `unknown` (`?`/illegible), `single_token` (first OR last name only), `relational` (`Tante …`), `collective` (`Familie …`), `prose` (a description landed in a name column), `ambiguous_pair` (two given names → likely two people, not auto-split). Review highest-impact categories first; add decisions to `overrides/names.csv`. |
|
||||
```
|
||||
|
||||
If the README has no such row (older version), add the row above to the review-file table.
|
||||
|
||||
- [ ] **Step 2: Add a note** to the iteration-loop section of `README.md` (after the table):
|
||||
|
||||
```markdown
|
||||
> `unresolved-names.csv` is the focused "names that need a human" list — distinct from
|
||||
> `unmatched-names.csv` (which is just non-family correspondents that got provisional persons).
|
||||
> The given-name set that drives `ambiguous_pair` detection is the register's first names plus
|
||||
> `config.EXTRA_GIVEN_NAMES` — add names there if a real two-person cell isn't being flagged.
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Verify the suite is still green** (README-only change, but confirm nothing references the old file)
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/ -q && cd -`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/README.md
|
||||
git commit -m "docs(normalizer): document unresolved-names.csv review report"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage** (against the agreed proposal):
|
||||
- Focused report isolating problem name classes → Task 4 writes `review/unresolved-names.csv` with a `category` column; categories defined in Task 2 `classify_name`. ✓
|
||||
- Fix ambiguous over-flagging of `First Surname` → Task 2 `AMBIGUOUS_PAIR` requires *both* tokens in the given-name set; `Mieze Schefold` → `RESOLVABLE` (tested). ✓
|
||||
- Distinguish "not fully known" (unknown/single-token/relational/collective/prose) from "can't split cleanly" (ambiguous_pair) → all are `NameClass` values, each its own category column value. ✓
|
||||
- Per-category counts in summary → Task 4 stats. ✓
|
||||
- Senders covered too (not just receivers) → classification happens in `resolve_one`, which both `resolve_sender` and `resolve_receivers` call. ✓
|
||||
|
||||
**Placeholder scan:** No TBD/TODO; every code step has complete code. The README replacement gives the exact row text.
|
||||
|
||||
**Type consistency:** `NameClass` (StrEnum) defined Task 2; `classify_name(raw, given_names)` and `build_given_names(register, extra)` signatures used consistently in Task 4; `ResolutionContext(alias_index, name_overrides, given_names=…)` matches the new `__init__`; `self.unresolved` is `list[tuple]` of `(raw, category, source_row)` and read with that shape in both the report and the stats. `str(category)` yields the StrEnum value (e.g. `"ambiguous_pair"`), matching the stat comparisons and the test assertions.
|
||||
|
||||
**Cross-task green:** Task 4 deliberately bundles the `persons.py` + `normalize.py` + test changes into one commit because removing `ctx.ambiguous` breaks its consumer otherwise — no red commit is left behind (lesson from the prior build).
|
||||
|
||||
**Out of scope (future):** Spanish month names + `Mon DD-YYYY` date form (separate date-parser enhancement); promoting `unresolved` rows into a document-level `needs_review` flag; auto-splitting confirmed `ambiguous_pair` entries via overrides.
|
||||
62
docs/import-migration/README.md
Normal file
62
docs/import-migration/README.md
Normal file
@@ -0,0 +1,62 @@
|
||||
# Import Migration — Working Folder
|
||||
|
||||
This folder tracks the iterative work of mass-importing the **real, raw family archive**
|
||||
spreadsheets (≈7,600 letter rows + ~7,000 PDFs that arrive later) into Familienarchiv.
|
||||
|
||||
It is intentionally **local docs, not Gitea issues**. We only open a Gitea issue when a
|
||||
finding requires a *software* change (e.g. a new date parser). Pure data observations and
|
||||
the running plan live here so any agent can pick the work up cold.
|
||||
|
||||
## Source files (in `/import`)
|
||||
|
||||
| File | What it is | Importer support today |
|
||||
| --- | --- | --- |
|
||||
| `zzfamilienarchiv aktuell 2 - Kopie 2025-07-05.xlsx` | The **real raw archive** — 7,943 rows, sheet `Familienarchiv`. Human-readable, dates as written in the letters. | ❌ layout does **not** match importer defaults |
|
||||
| `Personendatei 2.xlsx` | Genealogical **person register** — 163 people, sheet `Tabelle1` (maiden names, birth/death, marriages, relationships). | ❌ no importer at all |
|
||||
| `zzfamilienarchiv Walter und Eugenie 2025-04-10.ods` | A small, **already-normalized** subset (Walter & Eugenie brautbriefe). 14 clean columns incl. ISO dates. | ✅ this is what `MassImportService` was built for |
|
||||
|
||||
The PDFs (~7,000) will follow later. The importer matches files by the **Index** column
|
||||
(e.g. `W-0001` → `W-0001.pdf`), and already imports metadata-only when a file is missing —
|
||||
so we can import all metadata now and the PDFs will attach on a re-run.
|
||||
|
||||
## How to inspect the spreadsheets
|
||||
|
||||
`openpyxl` is installed in the OCR service venv:
|
||||
|
||||
```bash
|
||||
/home/marcel/Desktop/familienarchiv/ocr-service/.venv/bin/python3 -c "import openpyxl; print(openpyxl.__version__)"
|
||||
```
|
||||
|
||||
## Documents in this folder
|
||||
|
||||
- [`01-findings-spreadsheet-analysis.md`](./01-findings-spreadsheet-analysis.md) — full analysis of every data-quality / importer issue found (2026-05-25). Each issue has an ID `IMP-NN`.
|
||||
- [`02-normalization-spec.md`](./02-normalization-spec.md) — requirements spec for the offline **import normalizer** (the agreed strategy: normalize the raw sheets into a clean canonical dataset before import). Requirements `FR-*`/`NFR-*`, traceable to the `IMP-NN` findings.
|
||||
- `WORKLOG.md` — running log of what each session did and what's next. **Start here when resuming.**
|
||||
|
||||
## Strategy (decided 2026-05-25)
|
||||
|
||||
Normalize **before** import. A standalone Python tool (`tools/import-normalizer/`, not yet
|
||||
built) transforms the raw xlsx + person register into a clean canonical dataset
|
||||
(`canonical-documents.xlsx`, `canonical-persons.xlsx`) plus review CSVs. Residual cases
|
||||
(unparseable dates, unmatched names) are fixed via a version-controlled overrides file and
|
||||
re-run. The Java importer is adjusted to consume the canonical contract in a later **Phase 2**.
|
||||
See the spec for the full contract.
|
||||
|
||||
## Status board
|
||||
|
||||
| ID | Issue | Severity | Status |
|
||||
| --- | --- | --- | --- |
|
||||
| IMP-01 | New xlsx column layout ≠ importer defaults | 🔴 blocker | open |
|
||||
| IMP-02 | 90% of dates are free-text the parser can't read | 🔴 blocker | open |
|
||||
| IMP-03 | No ISO/normalized date column in the new xlsx | 🔴 blocker | open |
|
||||
| IMP-04 | Person register (`Personendatei 2.xlsx`) not imported | 🟠 major | open |
|
||||
| IMP-05 | Name variations = duplicate Persons (maiden vs married) | 🟠 major | open |
|
||||
| IMP-06 | 93 data rows with blank Index are silently dropped | 🟠 major | open |
|
||||
| IMP-07 | 43 duplicate Index values | 🟡 minor | open |
|
||||
| IMP-08 | Section/title rows interleaved in data | 🟡 minor | open |
|
||||
| IMP-09 | Index↔Datei filename mismatches | 🟡 minor | open |
|
||||
| IMP-10 | `x`-suffix rows (letter backsides/enclosures) | 🟡 minor | open |
|
||||
| IMP-11 | Multi-receiver separators incl. bare `u`/`u.` | 🟡 minor | open |
|
||||
| IMP-12 | Importer reads only the first sheet, no validation | 🟡 minor | open |
|
||||
|
||||
See the findings doc for detail and proposed approach per issue.
|
||||
147
docs/import-migration/WORKLOG.md
Normal file
147
docs/import-migration/WORKLOG.md
Normal file
@@ -0,0 +1,147 @@
|
||||
# Import Migration — Worklog
|
||||
|
||||
Running log of each working session. **Resume here.** Newest entry on top.
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 5) — Unresolved-name classification
|
||||
|
||||
**Did:** Implemented [`04-unresolved-names-plan.md`](./04-unresolved-names-plan.md) subagent-driven
|
||||
(5 tasks, TDD, per-task spec + code-quality review; 67 tests pass). Added `classify_name` +
|
||||
`NameClass` + `build_given_names` in `persons.py`; `ResolutionContext` now records non-RESOLVABLE
|
||||
names in `self.unresolved`; orchestrator writes `review/unresolved-names.csv` (replaces the noisy
|
||||
`ambiguous-receivers.csv`) with per-category stats.
|
||||
|
||||
**Why:** `unmatched-names.csv` mixes boring non-family correspondents (expected) with genuinely
|
||||
unresolvable entries. The new report isolates the latter so review focuses on ~440 real cases.
|
||||
|
||||
**Real-run result:** unresolved-names.csv = single_token 191 / prose 103 / unknown 74 /
|
||||
collective 46 / relational 21 / ambiguous_pair **5** (distinct). The ambiguous over-flagging fix
|
||||
cut `ambiguous_pair` from 303 → 5 (genuine two-given-name pairs only; `Mieze Schefold` etc. now
|
||||
correctly RESOLVABLE). given-name set = register first names ∪ `config.EXTRA_GIVEN_NAMES`.
|
||||
|
||||
**Next:** populate `overrides/names.csv` from unresolved-names.csv (highest-count first); extend
|
||||
`EXTRA_GIVEN_NAMES` if a real pair isn't flagged; still-open date work (Spanish months, 58–72 band).
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 4) — Built the normalizer (subagent-driven, all 17 tasks)
|
||||
|
||||
**Did:** Executed the plan subagent-driven (implementer + spec review + code-quality review per
|
||||
task). The tool `tools/import-normalizer/` is **complete and passing (57 tests)**. Final
|
||||
opus review: **READY** — determinism verified on the real corpus (two runs → identical cell
|
||||
matrices + byte-identical review files), zero silent drops.
|
||||
|
||||
**Per-task code review caught & fixed real issues** (all in the committed code): leading
|
||||
qualifiers `nach/vor/…` now → APPROX; English month-first matcher hardened to structurally
|
||||
not shadow `Mai 1895`; person-id collision de-dup suffixes *all* members; `split_receivers`
|
||||
returns `[]` for a `geb.`-only cell; boolean cells no longer coerced to `1/0`; duplicate-index
|
||||
flags every occurrence; provisional ids never steal a register id; CSV-injection defanged.
|
||||
|
||||
**REAL DRY-RUN** (`python normalize.py` over the actual archive — outputs are gitignored):
|
||||
- documents_emitted **7,582** (+225 empty +93 blank-index +42 x-suffix = 7,942 rows read, 0 dropped)
|
||||
- register_persons **163**, provisional_persons **942**
|
||||
- dates: DAY 6,509 / MONTH 36 / RANGE 36 / APPROX 28 / YEAR 17 / SEASON 1 / UNKNOWN 955
|
||||
- **unknown_date_rate 9.2%** (of dated rows; target ≤5% pre-override, ≤0.5% after overrides)
|
||||
- duplicate_index 85, index_file_mismatches 550, ambiguous_receivers 303
|
||||
|
||||
**⚠️ Concurrency incident:** a parallel Claude session committed reader-dashboard work to this
|
||||
branch and hard-reset it mid-execution, deleting the Task 15 files and orphaning a commit.
|
||||
Recovered via reflog (`reset --hard 366b4848` + `checkout 401160e3 -- <task15 files>`); no code
|
||||
lost. Casualty: my *during-execution* edits to the plan/spec docs (02/03) for Tasks 5–14 were
|
||||
discarded — **the committed code + tests are the source of truth**, not the plan doc, which now
|
||||
reflects the pre-execution + persona-review version.
|
||||
|
||||
**Next steps (iterative refinement — the overrides loop, as designed):**
|
||||
1. Shave the 9.2% UNKNOWN cheaply: add **Spanish month names** (Enero…Diciembre) and the
|
||||
`Mon DD-YYYY` dash form to `config.MONTHS`/the parser (Mexican-branch correspondence);
|
||||
revisit the 58–72 two-digit-year band (real `…58/59/60` dates = 1958–1960, just past the
|
||||
1873–1957 window — decide whether to extend the upper bound in `config`).
|
||||
2. `?` (99×) is genuinely "date unknown" — leave UNKNOWN or add a convention.
|
||||
3. Populate `overrides/dates.csv` + `overrides/names.csv` from the review CSVs and re-run.
|
||||
4. README note: a leading `'`/`!` in a `review/*.csv` `raw` cell may be a CSV-defang artifact —
|
||||
match against the true source value when writing overrides.
|
||||
5. Phase 2 (separate spec): wire the canonical contract into the Java `MassImportService`.
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 3) — Implementation plan + persona review
|
||||
|
||||
**Did:**
|
||||
- Wrote [`03-normalizer-implementation-plan.md`](./03-normalizer-implementation-plan.md): 17
|
||||
bite-sized TDD tasks for `tools/import-normalizer/` (Python, openpyxl), bottom-up — date
|
||||
parser w/ Easter computus first, then persons/alias, ingest, mapping, orchestrator, writers.
|
||||
- Ran a 6-persona inline review (architect, developer, tester, req-engineer, security, devops;
|
||||
ui-expert too) via parallel agents. Acted on all material findings.
|
||||
|
||||
**Key fixes from review (see plan §"Review feedback incorporated"):**
|
||||
- Idempotency redefined byte-identical → **content-deterministic** (spec G4/NFR-IDEM-01);
|
||||
pinned workbook timestamps + deterministic alias ordering + a real two-run equality test.
|
||||
- Real bug: duplicate-index only reported repeats → now flags/reports every occurrence.
|
||||
- Provisional `person_id` could overwrite a register id → now suffixed.
|
||||
- Date parser gaps: invalid-calendar-date → UNKNOWN, intra-month day-range (`7./8. Sept.1923`).
|
||||
- Multi-person sender now split + flagged (REQ-PERS-01); CSV-injection defanged in review files;
|
||||
pinned deps + hardened root `.gitignore`.
|
||||
|
||||
**Next:**
|
||||
- Marcel reviews the plan. Then execute it (subagent-driven or inline) — the date parser
|
||||
(Task 3/8 + Easter computus) is the meatiest piece.
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 2) — Strategy + normalizer spec
|
||||
|
||||
**Did:**
|
||||
- Decided strategy with Marcel: **normalize the raw sheets first**, then import (higher
|
||||
leverage than making the Java importer tolerate every mess).
|
||||
- Locked design decisions (see spec §3): new canonical layout; dates = parsed + raw +
|
||||
precision; include person register + dedup in this effort; overrides-file + re-run loop;
|
||||
Python tool at `tools/import-normalizer/`.
|
||||
- Century rule fixed by Marcel: archive spans **1873–1957**; 2-digit `00–57`→19YY,
|
||||
`73–99`→18YY, `58–72`→flag; 3-digit→1DDD; never 20xx.
|
||||
- Wrote [`02-normalization-spec.md`](./02-normalization-spec.md) in the requirements-engineer
|
||||
persona (FR/NFR, Given-When-Then ACs, traceability to IMP-NN, TBD register).
|
||||
|
||||
**All 6 open questions resolved (spec §9):** OQ-01 — movable feasts (Ostern, Pfingsten, …)
|
||||
**computed per year from Easter**, never a fixed month; seasons → mid-season month
|
||||
(Sommer=Jul, Herbst=Oct). OQ-02 ranges → start+RANGE. OQ-03 slug ids. OQ-04 — `x`-suffix rows
|
||||
**skipped + logged** this pass (they're transcriptions of the base letter, not yet mappable).
|
||||
OQ-05 → `.xlsx`. OQ-06 → conservative, no silent merge.
|
||||
|
||||
**Git:** moved off the unrelated `feat/issue-356-…` branch; pulled `main`; created clean
|
||||
branch **`docs/import-migration`** and committed these docs there. (The dirty `.venv`
|
||||
pycache + `skills/implement/SKILL.md` in the tree are pre-existing/environmental noise — left
|
||||
uncommitted, not ours.)
|
||||
|
||||
**Next:**
|
||||
- Marcel reviews the spec.
|
||||
- Then writing-plans → build the normalizer at `tools/import-normalizer/` (backlog B1–B7 are
|
||||
the Musts; B3 date parser incl. Easter computus is the big one).
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 1) — Initial analysis
|
||||
|
||||
**Did:**
|
||||
- Got the real raw archive xlsx (7,943 rows) + person register (163 people). PDFs to follow.
|
||||
- Compared the new xlsx layout against `MassImportService` defaults and the old ODS.
|
||||
- Full statistical scan of all rows: dates, indices, senders/receivers, file column.
|
||||
- Wrote [`01-findings-spreadsheet-analysis.md`](./01-findings-spreadsheet-analysis.md)
|
||||
with 12 issues (IMP-01..IMP-12) + recommended sequencing.
|
||||
- Installed `openpyxl` into the OCR service venv for inspection.
|
||||
|
||||
**Key facts established:**
|
||||
- Importer defaults match the **ODS**, not the new xlsx → wrong column mapping (IMP-01).
|
||||
- **90%** of dated rows (6,571 / 7,319) are free-text dates the ISO-only parser drops (IMP-02).
|
||||
- Person register is rich but **unimported**; holds the maiden-name dedup key (IMP-04/05).
|
||||
|
||||
**Decisions pending from Marcel (blockers for any code work):**
|
||||
1. IMP-01: positional re-config of `app.import.col.*` vs header-driven mapping rewrite?
|
||||
2. IMP-02: how to store imprecise dates — new `dateOriginal` + `precision` columns, or lossy?
|
||||
3. IMP-04/05: format for the person/alias mapping; import persons before documents?
|
||||
4. IMP-10: are `x`-suffix rows separate documents, attachments, or skipped?
|
||||
|
||||
**Next:**
|
||||
- Get Marcel's calls on the 4 decisions above.
|
||||
- Then split the code-change items into Gitea issues (IMP-01b, IMP-02, IMP-04, IMP-06, IMP-12).
|
||||
- Pure-data tasks (IMP-07 dup list, IMP-09 file reconcile) stay here.
|
||||
@@ -16,6 +16,10 @@ CMD ["npm", "run", "dev"]
|
||||
# Compiles the SvelteKit Node-adapter output to /app/build.
|
||||
FROM node:20.19.0-alpine3.21 AS build
|
||||
WORKDIR /app
|
||||
# VITE_SENTRY_DSN is a build-time variable — Vite bakes it into the bundle.
|
||||
# Passed via docker-compose build.args; empty string disables the SDK.
|
||||
ARG VITE_SENTRY_DSN
|
||||
ENV VITE_SENTRY_DSN=$VITE_SENTRY_DSN
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
|
||||
@@ -58,3 +58,20 @@ test.describe('Language selector', () => {
|
||||
await expect(deBtn).toHaveClass(/font-bold/);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('Mobile nav — i18n', () => {
|
||||
test('hamburger button aria-label translates to EN on narrow viewport', async ({ browser }) => {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 375, height: 812 },
|
||||
storageState: 'e2e/.auth/user.json'
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.goto('/');
|
||||
await page.waitForSelector('[data-hydrated]');
|
||||
await page.getByRole('banner').getByRole('button', { name: 'EN', exact: true }).click();
|
||||
|
||||
await expect(page.getByRole('button', { name: 'Open menu' })).toBeVisible();
|
||||
|
||||
await context.close();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -106,6 +106,31 @@ export default defineConfig(
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
// Forbid test fixtures (*.test-fixture.svelte) from being imported by
|
||||
// production code. Tree-shaking keeps them out of the production bundle
|
||||
// today (no route reaches them), but a lint rule makes the boundary
|
||||
// explicit so an accidental autocomplete import in a route or component
|
||||
// fails fast. Test files (*.spec.ts / *.test.ts) and the fixtures
|
||||
// themselves are exempt — see the next block. Nora #2 on PR #629
|
||||
// round 3.
|
||||
files: ['**/*.svelte', '**/*.svelte.ts', '**/*.svelte.js', '**/*.ts'],
|
||||
ignores: ['**/*.spec.ts', '**/*.test.ts', '**/*.test-fixture.svelte'],
|
||||
rules: {
|
||||
'no-restricted-imports': [
|
||||
'error',
|
||||
{
|
||||
patterns: [
|
||||
{
|
||||
group: ['**/*.test-fixture.svelte'],
|
||||
message:
|
||||
'Test fixtures (*.test-fixture.svelte) are test-only — do not import from production code. Tracked by #637.'
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
plugins: { boundaries },
|
||||
settings: {
|
||||
|
||||
@@ -28,6 +28,8 @@
|
||||
"nav_conversations": "Briefwechsel",
|
||||
"nav_admin": "Admin",
|
||||
"nav_logout": "Abmelden",
|
||||
"layout_menu_open": "Menü öffnen",
|
||||
"layout_menu_close": "Menü schließen",
|
||||
"theme_toggle_to_light": "Zu hellem Design wechseln",
|
||||
"theme_toggle_to_dark": "Zu dunklem Design wechseln",
|
||||
"btn_save": "Speichern",
|
||||
@@ -352,6 +354,11 @@
|
||||
"admin_system_import_status_running": "Import läuft…",
|
||||
"admin_system_import_status_done": "Import abgeschlossen",
|
||||
"admin_system_import_status_done_label": "Dokumente verarbeitet",
|
||||
"admin_system_import_skipped_label": "übersprungen",
|
||||
"import_reason_invalid_pdf_signature": "Keine gültige PDF-Signatur",
|
||||
"import_reason_file_read_error": "Fehler beim Lesen der Datei",
|
||||
"import_reason_s3_upload_failed": "Upload-Fehler (S3)",
|
||||
"import_reason_already_exists": "Bereits importiert",
|
||||
"admin_system_import_status_failed": "Import fehlgeschlagen",
|
||||
"admin_system_import_failed_no_spreadsheet": "Keine Tabellendatei gefunden.",
|
||||
"admin_system_import_failed_internal": "Interner Fehler beim Import.",
|
||||
@@ -389,6 +396,10 @@
|
||||
"doc_panel_discussion_annotation_tab": "Annotation · Seite {page}",
|
||||
"pdf_annotations_show": "Annotierungen anzeigen",
|
||||
"pdf_annotations_hide": "Annotierungen verbergen",
|
||||
"viewer_previous_page": "Zurück",
|
||||
"viewer_next_page": "Weiter",
|
||||
"viewer_zoom_out": "Verkleinern",
|
||||
"viewer_zoom_in": "Vergrößern",
|
||||
"upload_action": "Hochladen",
|
||||
"upload_drop_hint": "Einzeln oder mehrere Dateien auf einmal hochladen",
|
||||
"upload_accepted_types": "PDF, JPEG, PNG, TIFF",
|
||||
@@ -434,8 +445,12 @@
|
||||
"person_mention_load_error": "Person konnte nicht geladen werden.",
|
||||
"person_mention_loading": "Lade Person…",
|
||||
"person_mention_popup_empty": "Keine Personen gefunden",
|
||||
"person_mention_search_label": "Person suchen",
|
||||
"person_mention_search_prompt": "Namen eingeben…",
|
||||
"person_mention_btn_label": "Person verlinken",
|
||||
"person_mention_create_new": "Neue Person anlegen",
|
||||
"person_mention_results_count_singular": "1 Person gefunden",
|
||||
"person_mention_results_count_plural": "{count} Personen gefunden",
|
||||
"transcription_editor_aria_label": "Transkriptionstext",
|
||||
"person_born_name_prefix": "geb.",
|
||||
"page_title_home": "Archiv",
|
||||
@@ -511,6 +526,7 @@
|
||||
"notification_filter_unread": "Ungelesen",
|
||||
"notification_filter_mention": "Erwähnung",
|
||||
"notification_filter_reply": "Antwort",
|
||||
"notification_error_generic": "Aktion fehlgeschlagen. Bitte versuche es erneut.",
|
||||
"notification_mark_all_read_aria": "Alle Benachrichtigungen als gelesen markieren",
|
||||
"notification_load_more": "Ältere laden",
|
||||
"notification_empty_history": "Keine Benachrichtigungen",
|
||||
@@ -622,6 +638,9 @@
|
||||
"transcription_block_review": "Als geprüft markieren",
|
||||
"transcription_block_unreview": "Markierung aufheben",
|
||||
"transcription_reviewed_count": "{reviewed} von {total} geprüft",
|
||||
"transcription_mark_all_reviewed": "Alle als fertig markieren",
|
||||
"transcription_mark_all_reviewed_disabled": "Alle Blöcke sind bereits als fertig markiert",
|
||||
"transcription_mark_all_reviewed_error": "Markierung fehlgeschlagen. Bitte versuchen Sie es erneut.",
|
||||
"training_ocr_heading": "Kurrent-Erkennung trainieren",
|
||||
"training_ocr_description": "Starte ein neues Training mit den bisher geprüften OCR-Blöcken, um die Erkennungsgenauigkeit für Kurrentschrift zu verbessern.",
|
||||
"training_ocr_blocks_ready": "{blocks} geprüfte Blöcke bereit / {docs} Dokumente",
|
||||
@@ -650,6 +669,7 @@
|
||||
"transcription_block_segmentation_only": "Nur Segmentierung",
|
||||
"training_chip_kurrent": "Kurrent-Erkennung",
|
||||
"training_chip_segmentation": "Segmentierung",
|
||||
"transcribe_mark_for_training": "Für Training vormerken",
|
||||
"training_col_type": "Typ",
|
||||
"training_type_base": "Basis",
|
||||
"training_type_personalized": "Personalisiert",
|
||||
|
||||
@@ -28,6 +28,8 @@
|
||||
"nav_conversations": "Letters",
|
||||
"nav_admin": "Admin",
|
||||
"nav_logout": "Sign out",
|
||||
"layout_menu_open": "Open menu",
|
||||
"layout_menu_close": "Close menu",
|
||||
"theme_toggle_to_light": "Switch to light mode",
|
||||
"theme_toggle_to_dark": "Switch to dark mode",
|
||||
"btn_save": "Save",
|
||||
@@ -352,6 +354,11 @@
|
||||
"admin_system_import_status_running": "Import running…",
|
||||
"admin_system_import_status_done": "Import complete",
|
||||
"admin_system_import_status_done_label": "Documents processed",
|
||||
"admin_system_import_skipped_label": "skipped",
|
||||
"import_reason_invalid_pdf_signature": "Invalid PDF signature",
|
||||
"import_reason_file_read_error": "File read error",
|
||||
"import_reason_s3_upload_failed": "Upload error (S3)",
|
||||
"import_reason_already_exists": "Already imported",
|
||||
"admin_system_import_status_failed": "Import failed",
|
||||
"admin_system_import_failed_no_spreadsheet": "No spreadsheet file found.",
|
||||
"admin_system_import_failed_internal": "Import failed due to an internal error.",
|
||||
@@ -389,6 +396,10 @@
|
||||
"doc_panel_discussion_annotation_tab": "Annotation · Page {page}",
|
||||
"pdf_annotations_show": "Show annotations",
|
||||
"pdf_annotations_hide": "Hide annotations",
|
||||
"viewer_previous_page": "Previous page",
|
||||
"viewer_next_page": "Next page",
|
||||
"viewer_zoom_out": "Zoom out",
|
||||
"viewer_zoom_in": "Zoom in",
|
||||
"upload_action": "Upload",
|
||||
"upload_drop_hint": "Drop one or multiple files at once",
|
||||
"upload_accepted_types": "PDF, JPEG, PNG, TIFF",
|
||||
@@ -434,8 +445,12 @@
|
||||
"person_mention_load_error": "Could not load person.",
|
||||
"person_mention_loading": "Loading person…",
|
||||
"person_mention_popup_empty": "No persons found",
|
||||
"person_mention_search_label": "Search for a person",
|
||||
"person_mention_search_prompt": "Enter a name…",
|
||||
"person_mention_btn_label": "Link person",
|
||||
"person_mention_create_new": "Create new person",
|
||||
"person_mention_results_count_singular": "1 person found",
|
||||
"person_mention_results_count_plural": "{count} persons found",
|
||||
"transcription_editor_aria_label": "Transcription text",
|
||||
"person_born_name_prefix": "née",
|
||||
"page_title_home": "Archive",
|
||||
@@ -511,6 +526,7 @@
|
||||
"notification_filter_unread": "Unread",
|
||||
"notification_filter_mention": "Mention",
|
||||
"notification_filter_reply": "Reply",
|
||||
"notification_error_generic": "Action failed. Please try again.",
|
||||
"notification_mark_all_read_aria": "Mark all notifications as read",
|
||||
"notification_load_more": "Load older",
|
||||
"notification_empty_history": "No notifications",
|
||||
@@ -622,6 +638,9 @@
|
||||
"transcription_block_review": "Mark as reviewed",
|
||||
"transcription_block_unreview": "Unmark as reviewed",
|
||||
"transcription_reviewed_count": "{reviewed} of {total} reviewed",
|
||||
"transcription_mark_all_reviewed": "Mark all as reviewed",
|
||||
"transcription_mark_all_reviewed_disabled": "All blocks are already marked as reviewed",
|
||||
"transcription_mark_all_reviewed_error": "Failed to mark all as reviewed. Please try again.",
|
||||
"training_ocr_heading": "Train Kurrent recognition",
|
||||
"training_ocr_description": "Start a new training run using the reviewed OCR blocks to improve recognition accuracy for Kurrent script.",
|
||||
"training_ocr_blocks_ready": "{blocks} reviewed blocks ready / {docs} documents",
|
||||
@@ -650,6 +669,7 @@
|
||||
"transcription_block_segmentation_only": "Segmentation only",
|
||||
"training_chip_kurrent": "Kurrent recognition",
|
||||
"training_chip_segmentation": "Segmentation",
|
||||
"transcribe_mark_for_training": "Mark for OCR training",
|
||||
"training_col_type": "Type",
|
||||
"training_type_base": "Base",
|
||||
"training_type_personalized": "Personalized",
|
||||
|
||||
@@ -28,6 +28,8 @@
|
||||
"nav_conversations": "Cartas",
|
||||
"nav_admin": "Admin",
|
||||
"nav_logout": "Cerrar sesión",
|
||||
"layout_menu_open": "Abrir menú",
|
||||
"layout_menu_close": "Cerrar menú",
|
||||
"theme_toggle_to_light": "Cambiar a modo claro",
|
||||
"theme_toggle_to_dark": "Cambiar a modo oscuro",
|
||||
"btn_save": "Guardar",
|
||||
@@ -352,6 +354,11 @@
|
||||
"admin_system_import_status_running": "Importación en curso…",
|
||||
"admin_system_import_status_done": "Importación completada",
|
||||
"admin_system_import_status_done_label": "Documentos procesados",
|
||||
"admin_system_import_skipped_label": "omitidos",
|
||||
"import_reason_invalid_pdf_signature": "Firma PDF no válida",
|
||||
"import_reason_file_read_error": "Error al leer el archivo",
|
||||
"import_reason_s3_upload_failed": "Error de carga (S3)",
|
||||
"import_reason_already_exists": "Ya importado",
|
||||
"admin_system_import_status_failed": "Importación fallida",
|
||||
"admin_system_import_failed_no_spreadsheet": "No se encontró ninguna hoja de cálculo.",
|
||||
"admin_system_import_failed_internal": "Error interno durante la importación.",
|
||||
@@ -389,6 +396,10 @@
|
||||
"doc_panel_discussion_annotation_tab": "Anotación · Página {page}",
|
||||
"pdf_annotations_show": "Mostrar anotaciones",
|
||||
"pdf_annotations_hide": "Ocultar anotaciones",
|
||||
"viewer_previous_page": "Página anterior",
|
||||
"viewer_next_page": "Página siguiente",
|
||||
"viewer_zoom_out": "Reducir",
|
||||
"viewer_zoom_in": "Ampliar",
|
||||
"upload_action": "Subir",
|
||||
"upload_drop_hint": "Uno o varios archivos a la vez",
|
||||
"upload_accepted_types": "PDF, JPEG, PNG, TIFF",
|
||||
@@ -434,8 +445,12 @@
|
||||
"person_mention_load_error": "No se pudo cargar la persona.",
|
||||
"person_mention_loading": "Cargando persona…",
|
||||
"person_mention_popup_empty": "No se encontraron personas",
|
||||
"person_mention_search_label": "Buscar persona",
|
||||
"person_mention_search_prompt": "Escribe un nombre…",
|
||||
"person_mention_btn_label": "Vincular persona",
|
||||
"person_mention_create_new": "Crear nueva persona",
|
||||
"person_mention_results_count_singular": "1 persona encontrada",
|
||||
"person_mention_results_count_plural": "{count} personas encontradas",
|
||||
"transcription_editor_aria_label": "Texto de transcripción",
|
||||
"person_born_name_prefix": "n.",
|
||||
"page_title_home": "Archivo",
|
||||
@@ -511,6 +526,7 @@
|
||||
"notification_filter_unread": "No leídas",
|
||||
"notification_filter_mention": "Mención",
|
||||
"notification_filter_reply": "Respuesta",
|
||||
"notification_error_generic": "La acción ha fallado. Por favor, inténtalo de nuevo.",
|
||||
"notification_mark_all_read_aria": "Marcar todas las notificaciones como leídas",
|
||||
"notification_load_more": "Cargar anteriores",
|
||||
"notification_empty_history": "Sin notificaciones",
|
||||
@@ -622,6 +638,9 @@
|
||||
"transcription_block_review": "Marcar como revisado",
|
||||
"transcription_block_unreview": "Desmarcar como revisado",
|
||||
"transcription_reviewed_count": "{reviewed} de {total} revisados",
|
||||
"transcription_mark_all_reviewed": "Marcar todo como revisado",
|
||||
"transcription_mark_all_reviewed_disabled": "Todos los bloques ya están marcados como revisados",
|
||||
"transcription_mark_all_reviewed_error": "Error al marcar como revisado. Intente de nuevo.",
|
||||
"training_ocr_heading": "Entrenar reconocimiento Kurrent",
|
||||
"training_ocr_description": "Inicia un nuevo entrenamiento con los bloques OCR revisados para mejorar la precisión de reconocimiento del script Kurrent.",
|
||||
"training_ocr_blocks_ready": "{blocks} bloques revisados listos / {docs} documentos",
|
||||
@@ -650,6 +669,7 @@
|
||||
"transcription_block_segmentation_only": "Solo segmentación",
|
||||
"training_chip_kurrent": "Reconocimiento Kurrent",
|
||||
"training_chip_segmentation": "Segmentación",
|
||||
"transcribe_mark_for_training": "Marcar para entrenamiento de OCR",
|
||||
"training_col_type": "Tipo",
|
||||
"training_type_base": "Base",
|
||||
"training_type_personalized": "Personalizado",
|
||||
|
||||
2878
frontend/package-lock.json
generated
2878
frontend/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -16,7 +16,7 @@
|
||||
"lint:boundary-demo": "eslint src/lib/tag/__fixtures__/",
|
||||
"test:unit": "vitest",
|
||||
"test": "npm run test:unit -- --run",
|
||||
"test:coverage": "vitest run --coverage --project=server; vitest run -c vitest.client-coverage.config.ts --coverage",
|
||||
"test:coverage": "vitest run --coverage --project=server && vitest run -c vitest.client-coverage.config.ts --coverage",
|
||||
"test:e2e": "playwright test",
|
||||
"test:e2e:headed": "playwright test --headed",
|
||||
"test:e2e:ui": "playwright test --ui",
|
||||
@@ -24,9 +24,9 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@sentry/sveltekit": "^10.53.1",
|
||||
"@tiptap/core": "3.22.5",
|
||||
"@tiptap/extension-mention": "3.22.5",
|
||||
"@tiptap/starter-kit": "3.22.5",
|
||||
"@tiptap/core": "3.23.4",
|
||||
"@tiptap/extension-mention": "3.23.4",
|
||||
"@tiptap/starter-kit": "3.23.4",
|
||||
"diff": "^8.0.3",
|
||||
"isomorphic-dompurify": "^3.12.0",
|
||||
"openapi-fetch": "^0.13.5",
|
||||
@@ -37,9 +37,9 @@
|
||||
"@eslint/compat": "^1.4.0",
|
||||
"@eslint/js": "^9.39.1",
|
||||
"@inlang/paraglide-js": "^2.5.0",
|
||||
"@playwright/test": "^1.58.2",
|
||||
"@sveltejs/adapter-node": "^5.4.0",
|
||||
"@sveltejs/kit": "^2.48.5",
|
||||
"@playwright/test": "^1.60.0",
|
||||
"@sveltejs/adapter-node": "^5.5.4",
|
||||
"@sveltejs/kit": "^2.60.1",
|
||||
"@sveltejs/vite-plugin-svelte": "^6.2.1",
|
||||
"@tailwindcss/forms": "^0.5.10",
|
||||
"@tailwindcss/typography": "^0.5.19",
|
||||
@@ -57,7 +57,7 @@
|
||||
"globals": "^16.5.0",
|
||||
"openapi-typescript": "^7.8.0",
|
||||
"patch-package": "^8.0.0",
|
||||
"playwright": "^1.56.1",
|
||||
"playwright": "^1.60.0",
|
||||
"prettier": "^3.6.2",
|
||||
"prettier-plugin-svelte": "^3.4.0",
|
||||
"prettier-plugin-tailwindcss": "^0.7.1",
|
||||
@@ -66,7 +66,7 @@
|
||||
"tailwindcss": "^4.1.17",
|
||||
"typescript": "^5.9.3",
|
||||
"typescript-eslint": "^8.47.0",
|
||||
"vite": "^7.2.2",
|
||||
"vite": "^7.3.3",
|
||||
"vite-plugin-devtools-json": "^1.0.0",
|
||||
"vitest": "^4.0.10",
|
||||
"vitest-browser-svelte": "^2.0.1"
|
||||
|
||||
@@ -1,30 +1,30 @@
|
||||
diff --git a/node_modules/@vitest/browser-playwright/dist/index.js b/node_modules/@vitest/browser-playwright/dist/index.js
|
||||
index 5d0d37b..821d7b4 100644
|
||||
index c01e754..f1bb7be 100644
|
||||
--- a/node_modules/@vitest/browser-playwright/dist/index.js
|
||||
+++ b/node_modules/@vitest/browser-playwright/dist/index.js
|
||||
@@ -935,7 +935,7 @@ class PlaywrightBrowserProvider {
|
||||
@@ -936,7 +936,7 @@ class PlaywrightBrowserProvider {
|
||||
createMocker() {
|
||||
const idPreficates = new Map();
|
||||
const idPredicates = new Map();
|
||||
const sessionIds = new Map();
|
||||
- function createPredicate(sessionId, url) {
|
||||
+ function createPredicate(url) {
|
||||
const moduleUrl = new URL(url, "http://localhost");
|
||||
const predicate = (url) => {
|
||||
if (url.searchParams.has("_vitest_original")) {
|
||||
@@ -960,11 +960,7 @@ class PlaywrightBrowserProvider {
|
||||
@@ -961,11 +961,7 @@ class PlaywrightBrowserProvider {
|
||||
}
|
||||
return true;
|
||||
};
|
||||
- const ids = sessionIds.get(sessionId) || [];
|
||||
- ids.push(moduleUrl.href);
|
||||
- sessionIds.set(sessionId, ids);
|
||||
- idPreficates.set(predicateKey(sessionId, moduleUrl.href), predicate);
|
||||
- idPredicates.set(predicateKey(sessionId, moduleUrl.href), predicate);
|
||||
- return predicate;
|
||||
+ return { url: moduleUrl.href, predicate };
|
||||
}
|
||||
function predicateKey(sessionId, url) {
|
||||
return `${sessionId}:${url}`;
|
||||
@@ -972,7 +968,23 @@ class PlaywrightBrowserProvider {
|
||||
@@ -973,7 +969,23 @@ class PlaywrightBrowserProvider {
|
||||
return {
|
||||
register: async (sessionId, module) => {
|
||||
const page = this.getPage(sessionId);
|
||||
@@ -37,19 +37,19 @@ index 5d0d37b..821d7b4 100644
|
||||
+ // duplicate-id mocks (e.g. '$lib/foo.svelte' + '$lib/foo.svelte.js')
|
||||
+ // leak an orphan route whose handler crashes after the next
|
||||
+ // session's birpc channel closes.
|
||||
+ const existingPredicate = idPreficates.get(key);
|
||||
+ const existingPredicate = idPredicates.get(key);
|
||||
+ if (existingPredicate) {
|
||||
+ await page.context().unroute(existingPredicate);
|
||||
+ }
|
||||
+ const ids = sessionIds.get(sessionId) ?? new Set();
|
||||
+ ids.add(moduleUrl);
|
||||
+ sessionIds.set(sessionId, ids);
|
||||
+ idPreficates.set(key, predicate);
|
||||
+ idPredicates.set(key, predicate);
|
||||
+ await page.context().route(predicate, async (route) => {
|
||||
if (module.type === "manual") {
|
||||
const exports$1 = Object.keys(await module.resolve());
|
||||
const body = createManualModuleSource(module.url, exports$1);
|
||||
@@ -1033,8 +1045,8 @@ class PlaywrightBrowserProvider {
|
||||
@@ -1034,8 +1046,8 @@ class PlaywrightBrowserProvider {
|
||||
},
|
||||
clear: async (sessionId) => {
|
||||
const page = this.getPage(sessionId);
|
||||
@@ -58,5 +58,5 @@ index 5d0d37b..821d7b4 100644
|
||||
+ const ids = sessionIds.get(sessionId) ?? new Set();
|
||||
+ const promises = [...ids].map((id) => {
|
||||
const key = predicateKey(sessionId, id);
|
||||
const predicate = idPreficates.get(key);
|
||||
const predicate = idPredicates.get(key);
|
||||
if (predicate) {
|
||||
@@ -111,7 +111,7 @@ const PUBLIC_API_PATHS = [
|
||||
|
||||
export const handleFetch: HandleFetch = async ({ event, request, fetch }) => {
|
||||
const apiUrl = env.API_INTERNAL_URL || 'http://localhost:8080';
|
||||
const isApi = request.url.startsWith(apiUrl) || request.url.includes('/api/');
|
||||
const isApi = request.url.startsWith(apiUrl) || new URL(request.url).pathname.startsWith('/api/');
|
||||
|
||||
if (!isApi) return fetch(request);
|
||||
|
||||
@@ -131,14 +131,13 @@ export const handleFetch: HandleFetch = async ({ event, request, fetch }) => {
|
||||
if (sessionId) cookieParts.push(`fa_session=${sessionId}`);
|
||||
if (xsrfToken) cookieParts.push(`XSRF-TOKEN=${xsrfToken}`);
|
||||
|
||||
if (cookieParts.length === 0 && !xsrfToken) {
|
||||
if (cookieParts.length === 0) {
|
||||
return fetch(request);
|
||||
}
|
||||
|
||||
// Clone first so the body stream is preserved on the new Request.
|
||||
const cloned = request.clone();
|
||||
const extraHeaders: Record<string, string> = {};
|
||||
if (cookieParts.length > 0) extraHeaders['Cookie'] = cookieParts.join('; ');
|
||||
const extraHeaders: Record<string, string> = { Cookie: cookieParts.join('; ') };
|
||||
if (xsrfToken) extraHeaders['X-XSRF-TOKEN'] = xsrfToken;
|
||||
|
||||
const modified = new Request(cloned, {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script lang="ts">
|
||||
import { enhance } from '$app/forms';
|
||||
import * as m from '$lib/paraglide/messages.js';
|
||||
import { relativeTime } from '$lib/shared/utils/time';
|
||||
import type { NotificationItem } from '$lib/notification/notifications.svelte';
|
||||
@@ -6,11 +7,13 @@ import { buildCommentHref } from '$lib/shared/discussion/commentDeepLink';
|
||||
|
||||
interface Props {
|
||||
unread: NotificationItem[];
|
||||
onMarkRead: (n: NotificationItem) => void;
|
||||
onMarkAllRead: () => void;
|
||||
optimisticMarkRead: (id: string) => void;
|
||||
optimisticMarkAllRead: () => void;
|
||||
}
|
||||
|
||||
const { unread, onMarkRead, onMarkAllRead }: Props = $props();
|
||||
const { unread, optimisticMarkRead, optimisticMarkAllRead }: Props = $props();
|
||||
|
||||
let errorMessage: string | null = $state(null);
|
||||
|
||||
function verb(type: NotificationItem['type'], actor: string): string {
|
||||
return type === 'REPLY'
|
||||
@@ -24,6 +27,9 @@ function href(n: NotificationItem): string {
|
||||
</script>
|
||||
|
||||
<section class="rounded-sm border border-line bg-surface p-5">
|
||||
{#if errorMessage}
|
||||
<p role="alert" class="px-4 py-2 text-sm text-red-600">{errorMessage}</p>
|
||||
{/if}
|
||||
{#if unread.length === 0}
|
||||
<div data-testid="chronik-inbox-zero" class="flex flex-col items-center gap-3 py-6 text-center">
|
||||
<svg
|
||||
@@ -66,14 +72,28 @@ function href(n: NotificationItem): string {
|
||||
{m.chronik_for_you_count({ count: unread.length })}
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
data-testid="chronik-mark-all-read"
|
||||
onclick={onMarkAllRead}
|
||||
class="font-sans text-xs font-medium text-ink-3 transition-colors hover:text-ink"
|
||||
<form
|
||||
action="/aktivitaeten?/mark-all-read"
|
||||
method="POST"
|
||||
use:enhance={() => {
|
||||
errorMessage = null;
|
||||
optimisticMarkAllRead();
|
||||
return async ({ result, update }) => {
|
||||
if (result.type === 'failure' || result.type === 'error') {
|
||||
errorMessage = m.notification_error_generic();
|
||||
await update({ reset: false, invalidateAll: false });
|
||||
}
|
||||
};
|
||||
}}
|
||||
>
|
||||
{m.chronik_mark_all_read()}
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
data-testid="chronik-mark-all-read"
|
||||
class="font-sans text-xs font-medium text-ink-3 transition-colors hover:text-ink"
|
||||
>
|
||||
{m.chronik_mark_all_read()}
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<ul role="list" class="flex flex-col gap-2">
|
||||
@@ -89,7 +109,7 @@ function href(n: NotificationItem): string {
|
||||
aria-hidden="true"
|
||||
class="mt-0.5 inline-flex h-6 w-6 shrink-0 items-center justify-center rounded-full bg-accent-bg font-sans text-xs font-bold text-accent"
|
||||
>
|
||||
{n.type === 'MENTION' ? '@' : '\u21A9'}
|
||||
{n.type === 'MENTION' ? '@' : '↩'}
|
||||
</span>
|
||||
<div class="min-w-0 flex-1">
|
||||
<p class="font-sans text-sm leading-snug text-ink">
|
||||
@@ -100,25 +120,40 @@ function href(n: NotificationItem): string {
|
||||
</p>
|
||||
</div>
|
||||
</a>
|
||||
<button
|
||||
type="button"
|
||||
data-testid="chronik-fuerdich-dismiss"
|
||||
aria-label={m.chronik_mark_read_aria()}
|
||||
onclick={() => onMarkRead(n)}
|
||||
class="mt-0.5 shrink-0 rounded-sm p-1 text-ink-3 transition-colors hover:bg-muted hover:text-ink focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:outline-none"
|
||||
<form
|
||||
action="/aktivitaeten?/dismiss-notification"
|
||||
method="POST"
|
||||
use:enhance={() => {
|
||||
errorMessage = null;
|
||||
optimisticMarkRead(n.id);
|
||||
return async ({ result, update }) => {
|
||||
if (result.type === 'failure' || result.type === 'error') {
|
||||
errorMessage = m.notification_error_generic();
|
||||
await update({ reset: false, invalidateAll: false });
|
||||
}
|
||||
};
|
||||
}}
|
||||
>
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
class="h-4 w-4"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
aria-hidden="true"
|
||||
<input type="hidden" name="notificationId" value={n.id} />
|
||||
<button
|
||||
type="submit"
|
||||
data-testid="chronik-fuerdich-dismiss"
|
||||
aria-label={m.chronik_mark_read_aria()}
|
||||
class="mt-0.5 shrink-0 rounded-sm p-1 text-ink-3 transition-colors hover:bg-muted hover:text-ink focus-visible:ring-2 focus-visible:ring-focus-ring focus-visible:outline-none"
|
||||
>
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
class="h-4 w-4"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
</form>
|
||||
</li>
|
||||
{/each}
|
||||
</ul>
|
||||
|
||||
@@ -5,7 +5,36 @@ import { page, userEvent } from 'vitest/browser';
|
||||
import ChronikFuerDichBox from './ChronikFuerDichBox.svelte';
|
||||
import type { NotificationItem } from '$lib/notification/notifications.svelte';
|
||||
|
||||
afterEach(cleanup);
|
||||
const mockFormResult = vi.hoisted(() => ({ type: 'success' as string }));
|
||||
|
||||
vi.mock('$app/forms', () => ({
|
||||
enhance(
|
||||
node: HTMLFormElement,
|
||||
submit?: (opts: {
|
||||
formData: FormData;
|
||||
}) => (opts: {
|
||||
result: { type: string; data?: Record<string, unknown> };
|
||||
update: () => Promise<void>;
|
||||
}) => Promise<void>
|
||||
) {
|
||||
const handler = async (e: Event) => {
|
||||
e.preventDefault();
|
||||
const cb = submit?.({ formData: new FormData(node) } as never);
|
||||
if (typeof cb === 'function') {
|
||||
await (
|
||||
cb as (o: { result: typeof mockFormResult; update: () => Promise<void> }) => Promise<void>
|
||||
)({ result: mockFormResult, update: async () => {} });
|
||||
}
|
||||
};
|
||||
node.addEventListener('submit', handler);
|
||||
return { destroy: () => node.removeEventListener('submit', handler) };
|
||||
}
|
||||
}));
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
mockFormResult.type = 'success';
|
||||
});
|
||||
|
||||
function notif(partial: Partial<NotificationItem>): NotificationItem {
|
||||
return {
|
||||
@@ -26,8 +55,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
it('renders inbox-zero state when there are no unread items', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
const zero = document.querySelector('[data-testid="chronik-inbox-zero"]');
|
||||
expect(zero).not.toBeNull();
|
||||
@@ -37,8 +66,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
it('links to the archived mentions in the inbox-zero state', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
const link = document.querySelector('a[href="/aktivitaeten?filter=fuer-dich"]');
|
||||
expect(link).not.toBeNull();
|
||||
@@ -47,8 +76,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
it('renders the count badge with correct total when unread exists', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [notif({ id: 'a' }), notif({ id: 'b' })],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
await expect.element(page.getByText('2 neu')).toBeInTheDocument();
|
||||
});
|
||||
@@ -56,8 +85,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
it('count badge has aria-live=polite when unread exists', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [notif({ id: 'a' })],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
// Wait for render
|
||||
await expect.element(page.getByText('1 neu')).toBeInTheDocument();
|
||||
@@ -69,8 +98,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
it('does not render the "Alle gelesen" button when there are no unread items', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
await expect.element(page.getByText('Keine neuen Erwähnungen')).toBeInTheDocument();
|
||||
const all = document.querySelector('[data-testid="chronik-mark-all-read"]');
|
||||
@@ -80,38 +109,38 @@ describe('ChronikFuerDichBox', () => {
|
||||
it('renders the "Alle gelesen" button when unread exists', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [notif({ id: 'a' })],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
await expect.element(page.getByText('Alle gelesen')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('calls onMarkAllRead when the "Alle gelesen" button is clicked', async () => {
|
||||
const onMarkAllRead = vi.fn();
|
||||
it('calls optimisticMarkAllRead when the "Alle gelesen" button is submitted', async () => {
|
||||
const optimisticMarkAllRead = vi.fn();
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [notif({ id: 'a' })],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead
|
||||
});
|
||||
await userEvent.click(page.getByText('Alle gelesen'));
|
||||
expect(onMarkAllRead).toHaveBeenCalledTimes(1);
|
||||
expect(optimisticMarkAllRead).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('calls onMarkRead (and not navigation) when a per-item Dismiss button is clicked', async () => {
|
||||
const onMarkRead = vi.fn();
|
||||
it('calls optimisticMarkRead with the notification id when its dismiss button is submitted', async () => {
|
||||
const optimisticMarkRead = vi.fn();
|
||||
const n = notif({ id: 'xyz' });
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [n],
|
||||
onMarkRead,
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead,
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
const dismiss = document.querySelector(
|
||||
'[data-testid="chronik-fuerdich-dismiss"]'
|
||||
) as HTMLButtonElement | null;
|
||||
expect(dismiss).not.toBeNull();
|
||||
dismiss?.click();
|
||||
expect(onMarkRead).toHaveBeenCalledTimes(1);
|
||||
expect(onMarkRead.mock.calls[0][0]).toEqual(n);
|
||||
expect(optimisticMarkRead).toHaveBeenCalledTimes(1);
|
||||
expect(optimisticMarkRead.mock.calls[0][0]).toBe('xyz');
|
||||
});
|
||||
|
||||
it('mention row href includes both commentId and annotationId when annotationId is present', async () => {
|
||||
@@ -124,8 +153,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
annotationId: 'annot-9'
|
||||
})
|
||||
],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
const link = document.querySelector(
|
||||
'a[href="/documents/doc-42?commentId=comment-7&annotationId=annot-9"]'
|
||||
@@ -136,8 +165,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
it('Dismiss button is a sibling of the document link, never nested inside <a>', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [notif({ id: 'x' })],
|
||||
onMarkRead: vi.fn(),
|
||||
onMarkAllRead: vi.fn()
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
const dismiss = document.querySelector('[data-testid="chronik-fuerdich-dismiss"]');
|
||||
expect(dismiss).not.toBeNull();
|
||||
@@ -145,4 +174,22 @@ describe('ChronikFuerDichBox', () => {
|
||||
// Prevents the senior-audience tap-drag bug flagged by Leonie.
|
||||
expect(dismiss?.closest('a')).toBeNull();
|
||||
});
|
||||
|
||||
it('shows an accessible error banner when the dismiss action returns a failure', async () => {
|
||||
mockFormResult.type = 'failure';
|
||||
render(ChronikFuerDichBox, {
|
||||
unread: [notif({ id: 'err-1' })],
|
||||
optimisticMarkRead: vi.fn(),
|
||||
optimisticMarkAllRead: vi.fn()
|
||||
});
|
||||
const dismiss = document.querySelector(
|
||||
'[data-testid="chronik-fuerdich-dismiss"]'
|
||||
) as HTMLButtonElement | null;
|
||||
expect(dismiss).not.toBeNull();
|
||||
dismiss?.click();
|
||||
// Allow microtask queue to flush
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
const alert = document.querySelector('[role="alert"]');
|
||||
expect(alert).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,7 +4,36 @@ import { page } from 'vitest/browser';
|
||||
import ChronikFuerDichBox from './ChronikFuerDichBox.svelte';
|
||||
import type { NotificationItem } from '$lib/notification/notifications';
|
||||
|
||||
afterEach(cleanup);
|
||||
const mockFormResult = vi.hoisted(() => ({ type: 'success' as string }));
|
||||
|
||||
vi.mock('$app/forms', () => ({
|
||||
enhance(
|
||||
node: HTMLFormElement,
|
||||
submit?: (opts: {
|
||||
formData: FormData;
|
||||
}) => (opts: {
|
||||
result: { type: string; data?: Record<string, unknown> };
|
||||
update: () => Promise<void>;
|
||||
}) => Promise<void>
|
||||
) {
|
||||
const handler = async (e: Event) => {
|
||||
e.preventDefault();
|
||||
const cb = submit?.({ formData: new FormData(node) } as never);
|
||||
if (typeof cb === 'function') {
|
||||
await (
|
||||
cb as (o: { result: typeof mockFormResult; update: () => Promise<void> }) => Promise<void>
|
||||
)({ result: mockFormResult, update: async () => {} });
|
||||
}
|
||||
};
|
||||
node.addEventListener('submit', handler);
|
||||
return { destroy: () => node.removeEventListener('submit', handler) };
|
||||
}
|
||||
}));
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
mockFormResult.type = 'success';
|
||||
});
|
||||
|
||||
const mention = (overrides: Partial<NotificationItem> = {}): NotificationItem => ({
|
||||
id: 'n-1',
|
||||
@@ -22,7 +51,7 @@ const mention = (overrides: Partial<NotificationItem> = {}): NotificationItem =>
|
||||
describe('ChronikFuerDichBox', () => {
|
||||
it('renders the inbox-zero state when there are no unread', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
props: { unread: [], onMarkRead: () => {}, onMarkAllRead: () => {} }
|
||||
props: { unread: [], optimisticMarkRead: () => {}, optimisticMarkAllRead: () => {} }
|
||||
});
|
||||
|
||||
await expect.element(page.getByText(/keine neuen erwähnungen/i)).toBeVisible();
|
||||
@@ -34,8 +63,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
props: {
|
||||
unread: [mention(), mention({ id: 'n-2' }), mention({ id: 'n-3' })],
|
||||
onMarkRead: () => {},
|
||||
onMarkAllRead: () => {}
|
||||
optimisticMarkRead: () => {},
|
||||
optimisticMarkAllRead: () => {}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -47,8 +76,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
props: {
|
||||
unread: [mention({ id: 'n-m', type: 'MENTION' }), mention({ id: 'n-r', type: 'REPLY' })],
|
||||
onMarkRead: () => {},
|
||||
onMarkAllRead: () => {}
|
||||
optimisticMarkRead: () => {},
|
||||
optimisticMarkAllRead: () => {}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -62,8 +91,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
props: {
|
||||
unread: [mention({ actorName: 'Bertha' })],
|
||||
onMarkRead: () => {},
|
||||
onMarkAllRead: () => {}
|
||||
optimisticMarkRead: () => {},
|
||||
optimisticMarkAllRead: () => {}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -76,8 +105,8 @@ describe('ChronikFuerDichBox', () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
props: {
|
||||
unread: [mention({ type: 'REPLY', actorName: 'Carl' })],
|
||||
onMarkRead: () => {},
|
||||
onMarkAllRead: () => {}
|
||||
optimisticMarkRead: () => {},
|
||||
optimisticMarkAllRead: () => {}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -86,11 +115,11 @@ describe('ChronikFuerDichBox', () => {
|
||||
.toBeVisible();
|
||||
});
|
||||
|
||||
it('calls onMarkRead with the notification when its dismiss button is clicked', async () => {
|
||||
const onMarkRead = vi.fn();
|
||||
it('calls optimisticMarkRead with the notification id when its dismiss button is clicked', async () => {
|
||||
const optimisticMarkRead = vi.fn();
|
||||
const item = mention({ id: 'n-7' });
|
||||
render(ChronikFuerDichBox, {
|
||||
props: { unread: [item], onMarkRead, onMarkAllRead: () => {} }
|
||||
props: { unread: [item], optimisticMarkRead, optimisticMarkAllRead: () => {} }
|
||||
});
|
||||
|
||||
const dismiss = document.querySelector(
|
||||
@@ -98,35 +127,55 @@ describe('ChronikFuerDichBox', () => {
|
||||
) as HTMLElement;
|
||||
dismiss.click();
|
||||
|
||||
expect(onMarkRead).toHaveBeenCalledWith(item);
|
||||
expect(optimisticMarkRead).toHaveBeenCalledWith('n-7');
|
||||
});
|
||||
|
||||
it('calls onMarkAllRead when the mark-all-read button is clicked', async () => {
|
||||
const onMarkAllRead = vi.fn();
|
||||
it('calls optimisticMarkAllRead when the mark-all-read button is clicked', async () => {
|
||||
const optimisticMarkAllRead = vi.fn();
|
||||
render(ChronikFuerDichBox, {
|
||||
props: {
|
||||
unread: [mention()],
|
||||
onMarkRead: () => {},
|
||||
onMarkAllRead
|
||||
optimisticMarkRead: () => {},
|
||||
optimisticMarkAllRead
|
||||
}
|
||||
});
|
||||
|
||||
const btn = document.querySelector('[data-testid="chronik-mark-all-read"]') as HTMLElement;
|
||||
btn.click();
|
||||
|
||||
expect(onMarkAllRead).toHaveBeenCalledOnce();
|
||||
expect(optimisticMarkAllRead).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('builds a deep-link href to the comment for each notification', async () => {
|
||||
render(ChronikFuerDichBox, {
|
||||
props: {
|
||||
unread: [mention({ documentId: 'doc-x', referenceId: 'ref-y', annotationId: null })],
|
||||
onMarkRead: () => {},
|
||||
onMarkAllRead: () => {}
|
||||
optimisticMarkRead: () => {},
|
||||
optimisticMarkAllRead: () => {}
|
||||
}
|
||||
});
|
||||
|
||||
const link = document.querySelector('ul[role="list"] li a') as HTMLAnchorElement;
|
||||
expect(link.getAttribute('href')).toContain('doc-x');
|
||||
});
|
||||
|
||||
it('shows an accessible error banner when the dismiss action returns a failure', async () => {
|
||||
mockFormResult.type = 'failure';
|
||||
render(ChronikFuerDichBox, {
|
||||
props: {
|
||||
unread: [mention({ id: 'err-1' })],
|
||||
optimisticMarkRead: () => {},
|
||||
optimisticMarkAllRead: () => {}
|
||||
}
|
||||
});
|
||||
|
||||
const dismiss = document.querySelector(
|
||||
'[data-testid="chronik-fuerdich-dismiss"]'
|
||||
) as HTMLElement;
|
||||
dismiss.click();
|
||||
// Allow microtask queue to flush
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
const alert = document.querySelector('[role="alert"]');
|
||||
expect(alert).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -17,6 +17,7 @@ import PdfViewer from '$lib/document/viewer/PdfViewer.svelte';
|
||||
import { bulkTitleFromFilename } from '$lib/document/filename';
|
||||
import type { Tag } from '$lib/tag/TagInput.svelte';
|
||||
import type { components } from '$lib/generated/api';
|
||||
import { withCsrf } from '$lib/shared/cookies';
|
||||
|
||||
type Person = components['schemas']['Person'];
|
||||
|
||||
@@ -183,7 +184,10 @@ async function saveUpload() {
|
||||
// FormData with per-chunk progress. Session cookie is sent automatically
|
||||
// by the browser for same-origin requests.
|
||||
try {
|
||||
const res = await fetch('/api/documents/quick-upload', { method: 'POST', body: formData });
|
||||
const res = await fetch(
|
||||
'/api/documents/quick-upload',
|
||||
withCsrf({ method: 'POST', body: formData })
|
||||
);
|
||||
const body = await res.json().catch(() => ({ errors: [] }));
|
||||
const errorFilenames = new Set<string>(
|
||||
(body.errors ?? []).map((err: { filename: string }) => err.filename)
|
||||
|
||||
@@ -5,7 +5,7 @@ import { clickOutside } from '$lib/shared/actions/clickOutside';
|
||||
import { formatDate } from '$lib/shared/utils/date';
|
||||
|
||||
type Document = components['schemas']['Document'];
|
||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
||||
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||
|
||||
interface Props {
|
||||
selectedDocuments?: Document[];
|
||||
@@ -45,8 +45,12 @@ function handleInput() {
|
||||
try {
|
||||
const res = await fetch(`/api/documents/search?q=${encodeURIComponent(searchTerm)}&size=10`);
|
||||
if (res.ok) {
|
||||
const body: { items: DocumentSearchItem[] } = await res.json();
|
||||
const docs = body.items.map((it) => it.document);
|
||||
const body: { items: DocumentListItem[] } = await res.json();
|
||||
const docs = body.items.map((it) => ({
|
||||
id: it.id,
|
||||
title: it.title,
|
||||
documentDate: it.documentDate
|
||||
})) as unknown as Document[];
|
||||
results = docs.filter((d) => !selectedDocuments.some((s) => s.id === d.id));
|
||||
}
|
||||
} catch {
|
||||
|
||||
@@ -10,7 +10,19 @@ const docFactory = (id: string, title: string, date = '1880-01-01') => ({
|
||||
title,
|
||||
documentDate: date,
|
||||
originalFilename: `${title}.pdf`,
|
||||
status: 'UPLOADED',
|
||||
receivers: [],
|
||||
tags: [],
|
||||
completionPercentage: 0,
|
||||
contributors: [],
|
||||
matchData: {
|
||||
titleOffsets: [],
|
||||
senderMatched: false,
|
||||
matchedReceiverIds: [],
|
||||
matchedTagIds: [],
|
||||
snippetOffsets: [],
|
||||
summaryOffsets: []
|
||||
},
|
||||
status: 'UPLOADED' as const,
|
||||
metadataComplete: false,
|
||||
scriptType: 'UNKNOWN' as const,
|
||||
createdAt: '2024-01-01T00:00:00',
|
||||
@@ -22,7 +34,7 @@ function mockSearchResponse(items: ReturnType<typeof docFactory>[]) {
|
||||
'fetch',
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: vi.fn().mockResolvedValue({ items: items.map((document) => ({ document })) })
|
||||
json: vi.fn().mockResolvedValue({ items })
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -91,10 +103,7 @@ describe('DocumentMultiSelect — search and select', () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: vi.fn().mockResolvedValue({
|
||||
items: [
|
||||
{ document: docFactory('d1', 'Already attached') },
|
||||
{ document: docFactory('d2', 'Not attached') }
|
||||
]
|
||||
items: [docFactory('d1', 'Already attached'), docFactory('d2', 'Not attached')]
|
||||
})
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
@@ -9,11 +9,11 @@ import ProgressRing from '$lib/shared/primitives/ProgressRing.svelte';
|
||||
import ContributorStack from '$lib/shared/primitives/ContributorStack.svelte';
|
||||
import DocumentThumbnail from './DocumentThumbnail.svelte';
|
||||
|
||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
||||
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||
|
||||
let { item, canWrite = false }: { item: DocumentSearchItem; canWrite?: boolean } = $props();
|
||||
let { item, canWrite = false }: { item: DocumentListItem; canWrite?: boolean } = $props();
|
||||
|
||||
const doc = $derived(item.document);
|
||||
const doc = $derived(item);
|
||||
const titleText = $derived(doc.title || doc.originalFilename);
|
||||
const titleOffsets = $derived(item.matchData?.titleOffsets ?? []);
|
||||
const titleSegments = $derived(applyOffsets(titleText, titleOffsets));
|
||||
|
||||
@@ -14,24 +14,17 @@ afterEach(() => {
|
||||
bulkSelectionStore.clear();
|
||||
});
|
||||
|
||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
||||
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||
|
||||
function makeItem(overrides: Partial<DocumentSearchItem> = {}): DocumentSearchItem {
|
||||
function makeItem(overrides: Partial<DocumentListItem> = {}): DocumentListItem {
|
||||
return {
|
||||
document: {
|
||||
id: '1',
|
||||
title: 'Testbrief',
|
||||
originalFilename: 'testbrief.pdf',
|
||||
status: 'UPLOADED',
|
||||
documentDate: '2024-03-15',
|
||||
sender: null,
|
||||
receivers: [],
|
||||
tags: [],
|
||||
createdAt: '2024-01-01T00:00:00Z',
|
||||
updatedAt: '2024-01-01T00:00:00Z',
|
||||
metadataComplete: false,
|
||||
scriptType: 'UNKNOWN'
|
||||
},
|
||||
id: '1',
|
||||
title: 'Testbrief',
|
||||
originalFilename: 'testbrief.pdf',
|
||||
documentDate: '2024-03-15',
|
||||
sender: undefined,
|
||||
receivers: [],
|
||||
tags: [],
|
||||
matchData: {
|
||||
titleOffsets: [],
|
||||
senderMatched: false,
|
||||
@@ -55,14 +48,14 @@ describe('DocumentRow – title', () => {
|
||||
});
|
||||
|
||||
it('falls back to originalFilename when title is null', async () => {
|
||||
const item = makeItem({ document: { ...makeItem().document, title: null } });
|
||||
const item = makeItem({ title: null as unknown as string });
|
||||
render(DocumentRow, { item });
|
||||
await expect.element(page.getByRole('heading', { name: 'testbrief.pdf' })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders a mark element for highlighted title offsets', async () => {
|
||||
const item = makeItem({
|
||||
document: { ...makeItem().document, title: 'Brief an Anna' },
|
||||
title: 'Brief an Anna',
|
||||
matchData: {
|
||||
titleOffsets: [{ start: 0, length: 5 }],
|
||||
senderMatched: false,
|
||||
@@ -109,9 +102,12 @@ describe('DocumentRow – snippet', () => {
|
||||
describe('DocumentRow – sender', () => {
|
||||
it('shows sender display name', async () => {
|
||||
const item = makeItem({
|
||||
document: {
|
||||
...makeItem().document,
|
||||
sender: { id: 's1', displayName: 'Großmutter Maria' }
|
||||
sender: {
|
||||
id: 's1',
|
||||
lastName: 'Maria',
|
||||
displayName: 'Großmutter Maria',
|
||||
personType: 'PERSON',
|
||||
familyMember: false
|
||||
}
|
||||
});
|
||||
render(DocumentRow, { item });
|
||||
@@ -126,9 +122,12 @@ describe('DocumentRow – sender', () => {
|
||||
|
||||
it('highlights the sender when senderMatched is true', async () => {
|
||||
const item = makeItem({
|
||||
document: {
|
||||
...makeItem().document,
|
||||
sender: { id: 's1', displayName: 'Großmutter Maria' }
|
||||
sender: {
|
||||
id: 's1',
|
||||
lastName: 'Maria',
|
||||
displayName: 'Großmutter Maria',
|
||||
personType: 'PERSON',
|
||||
familyMember: false
|
||||
},
|
||||
matchData: {
|
||||
...makeItem().matchData,
|
||||
@@ -142,10 +141,15 @@ describe('DocumentRow – sender', () => {
|
||||
|
||||
it('highlights a receiver when matchedReceiverIds includes its id', async () => {
|
||||
const item = makeItem({
|
||||
document: {
|
||||
...makeItem().document,
|
||||
receivers: [{ id: 'r1', displayName: 'Onkel Karl' }]
|
||||
},
|
||||
receivers: [
|
||||
{
|
||||
id: 'r1',
|
||||
lastName: 'Karl',
|
||||
displayName: 'Onkel Karl',
|
||||
personType: 'PERSON',
|
||||
familyMember: false
|
||||
}
|
||||
],
|
||||
matchData: {
|
||||
...makeItem().matchData,
|
||||
matchedReceiverIds: ['r1']
|
||||
@@ -162,10 +166,7 @@ describe('DocumentRow – sender', () => {
|
||||
describe('DocumentRow – summary', () => {
|
||||
it('renders the document summary when present', async () => {
|
||||
const item = makeItem({
|
||||
document: {
|
||||
...makeItem().document,
|
||||
summary: 'Brief von Eugenie über die Heimreise aus dem Süden.'
|
||||
}
|
||||
summary: 'Brief von Eugenie über die Heimreise aus dem Süden.'
|
||||
});
|
||||
render(DocumentRow, { item });
|
||||
await expect
|
||||
@@ -180,7 +181,7 @@ describe('DocumentRow – summary', () => {
|
||||
|
||||
it('applies summary search-match highlight via summaryOffsets', async () => {
|
||||
const item = makeItem({
|
||||
document: { ...makeItem().document, summary: 'Brief über Menton' },
|
||||
summary: 'Brief über Menton',
|
||||
matchData: {
|
||||
...makeItem().matchData,
|
||||
summaryOffsets: [{ start: 11, length: 6 }]
|
||||
@@ -196,25 +197,19 @@ describe('DocumentRow – summary', () => {
|
||||
|
||||
describe('DocumentRow – archive chips', () => {
|
||||
it('renders the archive box chip when set', async () => {
|
||||
const item = makeItem({
|
||||
document: { ...makeItem().document, archiveBox: 'K3' }
|
||||
});
|
||||
const item = makeItem({ archiveBox: 'K3' });
|
||||
render(DocumentRow, { item });
|
||||
await expect.element(page.getByText('K3')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders the archive folder chip when set', async () => {
|
||||
const item = makeItem({
|
||||
document: { ...makeItem().document, archiveFolder: 'Mappe A' }
|
||||
});
|
||||
const item = makeItem({ archiveFolder: 'Mappe A' });
|
||||
render(DocumentRow, { item });
|
||||
await expect.element(page.getByText('Mappe A')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders the location chip when meta_location is set', async () => {
|
||||
const item = makeItem({
|
||||
document: { ...makeItem().document, location: 'Berlin' }
|
||||
});
|
||||
const item = makeItem({ location: 'Berlin' });
|
||||
render(DocumentRow, { item });
|
||||
await expect.element(page.getByText('Berlin')).toBeInTheDocument();
|
||||
});
|
||||
@@ -225,10 +220,7 @@ describe('DocumentRow – archive chips', () => {
|
||||
describe('DocumentRow – tags', () => {
|
||||
it('renders tag buttons', async () => {
|
||||
const item = makeItem({
|
||||
document: {
|
||||
...makeItem().document,
|
||||
tags: [{ id: 't1', name: 'Familie', color: null, parentId: null }]
|
||||
}
|
||||
tags: [{ id: 't1', name: 'Familie' }]
|
||||
});
|
||||
render(DocumentRow, { item });
|
||||
await expect.element(page.getByRole('button', { name: 'Familie' })).toBeInTheDocument();
|
||||
@@ -236,10 +228,7 @@ describe('DocumentRow – tags', () => {
|
||||
|
||||
it('navigates to /documents?tag=… on tag click', async () => {
|
||||
const item = makeItem({
|
||||
document: {
|
||||
...makeItem().document,
|
||||
tags: [{ id: 't1', name: 'Urlaub & Reise', color: null, parentId: null }]
|
||||
}
|
||||
tags: [{ id: 't1', name: 'Urlaub & Reise' }]
|
||||
});
|
||||
render(DocumentRow, { item });
|
||||
// Tailwind CSS isn't loaded in the vitest-browser client project, so the
|
||||
@@ -255,10 +244,7 @@ describe('DocumentRow – tags', () => {
|
||||
|
||||
it('tag click does not navigate to the document detail page', async () => {
|
||||
const item = makeItem({
|
||||
document: {
|
||||
...makeItem().document,
|
||||
tags: [{ id: 't2', name: 'Familie', color: null, parentId: null }]
|
||||
}
|
||||
tags: [{ id: 't2', name: 'Familie' }]
|
||||
});
|
||||
render(DocumentRow, { item });
|
||||
const before = window.location.href;
|
||||
@@ -281,7 +267,7 @@ describe('DocumentRow – bulk selection checkbox', () => {
|
||||
});
|
||||
|
||||
it('checkbox aria-label includes the document title', async () => {
|
||||
const item = makeItem({ document: { ...makeItem().document, title: 'Brief an Anna' } });
|
||||
const item = makeItem({ title: 'Brief an Anna' });
|
||||
render(DocumentRow, { item, canWrite: true });
|
||||
await expect
|
||||
.element(page.getByRole('checkbox', { name: /Brief an Anna/i }))
|
||||
@@ -289,7 +275,7 @@ describe('DocumentRow – bulk selection checkbox', () => {
|
||||
});
|
||||
|
||||
it('toggling the checkbox calls bulkSelectionStore.toggle', async () => {
|
||||
const item = makeItem({ document: { ...makeItem().document, id: 'doc-42' } });
|
||||
const item = makeItem({ id: 'doc-42' });
|
||||
render(DocumentRow, { item, canWrite: true });
|
||||
expect(bulkSelectionStore.has('doc-42')).toBe(false);
|
||||
|
||||
@@ -300,7 +286,7 @@ describe('DocumentRow – bulk selection checkbox', () => {
|
||||
|
||||
it('checked state mirrors the store', async () => {
|
||||
bulkSelectionStore.add('doc-99');
|
||||
const item = makeItem({ document: { ...makeItem().document, id: 'doc-99' } });
|
||||
const item = makeItem({ id: 'doc-99' });
|
||||
render(DocumentRow, { item, canWrite: true });
|
||||
await expect.element(page.getByRole('checkbox')).toBeChecked();
|
||||
});
|
||||
|
||||
@@ -20,10 +20,31 @@ const { default: DocumentRow } = await import('./DocumentRow.svelte');
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
const sender = { id: 's1', displayName: 'Anna Schmidt' };
|
||||
const receiver = { id: 'r1', displayName: 'Bert Meier' };
|
||||
const sender = {
|
||||
id: 's1',
|
||||
lastName: 'Schmidt',
|
||||
displayName: 'Anna Schmidt',
|
||||
personType: 'PERSON' as const,
|
||||
familyMember: false
|
||||
};
|
||||
const receiver = {
|
||||
id: 'r1',
|
||||
lastName: 'Meier',
|
||||
displayName: 'Bert Meier',
|
||||
personType: 'PERSON' as const,
|
||||
familyMember: false
|
||||
};
|
||||
|
||||
const makeDoc = (overrides: Record<string, unknown> = {}) => ({
|
||||
const emptyMatchData = {
|
||||
titleOffsets: [],
|
||||
senderMatched: false,
|
||||
matchedReceiverIds: [],
|
||||
matchedTagIds: [],
|
||||
snippetOffsets: [],
|
||||
summaryOffsets: []
|
||||
};
|
||||
|
||||
const baseItem = (overrides: Record<string, unknown> = {}) => ({
|
||||
id: 'd1',
|
||||
title: 'Brief 1923',
|
||||
originalFilename: 'b.pdf',
|
||||
@@ -31,20 +52,14 @@ const makeDoc = (overrides: Record<string, unknown> = {}) => ({
|
||||
sender,
|
||||
receivers: [receiver],
|
||||
tags: [],
|
||||
thumbnailUrl: null,
|
||||
contentType: 'application/pdf',
|
||||
summary: null,
|
||||
archiveBox: null,
|
||||
archiveFolder: null,
|
||||
location: null,
|
||||
...overrides
|
||||
});
|
||||
|
||||
const baseItem = (docOverrides: Record<string, unknown> = {}) => ({
|
||||
document: makeDoc(docOverrides),
|
||||
matchData: null,
|
||||
summary: undefined,
|
||||
archiveBox: undefined,
|
||||
archiveFolder: undefined,
|
||||
location: undefined,
|
||||
matchData: emptyMatchData,
|
||||
completionPercentage: 0,
|
||||
contributors: []
|
||||
contributors: [],
|
||||
...overrides
|
||||
});
|
||||
|
||||
describe('DocumentRow', () => {
|
||||
@@ -121,12 +136,9 @@ describe('DocumentRow', () => {
|
||||
it('renders the snippet when matchData provides a transcriptionSnippet', async () => {
|
||||
render(DocumentRow, {
|
||||
props: {
|
||||
item: {
|
||||
document: makeDoc(),
|
||||
matchData: { transcriptionSnippet: 'Hello world snippet' },
|
||||
completionPercentage: 50,
|
||||
contributors: []
|
||||
}
|
||||
item: baseItem({
|
||||
matchData: { ...emptyMatchData, transcriptionSnippet: 'Hello world snippet' }
|
||||
})
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { cleanup, render } from 'vitest-browser-svelte';
|
||||
import { page } from 'vitest/browser';
|
||||
import TranscriptionBlockHost from './TranscriptionBlock.test-host.svelte';
|
||||
import TranscriptionBlockHost from './TranscriptionBlock.test-fixture.svelte';
|
||||
import type { ConfirmService } from '$lib/shared/services/confirm.svelte.js';
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
@@ -6,6 +6,7 @@ import TranscribeCoachEmptyState from '$lib/shared/help/TranscribeCoachEmptyStat
|
||||
import type { PersonMention, TranscriptionBlockData } from '$lib/shared/types';
|
||||
import { createBlockAutoSave } from '$lib/document/transcription/useBlockAutoSave.svelte';
|
||||
import { createBlockDragDrop } from '$lib/document/transcription/useBlockDragDrop.svelte';
|
||||
import { withCsrf } from '$lib/shared/cookies';
|
||||
|
||||
type Props = {
|
||||
documentId: string;
|
||||
@@ -49,6 +50,7 @@ let activeBlockId: string | null = $state(null);
|
||||
let localLabels: string[] = $derived.by(() => [...trainingLabels]);
|
||||
let listEl: HTMLElement | null = $state(null);
|
||||
let markingAllReviewed = $state(false);
|
||||
let markAllError = $state<string | null>(null);
|
||||
|
||||
const sortedBlocks = $derived([...blocks].sort((a, b) => a.sortOrder - b.sortOrder));
|
||||
const hasBlocks = $derived(blocks.length > 0);
|
||||
@@ -67,8 +69,11 @@ $effect(() => {
|
||||
async function handleMarkAllReviewed() {
|
||||
if (!onMarkAllReviewed) return;
|
||||
markingAllReviewed = true;
|
||||
markAllError = null;
|
||||
try {
|
||||
await onMarkAllReviewed();
|
||||
} catch {
|
||||
markAllError = m.transcription_mark_all_reviewed_error();
|
||||
} finally {
|
||||
markingAllReviewed = false;
|
||||
}
|
||||
@@ -109,11 +114,14 @@ function handleDelete(blockId: string) {
|
||||
|
||||
async function reorder(newOrder: string[]) {
|
||||
try {
|
||||
const res = await fetch(`/api/documents/${documentId}/transcription-blocks/reorder`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ blockIds: newOrder })
|
||||
});
|
||||
const res = await fetch(
|
||||
`/api/documents/${documentId}/transcription-blocks/reorder`,
|
||||
withCsrf({
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ blockIds: newOrder })
|
||||
})
|
||||
);
|
||||
if (!res.ok) return;
|
||||
const updated = await res.json();
|
||||
for (const b of updated) {
|
||||
@@ -169,7 +177,7 @@ async function handleLabelToggle(label: string) {
|
||||
<button
|
||||
onclick={handleMarkAllReviewed}
|
||||
disabled={allReviewed || markingAllReviewed}
|
||||
title={allReviewed ? 'Alle Blöcke sind bereits als fertig markiert' : undefined}
|
||||
title={allReviewed ? m.transcription_mark_all_reviewed_disabled() : undefined}
|
||||
class="flex min-h-[44px] items-center gap-1.5 rounded-sm px-3 font-sans text-xs font-medium text-brand-navy/80 transition-colors hover:text-brand-navy focus-visible:ring-2 focus-visible:ring-brand-navy disabled:opacity-40"
|
||||
>
|
||||
{#if markingAllReviewed}
|
||||
@@ -207,7 +215,7 @@ async function handleLabelToggle(label: string) {
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M5 13l4 4L19 7" />
|
||||
</svg>
|
||||
{/if}
|
||||
Alle als fertig markieren
|
||||
{m.transcription_mark_all_reviewed()}
|
||||
</button>
|
||||
{/if}
|
||||
</div>
|
||||
@@ -217,6 +225,31 @@ async function handleLabelToggle(label: string) {
|
||||
style="width: {reviewProgress}%"
|
||||
></div>
|
||||
</div>
|
||||
{#if markAllError}
|
||||
<div
|
||||
role="alert"
|
||||
class="mt-1.5 flex items-center gap-2 rounded-sm border border-red-200 bg-red-50 px-3 py-2 font-sans text-sm text-red-700"
|
||||
>
|
||||
<span class="flex-1">{markAllError}</span>
|
||||
<button
|
||||
onclick={() => (markAllError = null)}
|
||||
aria-label={m.comp_dismiss()}
|
||||
class="flex min-h-[44px] min-w-[44px] items-center justify-center rounded text-red-600 hover:text-red-700 focus-visible:ring-2 focus-visible:ring-red-500"
|
||||
>
|
||||
<svg
|
||||
class="h-4 w-4"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
viewBox="0 0 24 24"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" />
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
<div class="p-4">
|
||||
<!-- svelte-ignore a11y_no_static_element_interactions -->
|
||||
@@ -303,7 +336,9 @@ async function handleLabelToggle(label: string) {
|
||||
|
||||
{#if canWrite && hasBlocks}
|
||||
<div class="border-t border-line px-4 py-3">
|
||||
<p class="mb-2 font-sans text-xs font-medium text-ink-2">Für Training vormerken</p>
|
||||
<p class="mb-2 font-sans text-xs font-medium text-ink-2">
|
||||
{m.transcribe_mark_for_training()}
|
||||
</p>
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{#each [{ label: 'KURRENT_RECOGNITION', display: m.training_chip_kurrent() }, { label: 'KURRENT_SEGMENTATION', display: m.training_chip_segmentation() }] as chip (chip.label)}
|
||||
<button
|
||||
|
||||
@@ -3,6 +3,7 @@ import { cleanup, render } from 'vitest-browser-svelte';
|
||||
import { page, userEvent } from 'vitest/browser';
|
||||
import TranscriptionEditView from './TranscriptionEditView.svelte';
|
||||
import { createConfirmService, CONFIRM_KEY } from '$lib/shared/services/confirm.svelte.js';
|
||||
import { m } from '$lib/paraglide/messages.js';
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
@@ -312,14 +313,14 @@ describe('TranscriptionEditView — mark all reviewed', () => {
|
||||
onMarkAllReviewed: vi.fn().mockResolvedValue(undefined)
|
||||
});
|
||||
await expect
|
||||
.element(page.getByRole('button', { name: /Alle als fertig markieren/ }))
|
||||
.element(page.getByRole('button', { name: m.transcription_mark_all_reviewed() }))
|
||||
.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('does not show "Alle als fertig markieren" button when onMarkAllReviewed is not provided', async () => {
|
||||
renderView({ blocks: [unreviewedBlock1, unreviewedBlock2] });
|
||||
await expect
|
||||
.element(page.getByRole('button', { name: /Alle als fertig markieren/ }))
|
||||
.element(page.getByRole('button', { name: m.transcription_mark_all_reviewed() }))
|
||||
.not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
@@ -329,7 +330,7 @@ describe('TranscriptionEditView — mark all reviewed', () => {
|
||||
onMarkAllReviewed: vi.fn().mockResolvedValue(undefined)
|
||||
});
|
||||
await expect
|
||||
.element(page.getByRole('button', { name: /Alle als fertig markieren/ }))
|
||||
.element(page.getByRole('button', { name: m.transcription_mark_all_reviewed() }))
|
||||
.toBeDisabled();
|
||||
});
|
||||
|
||||
@@ -343,7 +344,7 @@ describe('TranscriptionEditView — mark all reviewed', () => {
|
||||
// userEvent.click() via Playwright CDP doesn't reliably trigger Svelte 5 onclick
|
||||
// handlers when a TipTap editor is mounted in the same component tree.
|
||||
const btn = (await page
|
||||
.getByRole('button', { name: /Alle als fertig markieren/ })
|
||||
.getByRole('button', { name: m.transcription_mark_all_reviewed() })
|
||||
.element()) as HTMLButtonElement;
|
||||
btn.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await vi.waitFor(() => expect(onMarkAllReviewed).toHaveBeenCalledTimes(1));
|
||||
@@ -361,12 +362,83 @@ describe('TranscriptionEditView — mark all reviewed', () => {
|
||||
|
||||
// Same CDP click workaround: dispatch from browser JS to reliably fire Svelte 5 onclick
|
||||
const btnEl = (await page
|
||||
.getByRole('button', { name: /Alle als fertig markieren/ })
|
||||
.getByRole('button', { name: m.transcription_mark_all_reviewed() })
|
||||
.element()) as HTMLButtonElement;
|
||||
btnEl.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await expect
|
||||
.element(page.getByRole('button', { name: /Alle als fertig markieren/ }))
|
||||
.element(page.getByRole('button', { name: m.transcription_mark_all_reviewed() }))
|
||||
.toBeDisabled();
|
||||
resolveMarkAll();
|
||||
});
|
||||
|
||||
it('shows error message when onMarkAllReviewed callback rejects', async () => {
|
||||
const onMarkAllReviewed = vi.fn().mockRejectedValue(new Error('INTERNAL_ERROR'));
|
||||
renderView({ blocks: [unreviewedBlock1, unreviewedBlock2], onMarkAllReviewed });
|
||||
|
||||
const btnEl = (await page
|
||||
.getByRole('button', { name: m.transcription_mark_all_reviewed() })
|
||||
.element()) as HTMLButtonElement;
|
||||
btnEl.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
|
||||
await expect.element(page.getByRole('alert')).toBeInTheDocument();
|
||||
await expect
|
||||
.element(page.getByRole('alert'))
|
||||
.toHaveTextContent(m.transcription_mark_all_reviewed_error());
|
||||
});
|
||||
|
||||
it('clears error when dismiss button is clicked', async () => {
|
||||
const onMarkAllReviewed = vi.fn().mockRejectedValue(new Error('INTERNAL_ERROR'));
|
||||
renderView({ blocks: [unreviewedBlock1, unreviewedBlock2], onMarkAllReviewed });
|
||||
|
||||
const btnEl = (await page
|
||||
.getByRole('button', { name: m.transcription_mark_all_reviewed() })
|
||||
.element()) as HTMLButtonElement;
|
||||
btnEl.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await expect.element(page.getByRole('alert')).toBeInTheDocument();
|
||||
|
||||
const dismissEl = (await page
|
||||
.getByRole('button', { name: m.comp_dismiss() })
|
||||
.element()) as HTMLButtonElement;
|
||||
dismissEl.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
|
||||
await expect.element(page.getByRole('alert')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('clears error on next successful markAllReviewed call', async () => {
|
||||
const onMarkAllReviewed = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('INTERNAL_ERROR'))
|
||||
.mockResolvedValue(undefined);
|
||||
renderView({ blocks: [unreviewedBlock1, unreviewedBlock2], onMarkAllReviewed });
|
||||
|
||||
const btnEl = (await page
|
||||
.getByRole('button', { name: m.transcription_mark_all_reviewed() })
|
||||
.element()) as HTMLButtonElement;
|
||||
btnEl.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await expect.element(page.getByRole('alert')).toBeInTheDocument();
|
||||
// Wait for the button to be re-enabled before the second click — ensures the first
|
||||
// async rejection has fully settled and Svelte has flushed state changes
|
||||
await expect
|
||||
.element(page.getByRole('button', { name: m.transcription_mark_all_reviewed() }))
|
||||
.not.toBeDisabled();
|
||||
|
||||
btnEl.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
|
||||
await expect.element(page.getByRole('alert')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('re-enables button after markAllReviewed failure', async () => {
|
||||
const onMarkAllReviewed = vi.fn().mockRejectedValue(new Error('INTERNAL_ERROR'));
|
||||
renderView({ blocks: [unreviewedBlock1, unreviewedBlock2], onMarkAllReviewed });
|
||||
|
||||
const btnEl = (await page
|
||||
.getByRole('button', { name: m.transcription_mark_all_reviewed() })
|
||||
.element()) as HTMLButtonElement;
|
||||
btnEl.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await expect.element(page.getByRole('alert')).toBeInTheDocument();
|
||||
|
||||
await expect
|
||||
.element(page.getByRole('button', { name: m.transcription_mark_all_reviewed() }))
|
||||
.not.toBeDisabled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { SvelteMap } from 'svelte/reactivity';
|
||||
import type { PersonMention } from '$lib/shared/types';
|
||||
import { withCsrf } from '$lib/shared/cookies';
|
||||
|
||||
export type SaveState = 'idle' | 'saving' | 'saved' | 'fading' | 'error';
|
||||
|
||||
@@ -116,12 +117,15 @@ export function createBlockAutoSave({ saveFn, documentId }: Options) {
|
||||
for (const [blockId, text] of pendingTexts) {
|
||||
const mentions = pendingMentions.get(blockId) ?? [];
|
||||
clearDebounce(blockId);
|
||||
void fetch(`/api/documents/${documentId}/transcription-blocks/${blockId}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text, mentionedPersons: mentions }),
|
||||
keepalive: true
|
||||
});
|
||||
void fetch(
|
||||
`/api/documents/${documentId}/transcription-blocks/${blockId}`,
|
||||
withCsrf({
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text, mentionedPersons: mentions }),
|
||||
keepalive: true
|
||||
})
|
||||
);
|
||||
pendingTexts.delete(blockId);
|
||||
pendingMentions.delete(blockId);
|
||||
}
|
||||
|
||||
@@ -259,12 +259,15 @@ describe('createTranscriptionBlocks.markAllReviewed', () => {
|
||||
expect(ctrl.blocks.every((b) => b.reviewed)).toBe(true);
|
||||
});
|
||||
|
||||
it('is a no-op when PUT returns non-OK', async () => {
|
||||
it('throws and leaves blocks unchanged when PUT returns non-OK', async () => {
|
||||
const fetchImpl = vi.fn(async (url: RequestInfo | URL, init?: RequestInit) => {
|
||||
const u = url.toString();
|
||||
const method = init?.method ?? 'GET';
|
||||
if (u.includes('/review-all') && method === 'PUT') {
|
||||
return new Response('', { status: 500 });
|
||||
return new Response(JSON.stringify({ code: 'INTERNAL_ERROR' }), {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify([baseBlock({ id: 'b-1', reviewed: false })]), {
|
||||
status: 200,
|
||||
@@ -274,7 +277,26 @@ describe('createTranscriptionBlocks.markAllReviewed', () => {
|
||||
|
||||
const ctrl = createTranscriptionBlocks({ documentId: () => 'doc-1', fetchImpl });
|
||||
await ctrl.load();
|
||||
await ctrl.markAllReviewed();
|
||||
await expect(ctrl.markAllReviewed()).rejects.toThrow('INTERNAL_ERROR');
|
||||
expect(ctrl.blocks[0].reviewed).toBe(false);
|
||||
});
|
||||
|
||||
it('throws INTERNAL_ERROR when PUT returns non-JSON body (e.g. nginx 502)', async () => {
|
||||
const fetchImpl = vi.fn(async (url: RequestInfo | URL, init?: RequestInit) => {
|
||||
const u = url.toString();
|
||||
const method = init?.method ?? 'GET';
|
||||
if (u.includes('/review-all') && method === 'PUT') {
|
||||
return new Response('Bad Gateway', { status: 502 });
|
||||
}
|
||||
return new Response(JSON.stringify([baseBlock({ id: 'b-1', reviewed: false })]), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
});
|
||||
|
||||
const ctrl = createTranscriptionBlocks({ documentId: () => 'doc-1', fetchImpl });
|
||||
await ctrl.load();
|
||||
await expect(ctrl.markAllReviewed()).rejects.toThrow('INTERNAL_ERROR');
|
||||
expect(ctrl.blocks[0].reviewed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
lastEditedAt's $derived are scope-local to one computation; they're never
|
||||
stored on $state. */
|
||||
import type { TranscriptionBlockData, PersonMention } from '$lib/shared/types';
|
||||
import { makeCsrfFetch } from '$lib/shared/cookies';
|
||||
import { saveBlockWithConflictRetry } from './saveBlockWithConflictRetry';
|
||||
import { BlockConflictResolvedError } from './blockConflictMerge';
|
||||
|
||||
@@ -41,7 +42,7 @@ export function createTranscriptionBlocks(
|
||||
options: TranscriptionBlocksOptions
|
||||
): TranscriptionBlocksController {
|
||||
const { documentId } = options;
|
||||
const fetchImpl = options.fetchImpl ?? fetch;
|
||||
const fetchImpl = makeCsrfFetch(options.fetchImpl ?? fetch);
|
||||
|
||||
let blocks = $state<TranscriptionBlockData[]>([]);
|
||||
let annotationReloadKey = $state(0);
|
||||
@@ -119,7 +120,11 @@ export function createTranscriptionBlocks(
|
||||
const res = await fetchImpl(`/api/documents/${documentId()}/transcription-blocks/review-all`, {
|
||||
method: 'PUT'
|
||||
});
|
||||
if (!res.ok) return;
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}));
|
||||
// Never render body.message — route through getErrorMessage() to prevent leaking backend internals
|
||||
throw new Error((body as { code?: string })?.code ?? 'INTERNAL_ERROR');
|
||||
}
|
||||
const updated = (await res.json()) as { id: string; reviewed: boolean }[];
|
||||
for (const b of updated) {
|
||||
const existing = blocks.find((x) => x.id === b.id);
|
||||
|
||||
@@ -34,7 +34,7 @@ let {
|
||||
<button
|
||||
onclick={onPrev}
|
||||
disabled={currentPage <= 1}
|
||||
aria-label="Zurück"
|
||||
aria-label={m.viewer_previous_page()}
|
||||
class="min-h-[44px] min-w-[44px] rounded p-2 text-ink-3 transition hover:bg-surface/10 focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:ring-offset-1 disabled:opacity-40"
|
||||
>
|
||||
<svg class="h-4 w-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
@@ -51,7 +51,7 @@ let {
|
||||
<button
|
||||
onclick={onNext}
|
||||
disabled={!isLoaded || currentPage >= totalPages}
|
||||
aria-label="Weiter"
|
||||
aria-label={m.viewer_next_page()}
|
||||
class="min-h-[44px] min-w-[44px] rounded p-2 text-ink-3 transition hover:bg-surface/10 focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:ring-offset-1 disabled:opacity-40"
|
||||
>
|
||||
<svg class="h-4 w-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
@@ -64,7 +64,7 @@ let {
|
||||
<div class="flex items-center gap-1">
|
||||
<button
|
||||
onclick={onZoomOut}
|
||||
aria-label="Verkleinern"
|
||||
aria-label={m.viewer_zoom_out()}
|
||||
class="min-h-[44px] min-w-[44px] rounded p-2 text-ink-3 transition hover:bg-surface/10 focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:ring-offset-1"
|
||||
>
|
||||
<svg class="h-4 w-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
@@ -74,7 +74,7 @@ let {
|
||||
</button>
|
||||
<button
|
||||
onclick={onZoomIn}
|
||||
aria-label="Vergrößern"
|
||||
aria-label={m.viewer_zoom_in()}
|
||||
class="min-h-[44px] min-w-[44px] rounded p-2 text-ink-3 transition hover:bg-surface/10 focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:ring-offset-1"
|
||||
>
|
||||
<svg class="h-4 w-4" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user