Compare commits
157 Commits
fix/dev-do
...
docs/impor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d9cd644ec | ||
|
|
0a3d12b9af | ||
|
|
34e0eec1ba | ||
|
|
f5e2241fe0 | ||
|
|
f96b9fbffc | ||
|
|
a4c2b6289d | ||
|
|
658277e97c | ||
|
|
32d9a33550 | ||
|
|
f5eb227239 | ||
|
|
227116fe2d | ||
|
|
7183d15fe5 | ||
|
|
b52bf60913 | ||
|
|
45e63307bb | ||
|
|
995471082e | ||
|
|
c6137a26a2 | ||
|
|
a3c3f14aea | ||
|
|
19cd17d9cd | ||
|
|
508575eccb | ||
|
|
85372e3669 | ||
|
|
caec92e7de | ||
|
|
eacfd15f8e | ||
|
|
a345bba74b | ||
|
|
098c2c9def | ||
|
|
5d8bb70255 | ||
|
|
bca3f34cec | ||
|
|
f1fc3dc1ce | ||
|
|
268c31a49b | ||
|
|
39a462b2bb | ||
|
|
5f2ef823e1 | ||
|
|
929acf6964 | ||
|
|
362672cdbf | ||
|
|
1e3e420860 | ||
|
|
3a758393bf | ||
|
|
1a0be4130e | ||
|
|
98f8c0129a | ||
|
|
79e9cc5a2b | ||
|
|
300b236d7d | ||
|
|
6c3552dc6a | ||
|
|
9d859dcb05 | ||
|
|
888adcb185 | ||
|
|
67272178a9 | ||
|
|
529c92fcc3 | ||
|
|
ec357ac13c | ||
|
|
a24764e58a | ||
|
|
09b810afb6 | ||
|
|
4bc96c3772 | ||
|
|
f99673321c | ||
|
|
728078f1e5 | ||
|
|
38f065bc60 | ||
|
|
6cc622b4db | ||
|
|
4169373693 | ||
|
|
8ed5b1e9e3 | ||
|
|
b1b8fa4bed | ||
|
|
2bd5c82826 | ||
|
|
7245571ea8 | ||
|
|
b56b9dfa74 | ||
|
|
6538c9e59a | ||
|
|
c816934391 | ||
|
|
1caae38946 | ||
|
|
f2a74a6064 | ||
|
|
e4a154406e | ||
|
|
151d6aa03f | ||
|
|
fc53e777d5 | ||
|
|
4fa2b83c0d | ||
|
|
e9ddaed76a | ||
|
|
5f53c3670f | ||
|
|
7ebf7acd72 | ||
|
|
2f7ea37466 | ||
|
|
5cf8fd149e | ||
|
|
21c85ff081 | ||
|
|
9cc682cf72 | ||
|
|
459ba14207 | ||
|
|
c56ba6219c | ||
|
|
cbf1984430 | ||
|
|
f6bfb8f030 | ||
|
|
bcd928f12d | ||
|
|
3501382ff5 | ||
|
|
05dd824283 | ||
|
|
aa6de48a71 | ||
|
|
d8588f4b72 | ||
|
|
f6bf7b9f5e | ||
|
|
b959e312b1 | ||
|
|
ae674b14d4 | ||
|
|
c9fb14fd49 | ||
|
|
d959cb54f1 | ||
|
|
6f5ca47543 | ||
|
|
c27c83f58c | ||
|
|
0f07a95bfe | ||
|
|
662927f928 | ||
|
|
0398ebea2c | ||
|
|
99d8229858 | ||
|
|
fee3c7e27d | ||
|
|
fa3f4167e9 | ||
|
|
a2b77e5bfa | ||
|
|
e95c678271 | ||
|
|
b9f06f6c21 | ||
|
|
1136294c1f | ||
|
|
9238cba06a | ||
|
|
2e59c0ef5b | ||
|
|
309436b9a4 | ||
|
|
e326630318 | ||
|
|
34c40cb0ee | ||
|
|
ace41ad209 | ||
|
|
6f55489ec2 | ||
|
|
fa4b6b5fc2 | ||
|
|
1f2351e3c0 | ||
|
|
7012234e6a | ||
|
|
306f3b6fe6 | ||
|
|
47a0770758 | ||
|
|
889d301f16 | ||
|
|
443c7a48db | ||
|
|
9ae1196d1c | ||
|
|
b37fd1728b | ||
|
|
6103d5d229 | ||
|
|
7b483d357a | ||
|
|
94a40237f4 | ||
|
|
5efe3b8a7c | ||
|
|
0f1f9055c3 | ||
|
|
8cac63e938 | ||
|
|
97db718f81 | ||
|
|
06127724de | ||
|
|
7c017eca2a | ||
|
|
97ab9e38df | ||
|
|
f10b80a03f | ||
|
|
6478cc58ae | ||
|
|
a7c45b3a0e | ||
|
|
5ff0c25e10 | ||
|
|
7ba3a29592 | ||
|
|
d314fd9338 | ||
|
|
18d5a1e2da | ||
|
|
df00ea4238 | ||
|
|
ff1a7c07f1 | ||
|
|
366b484815 | ||
|
|
88c8063227 | ||
|
|
3066d3d3ff | ||
|
|
3e7ddea90a | ||
|
|
75b3ca8b9e | ||
|
|
74c4c390fc | ||
|
|
29087319e6 | ||
|
|
53457d9319 | ||
|
|
2d97595e9c | ||
|
|
a177077b40 | ||
|
|
b7a2332861 | ||
|
|
1da1a8d223 | ||
|
|
59715bdccd | ||
|
|
53a661adb6 | ||
|
|
4942c0ea07 | ||
|
|
7edc002ebb | ||
|
|
b43dd6cdd4 | ||
|
|
cff486dda7 | ||
|
|
df14e6b1ee | ||
|
|
1908dde859 | ||
|
|
4845e7a3c1 | ||
|
|
c6cceec6e9 | ||
|
|
8f6f4f2d62 | ||
|
|
6f7aa643c9 | ||
|
|
adfff420a5 |
@@ -65,6 +65,29 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Assert no raw document date rendered via {@html} (CWE-79 — #666)
|
||||
shell: bash
|
||||
run: |
|
||||
# meta_date_raw is untrusted verbatim spreadsheet text — it must render via
|
||||
# Svelte default escaping, never {@html}. This guard flags any {@html ...}
|
||||
# whose expression references a raw-date variable. A comment mentioning
|
||||
# "{@html}" without a raw token inside the braces does NOT match.
|
||||
# The token list MUST cover every variable that carries the raw value:
|
||||
# DocumentDate.svelte exposes it via the `raw` prop, so `\braw\b` is included.
|
||||
# Grow this list whenever a new raw-bearing variable name is introduced.
|
||||
pattern='\{@html[^}]*(metaDateRaw|documentDateRaw|rawDate|\braw\b)'
|
||||
# Self-test: the regex must catch the dangerous forms and ignore the comment form.
|
||||
printf '{@html doc.metaDateRaw}\n' | grep -qP "$pattern" \
|
||||
|| { echo "FAIL: guard self-test — regex missed the unsafe {@html metaDateRaw} form"; exit 1; }
|
||||
printf '{@html raw}\n' | grep -qP "$pattern" \
|
||||
|| { echo "FAIL: guard self-test — regex missed the unsafe {@html raw} form (DocumentDate prop)"; exit 1; }
|
||||
printf 'never use {@html} for this\n' | grep -qvP "$pattern" \
|
||||
|| { echo "FAIL: guard self-test — regex wrongly flagged a {@html} comment"; exit 1; }
|
||||
if grep -rPln "$pattern" --include='*.svelte' frontend/src/; then
|
||||
echo "FAIL: meta_date_raw rendered via {@html} — use default {…} escaping (CWE-79, #666)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Assert no (upload|download)-artifact past v3
|
||||
shell: bash
|
||||
run: |
|
||||
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@@ -26,3 +26,7 @@ node_modules/
|
||||
|
||||
# Repo uses npm; yarn.lock is ignored to avoid double-lockfile drift.
|
||||
frontend/yarn.lock
|
||||
|
||||
**/.venv/
|
||||
**/__pycache__/
|
||||
*.pyc
|
||||
|
||||
@@ -87,7 +87,7 @@ backend/src/main/java/org/raddatz/familienarchiv/
|
||||
├── exception/ DomainException, ErrorCode, GlobalExceptionHandler
|
||||
├── filestorage/ FileService (S3/MinIO)
|
||||
├── geschichte/ Geschichte (story) domain
|
||||
├── importing/ MassImportService
|
||||
├── importing/ CanonicalImportOrchestrator + four loaders (TagTree/PersonRegister/PersonTree/Document) + CanonicalSheetReader
|
||||
├── notification/ Notification domain + SseEmitterRegistry
|
||||
├── ocr/ OCR domain — OcrService, OcrBatchService, training
|
||||
├── person/ Person domain
|
||||
@@ -192,7 +192,8 @@ frontend/src/routes/
|
||||
├── persons/
|
||||
│ ├── [id]/ Person detail
|
||||
│ ├── [id]/edit/ Person edit form
|
||||
│ └── new/ Create person form
|
||||
│ ├── new/ Create person form
|
||||
│ └── review/ Triage view — confirm/rename/merge/delete provisional persons
|
||||
├── briefwechsel/ Bilateral conversation timeline (Briefwechsel)
|
||||
├── aktivitaeten/ Unified activity feed (Chronik)
|
||||
├── geschichten/ Stories — list, [id], [id]/edit, new
|
||||
|
||||
@@ -272,6 +272,7 @@ For multipart/form-data (file uploads): bypass the typed client and use `event.f
|
||||
| Form display | German `dd.mm.yyyy` with auto-dot insertion via `handleDateInput()` |
|
||||
| Wire format | ISO 8601 via a hidden `<input type="hidden" name="documentDate" value={dateIso}>` |
|
||||
| Display | `new Intl.DateTimeFormat('de-DE', …).format(new Date(val + 'T12:00:00'))` |
|
||||
| Honest precision display | `formatDocumentDate(iso, precision, end?, raw?, locale?)` (`$lib/shared/utils/documentDate.ts`) or the `<DocumentDate>` component — renders a document date at exactly its `meta_date_precision` (MONTH → "Juni 1916", never a fabricated day). It mirrors the Java `DocumentTitleFormatter`; both are pinned to `docs/date-label-fixtures.json` so the title and UI labels can't drift. `meta_date_raw` is untrusted — render it via default escaping, never `{@html}` (a CI guard enforces this). |
|
||||
|
||||
### Security checklist (new endpoint)
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ src/main/java/org/raddatz/familienarchiv/
|
||||
├── exception/ # DomainException, ErrorCode, GlobalExceptionHandler
|
||||
├── filestorage/ # FileService (S3/MinIO)
|
||||
├── geschichte/ # Geschichte (story) domain
|
||||
├── importing/ # MassImportService
|
||||
├── importing/ # CanonicalImportOrchestrator + 4 loaders + CanonicalSheetReader
|
||||
├── notification/ # Notification domain + SseEmitterRegistry
|
||||
├── ocr/ # OCR domain — OcrService, OcrBatchService, training
|
||||
├── person/ # Person domain — Person, PersonService, PersonController
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package org.raddatz.familienarchiv.document;
|
||||
|
||||
/**
|
||||
* Precision of a document's date. Verbatim mirror of the import normalizer's
|
||||
* {@code Precision} enum (tools/import-normalizer/dates.py) — the canonical output is the
|
||||
* contract, so there is no translation layer. Do not add, remove, or rename values without
|
||||
* also changing the normalizer; a mismatch silently breaks import idempotency (see ADR-025).
|
||||
*/
|
||||
public enum DatePrecision {
|
||||
DAY,
|
||||
MONTH,
|
||||
SEASON,
|
||||
YEAR,
|
||||
RANGE,
|
||||
APPROX,
|
||||
UNKNOWN
|
||||
}
|
||||
@@ -91,6 +91,29 @@ public class Document {
|
||||
@Column(name = "meta_date")
|
||||
private LocalDate documentDate; // Wann wurde der Brief geschrieben?
|
||||
|
||||
// Precision of documentDate — drives honest rendering ("ca. 1943", "Frühjahr 1943").
|
||||
// Verbatim mirror of the normalizer's Precision enum (see ADR-025).
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "meta_date_precision", nullable = false, length = 16)
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
@Builder.Default
|
||||
private DatePrecision metaDatePrecision = DatePrecision.UNKNOWN;
|
||||
|
||||
// Range end — only set when metaDatePrecision is RANGE (open-ended ranges allowed → may be null).
|
||||
@Column(name = "meta_date_end")
|
||||
private LocalDate metaDateEnd;
|
||||
|
||||
// Original date cell, verbatim, preserved for provenance and "as written" display.
|
||||
@Column(name = "meta_date_raw", columnDefinition = "TEXT")
|
||||
private String metaDateRaw;
|
||||
|
||||
// Raw attribution preserved even when a person is linked via sender/receivers.
|
||||
@Column(name = "sender_text", columnDefinition = "TEXT")
|
||||
private String senderText;
|
||||
|
||||
@Column(name = "receiver_text", columnDefinition = "TEXT")
|
||||
private String receiverText;
|
||||
|
||||
@Column(name = "meta_location")
|
||||
private String location;
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ public class DocumentBatchMetadataDTO {
|
||||
private UUID senderId;
|
||||
private List<UUID> receiverIds;
|
||||
private LocalDate documentDate;
|
||||
private DatePrecision metaDatePrecision;
|
||||
private LocalDate metaDateEnd;
|
||||
private String location;
|
||||
private List<String> tagNames;
|
||||
private Boolean metadataComplete;
|
||||
|
||||
@@ -313,9 +313,10 @@ public class DocumentController {
|
||||
@RequestParam(required = false) String tagQ,
|
||||
@RequestParam(required = false) DocumentStatus status,
|
||||
@RequestParam(required = false) String tagOp,
|
||||
@RequestParam(required = false) Boolean undated,
|
||||
Authentication authentication) {
|
||||
TagOperator operator = "OR".equalsIgnoreCase(tagOp) ? TagOperator.OR : TagOperator.AND;
|
||||
List<UUID> ids = documentService.findIdsForFilter(q, from, to, senderId, receiverId, tags, tagQ, status, operator);
|
||||
List<UUID> ids = documentService.findIdsForFilter(q, from, to, senderId, receiverId, tags, tagQ, status, operator, Boolean.TRUE.equals(undated));
|
||||
if (ids.size() > BULK_EDIT_FILTER_MAX_IDS) {
|
||||
throw DomainException.badRequest(ErrorCode.BULK_EDIT_TOO_MANY_IDS,
|
||||
"Filter matches " + ids.size() + " documents — refine filter (max " + BULK_EDIT_FILTER_MAX_IDS + ")");
|
||||
@@ -375,6 +376,7 @@ public class DocumentController {
|
||||
@Parameter(description = "Sort field") @RequestParam(required = false) DocumentSort sort,
|
||||
@Parameter(description = "Sort direction: ASC or DESC") @RequestParam(required = false, defaultValue = "DESC") String dir,
|
||||
@Parameter(description = "Tag operator: AND (default) or OR") @RequestParam(required = false) String tagOp,
|
||||
@Parameter(description = "Restrict to undated documents (meta_date IS NULL)") @RequestParam(required = false) Boolean undated,
|
||||
// @Max on page guards against overflow when pageable.getOffset() is computed
|
||||
// as page * size — Integer.MAX_VALUE * 50 would wrap to a negative long, which
|
||||
// Hibernate cheerfully turns into an invalid SQL OFFSET.
|
||||
@@ -387,7 +389,7 @@ public class DocumentController {
|
||||
// defaults to AND, which matches the frontend default and keeps old clients working.
|
||||
TagOperator operator = "OR".equalsIgnoreCase(tagOp) ? TagOperator.OR : TagOperator.AND;
|
||||
Pageable pageable = PageRequest.of(page, size);
|
||||
return ResponseEntity.ok(documentService.searchDocuments(q, from, to, senderId, receiverId, tags, tagQ, status, sort, dir, operator, pageable));
|
||||
return ResponseEntity.ok(documentService.searchDocuments(q, from, to, senderId, receiverId, tags, tagQ, status, sort, dir, operator, Boolean.TRUE.equals(undated), pageable));
|
||||
}
|
||||
|
||||
@GetMapping(value = "/density", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
|
||||
@@ -19,6 +19,9 @@ public record DocumentListItem(
|
||||
String originalFilename,
|
||||
String thumbnailUrl,
|
||||
LocalDate documentDate,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
DatePrecision metaDatePrecision,
|
||||
LocalDate metaDateEnd,
|
||||
Person sender,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
List<Person> receivers,
|
||||
|
||||
@@ -15,24 +15,45 @@ public record DocumentSearchResult(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int pageSize,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int totalPages
|
||||
int totalPages,
|
||||
/**
|
||||
* Total number of undated documents (meta_date IS NULL) matching the current
|
||||
* filter context (q/tags/sender/receiver/status) across ALL pages — not the
|
||||
* undated rows on the current page. Computed independently of the "Nur
|
||||
* undatierte" toggle so it never collapses to the page slice (issue #668).
|
||||
*/
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
long undatedCount
|
||||
) {
|
||||
/**
|
||||
* Single-page convenience factory used by empty-result shortcuts and by tests that
|
||||
* don't care about paging. Treats the whole list as page 0 of itself.
|
||||
* don't care about paging. Treats the whole list as page 0 of itself. The undated
|
||||
* count defaults to 0 — the service overlays the real global count via
|
||||
* {@link #withUndatedCount(long)} before returning.
|
||||
*/
|
||||
public static DocumentSearchResult of(List<DocumentListItem> items) {
|
||||
int size = items.size();
|
||||
return new DocumentSearchResult(items, size, 0, size, size == 0 ? 0 : 1);
|
||||
return new DocumentSearchResult(items, size, 0, size, size == 0 ? 0 : 1, 0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Paged factory used by the service when it has a real Pageable + full match count
|
||||
* (e.g. from Spring's Page<T> or from an in-memory sort-then-slice).
|
||||
* (e.g. from Spring's Page<T> or from an in-memory sort-then-slice). The undated
|
||||
* count defaults to 0 — the service overlays the real global count via
|
||||
* {@link #withUndatedCount(long)} before returning.
|
||||
*/
|
||||
public static DocumentSearchResult paged(List<DocumentListItem> slice, Pageable pageable, long totalElements) {
|
||||
int pageSize = pageable.getPageSize();
|
||||
int totalPages = pageSize == 0 ? 0 : (int) ((totalElements + pageSize - 1) / pageSize);
|
||||
return new DocumentSearchResult(slice, totalElements, pageable.getPageNumber(), pageSize, totalPages);
|
||||
return new DocumentSearchResult(slice, totalElements, pageable.getPageNumber(), pageSize, totalPages, 0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a copy with the global undated count overlaid, leaving every other
|
||||
* field untouched. Lets the service compute the count once and attach it to
|
||||
* whichever result shape the search path produced.
|
||||
*/
|
||||
public DocumentSearchResult withUndatedCount(long undatedCount) {
|
||||
return new DocumentSearchResult(items, totalElements, pageNumber, pageSize, totalPages, undatedCount);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,7 +171,7 @@ public class DocumentService {
|
||||
hasFts, ftsIds, null, null,
|
||||
filters.sender(), filters.receiver(),
|
||||
filters.tags(), filters.tagQ(),
|
||||
filters.status(), filters.tagOperator());
|
||||
filters.status(), filters.tagOperator(), false);
|
||||
return documentRepository.findAll(spec).stream()
|
||||
.map(Document::getDocumentDate)
|
||||
.filter(Objects::nonNull)
|
||||
@@ -378,6 +378,7 @@ public class DocumentService {
|
||||
// 1. Einfache Felder Update
|
||||
doc.setTitle(dto.getTitle());
|
||||
doc.setDocumentDate(dto.getDocumentDate());
|
||||
applyDatePrecision(doc, dto);
|
||||
doc.setLocation(dto.getLocation());
|
||||
doc.setTranscription(dto.getTranscription());
|
||||
doc.setSummary(dto.getSummary());
|
||||
@@ -446,6 +447,25 @@ public class DocumentService {
|
||||
return saved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the three date-precision fields only when the DTO carries them.
|
||||
* A null field means "not submitted" — overwriting the stored value with null
|
||||
* would fabricate a precision the user never chose, the exact dishonesty #666
|
||||
* exists to prevent. A row with a genuinely-unknown precision must keep it when
|
||||
* an unrelated edit (e.g. a location typo) is saved.
|
||||
*/
|
||||
private void applyDatePrecision(Document doc, DocumentUpdateDTO dto) {
|
||||
if (dto.getMetaDatePrecision() != null) {
|
||||
doc.setMetaDatePrecision(dto.getMetaDatePrecision());
|
||||
}
|
||||
if (dto.getMetaDateEnd() != null) {
|
||||
doc.setMetaDateEnd(dto.getMetaDateEnd());
|
||||
}
|
||||
if (dto.getMetaDateRaw() != null) {
|
||||
doc.setMetaDateRaw(dto.getMetaDateRaw());
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Document updateDocumentTags(UUID docId, List<String> tagNames) {
|
||||
Document doc = documentRepository.findById(docId)
|
||||
@@ -481,7 +501,8 @@ public class DocumentService {
|
||||
*/
|
||||
@Transactional(readOnly = true)
|
||||
public List<UUID> findIdsForFilter(String text, LocalDate from, LocalDate to, UUID sender, UUID receiver,
|
||||
List<String> tags, String tagQ, DocumentStatus status, TagOperator tagOperator) {
|
||||
List<String> tags, String tagQ, DocumentStatus status, TagOperator tagOperator,
|
||||
boolean undated) {
|
||||
boolean hasText = StringUtils.hasText(text);
|
||||
List<UUID> rankedIds = null;
|
||||
if (hasText) {
|
||||
@@ -490,7 +511,7 @@ public class DocumentService {
|
||||
}
|
||||
|
||||
Specification<Document> spec = buildSearchSpec(
|
||||
hasText, rankedIds, from, to, sender, receiver, tags, tagQ, status, tagOperator);
|
||||
hasText, rankedIds, from, to, sender, receiver, tags, tagQ, status, tagOperator, undated);
|
||||
return documentRepository.findAll(spec).stream().map(Document::getId).toList();
|
||||
}
|
||||
|
||||
@@ -504,7 +525,8 @@ public class DocumentService {
|
||||
LocalDate from, LocalDate to,
|
||||
UUID sender, UUID receiver,
|
||||
List<String> tags, String tagQ,
|
||||
DocumentStatus status, TagOperator tagOperator) {
|
||||
DocumentStatus status, TagOperator tagOperator,
|
||||
boolean undated) {
|
||||
boolean useOrLogic = tagOperator == TagOperator.OR;
|
||||
List<Set<UUID>> expandedTagSets = tagService.expandTagNamesToDescendantIdSets(tags);
|
||||
Specification<Document> textSpec = hasText ? hasIds(ftsIds) : (root, query, cb) -> null;
|
||||
@@ -514,7 +536,8 @@ public class DocumentService {
|
||||
.and(hasReceiver(receiver))
|
||||
.and(hasTags(expandedTagSets, useOrLogic))
|
||||
.and(hasTagPartial(tagQ))
|
||||
.and(hasStatus(status));
|
||||
.and(hasStatus(status))
|
||||
.and(undatedOnly(undated));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -643,22 +666,62 @@ public class DocumentService {
|
||||
}
|
||||
|
||||
// 1. Allgemeine Suche (für das Suchfeld im Frontend)
|
||||
public DocumentSearchResult searchDocuments(String text, LocalDate from, LocalDate to, UUID sender, UUID receiver, List<String> tags, String tagQ, DocumentStatus status, DocumentSort sort, String dir, TagOperator tagOperator, Pageable pageable) {
|
||||
public DocumentSearchResult searchDocuments(String text, LocalDate from, LocalDate to, UUID sender, UUID receiver, List<String> tags, String tagQ, DocumentStatus status, DocumentSort sort, String dir, TagOperator tagOperator, boolean undated, Pageable pageable) {
|
||||
boolean hasText = StringUtils.hasText(text);
|
||||
|
||||
// Pure-text RELEVANCE: push pagination into SQL — skip findAllMatchingIdsByFts entirely (ADR-008).
|
||||
if (isPureTextRelevance(hasText, sort, from, to, sender, receiver, tags, tagQ, status)) {
|
||||
// Pure-text RELEVANCE: push pagination + ts_rank ordering into SQL — skip
|
||||
// findAllMatchingIdsByFts entirely (ADR-008). This must run BEFORE any
|
||||
// findAllMatchingIdsByFts call so the fast path is preserved. An active undated
|
||||
// filter must NOT take this path: it bypasses buildSearchSpec, so the
|
||||
// undatedOnly predicate would be silently dropped. By definition this path has
|
||||
// no date/sender/receiver/tag/status filters, and undated documents are valid
|
||||
// FTS hits already folded into the ranked page, so there is no separate undated
|
||||
// count to report here.
|
||||
if (!undated && isPureTextRelevance(hasText, sort, from, to, sender, receiver, tags, tagQ, status)) {
|
||||
return relevanceSortedPageFromSql(text, pageable);
|
||||
}
|
||||
|
||||
List<UUID> rankedIds = null;
|
||||
if (hasText) {
|
||||
rankedIds = documentRepository.findAllMatchingIdsByFts(text);
|
||||
// FTS matched nothing → no results and, by definition, no undated matches either.
|
||||
if (rankedIds.isEmpty()) return DocumentSearchResult.of(List.of());
|
||||
}
|
||||
|
||||
// Global undated count for the current filter (q/tags/sender/receiver/status),
|
||||
// forcing undatedOnly(true) and IGNORING the user's "Nur undatierte" toggle so
|
||||
// it never collapses to the page slice and never double-counts (issue #668).
|
||||
long undatedCount = countUndatedForFilter(hasText, rankedIds, from, to, sender, receiver, tags, tagQ, status, tagOperator);
|
||||
|
||||
return runSearch(text, hasText, rankedIds, from, to, sender, receiver, tags, tagQ, status, sort, dir, tagOperator, undated, pageable)
|
||||
.withUndatedCount(undatedCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts every undated document (meta_date IS NULL) matching the active filter,
|
||||
* across all pages, independent of the undated toggle. Reuses {@link #buildSearchSpec}
|
||||
* with {@code undated=true} forced so the count tracks q/tags/sender/receiver/status.
|
||||
* A {@code from}/{@code to} range excludes undated rows by the collision rule (#668),
|
||||
* so the count is legitimately 0 inside a date range.
|
||||
*/
|
||||
private long countUndatedForFilter(boolean hasText, List<UUID> ftsIds,
|
||||
LocalDate from, LocalDate to, UUID sender, UUID receiver,
|
||||
List<String> tags, String tagQ, DocumentStatus status, TagOperator tagOperator) {
|
||||
Specification<Document> undatedSpec = buildSearchSpec(
|
||||
hasText, ftsIds, from, to, sender, receiver, tags, tagQ, status, tagOperator, true);
|
||||
return documentRepository.count(undatedSpec);
|
||||
}
|
||||
|
||||
/** The original search dispatch — produces the page slice + totals, sans undated count. */
|
||||
private DocumentSearchResult runSearch(String text, boolean hasText, List<UUID> rankedIds,
|
||||
LocalDate from, LocalDate to, UUID sender, UUID receiver,
|
||||
List<String> tags, String tagQ, DocumentStatus status,
|
||||
DocumentSort sort, String dir, TagOperator tagOperator,
|
||||
boolean undated, Pageable pageable) {
|
||||
// The pure-text RELEVANCE fast path is handled by the caller (searchDocuments)
|
||||
// before findAllMatchingIdsByFts runs, so it never reaches here (ADR-008).
|
||||
Specification<Document> spec = buildSearchSpec(
|
||||
hasText, rankedIds, from, to, sender, receiver, tags, tagQ, status, tagOperator);
|
||||
hasText, rankedIds, from, to, sender, receiver, tags, tagQ, status, tagOperator, undated);
|
||||
|
||||
// SENDER and RECEIVER sorts load the full match set and slice in-memory.
|
||||
// JPA's Sort.by("sender.lastName") generates an INNER JOIN that silently drops
|
||||
@@ -758,6 +821,8 @@ public class DocumentService {
|
||||
doc.getOriginalFilename(),
|
||||
doc.getThumbnailUrl(),
|
||||
doc.getDocumentDate(),
|
||||
doc.getMetaDatePrecision(),
|
||||
doc.getMetaDateEnd(),
|
||||
doc.getSender(),
|
||||
List.copyOf(doc.getReceivers()),
|
||||
List.copyOf(doc.getTags()),
|
||||
@@ -780,7 +845,15 @@ public class DocumentService {
|
||||
private Sort resolveSort(DocumentSort sort, String dir) {
|
||||
Sort.Direction direction = "ASC".equalsIgnoreCase(dir) ? Sort.Direction.ASC : Sort.Direction.DESC;
|
||||
if (sort == null || sort == DocumentSort.DATE || sort == DocumentSort.RELEVANCE) {
|
||||
return Sort.by(direction, "documentDate");
|
||||
// Undated documents (null documentDate) must order last regardless of
|
||||
// direction — Postgres puts NULLs FIRST on ASC by default, which would
|
||||
// surface the undated pile at the top with no explanation (issue #668).
|
||||
// The title tiebreaker gives a stable total order when every row is
|
||||
// null-dated (the "Nur undatierte" filter), so pagination is deterministic.
|
||||
// title is @Column(nullable=false), so it is always present.
|
||||
return Sort.by(
|
||||
new Sort.Order(direction, "documentDate").nullsLast(),
|
||||
Sort.Order.asc("title"));
|
||||
}
|
||||
// SENDER and RECEIVER are sorted in-memory before this method is called
|
||||
return switch (sort) {
|
||||
|
||||
@@ -55,6 +55,12 @@ public class DocumentSpecifications {
|
||||
return (root, query, cb) -> status == null ? null : cb.equal(root.get("status"), status);
|
||||
}
|
||||
|
||||
// Filtert auf undatierte Dokumente (meta_date IS NULL) — für die "Nur undatierte"-Triage.
|
||||
// false → kein Prädikat (no-op), true → documentDate IS NULL (issue #668).
|
||||
public static Specification<Document> undatedOnly(boolean undated) {
|
||||
return (root, query, cb) -> undated ? cb.isNull(root.get("documentDate")) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filtert nach vorausgeweiteten Tag-ID-Sets mit AND- oder OR-Logik.
|
||||
*
|
||||
|
||||
@@ -11,6 +11,11 @@ import org.raddatz.familienarchiv.ocr.ScriptType;
|
||||
public class DocumentUpdateDTO {
|
||||
private String title;
|
||||
private LocalDate documentDate;
|
||||
private DatePrecision metaDatePrecision;
|
||||
private LocalDate metaDateEnd;
|
||||
private String metaDateRaw;
|
||||
private String senderText;
|
||||
private String receiverText;
|
||||
private String location;
|
||||
private String documentLocation;
|
||||
private String archiveBox;
|
||||
|
||||
@@ -40,6 +40,8 @@ public enum ErrorCode {
|
||||
// --- Import ---
|
||||
/** A mass import is already in progress; only one can run at a time. 409 */
|
||||
IMPORT_ALREADY_RUNNING,
|
||||
/** A canonical import artifact is missing, unreadable, or missing a required header. 400 */
|
||||
IMPORT_ARTIFACT_INVALID,
|
||||
|
||||
// --- Thumbnails ---
|
||||
/** A thumbnail backfill is already in progress; only one can run at a time. 409 */
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Runs the four canonical loaders in their real dependency order — encoded explicitly
|
||||
* here, not implied by call order — and owns the async runner plus the {@link ImportStatus}
|
||||
* state machine the admin UI consumes. The orchestrator smoke-checks that all four
|
||||
* artifacts are present before starting, failing fast rather than half-loading tags but no
|
||||
* documents. A malformed artifact (a loader throwing) sets {@code FAILED}; an individual
|
||||
* bad file is surfaced through the {@link ImportStatus.SkippedFile} mechanism instead.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CanonicalImportOrchestrator {
|
||||
|
||||
private static final String TAG_TREE_ARTIFACT = "canonical-tag-tree.xlsx";
|
||||
private static final String PERSONS_ARTIFACT = "canonical-persons.xlsx";
|
||||
private static final String PERSONS_TREE_ARTIFACT = "canonical-persons-tree.json";
|
||||
private static final String DOCUMENTS_ARTIFACT = "canonical-documents.xlsx";
|
||||
|
||||
private final TagTreeImporter tagTreeImporter;
|
||||
private final PersonRegisterImporter personRegisterImporter;
|
||||
private final PersonTreeImporter personTreeImporter;
|
||||
private final DocumentImporter documentImporter;
|
||||
|
||||
@Value("${app.import.dir:/import}")
|
||||
private String canonicalDir;
|
||||
|
||||
private volatile ImportStatus currentStatus = new ImportStatus(
|
||||
ImportStatus.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
|
||||
public ImportStatus getStatus() {
|
||||
return currentStatus;
|
||||
}
|
||||
|
||||
@Async
|
||||
public void runImportAsync() {
|
||||
if (currentStatus.state() == ImportStatus.State.RUNNING) {
|
||||
throw DomainException.conflict(ErrorCode.IMPORT_ALREADY_RUNNING, "A mass import is already in progress");
|
||||
}
|
||||
runImport();
|
||||
}
|
||||
|
||||
/** Synchronous entry point — wrapped by {@link #runImportAsync()} and called directly in tests. */
|
||||
void runImport() {
|
||||
currentStatus = new ImportStatus(ImportStatus.State.RUNNING, "IMPORT_RUNNING",
|
||||
"Import läuft...", 0, List.of(), LocalDateTime.now());
|
||||
try {
|
||||
File tagTree = requireArtifact(TAG_TREE_ARTIFACT);
|
||||
File persons = requireArtifact(PERSONS_ARTIFACT);
|
||||
File personsTree = requireArtifact(PERSONS_TREE_ARTIFACT);
|
||||
File documents = requireArtifact(DOCUMENTS_ARTIFACT);
|
||||
|
||||
// Dependency DAG: documents need persons + tags; the tree needs persons.
|
||||
tagTreeImporter.load(tagTree);
|
||||
personRegisterImporter.load(persons);
|
||||
personTreeImporter.load(personsTree);
|
||||
DocumentImporter.LoadResult result = documentImporter.load(documents);
|
||||
|
||||
currentStatus = new ImportStatus(ImportStatus.State.DONE, "IMPORT_DONE",
|
||||
"Import abgeschlossen. " + result.processed() + " Dokumente verarbeitet.",
|
||||
result.processed(), result.skippedFiles(), currentStatus.startedAt());
|
||||
} catch (DomainException e) {
|
||||
log.error("Canonical import failed: {}", e.getMessage());
|
||||
currentStatus = new ImportStatus(ImportStatus.State.FAILED, "IMPORT_FAILED_ARTIFACT",
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
} catch (Exception e) {
|
||||
log.error("Canonical import failed", e);
|
||||
currentStatus = new ImportStatus(ImportStatus.State.FAILED, "IMPORT_FAILED_INTERNAL",
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
}
|
||||
}
|
||||
|
||||
private File requireArtifact(String name) {
|
||||
File artifact = new File(canonicalDir, name);
|
||||
if (!artifact.isFile()) {
|
||||
throw DomainException.badRequest(ErrorCode.IMPORT_ARTIFACT_INVALID,
|
||||
"Missing canonical artifact: " + name);
|
||||
}
|
||||
return artifact;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.DateUtil;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Value-level POI helper for the canonical import artifacts. No Spring, no domain
|
||||
* knowledge: it opens a workbook, maps the header row to column indices by name, and
|
||||
* yields typed rows whose cells are looked up by header name — the seam that replaces
|
||||
* the old positional {@code @Value app.import.col.*} indices. List columns are split on
|
||||
* the pipe delimiter the normalizer emits.
|
||||
*/
|
||||
public final class CanonicalSheetReader {
|
||||
|
||||
private CanonicalSheetReader() {
|
||||
}
|
||||
|
||||
/** A single data row, addressable by canonical header name (never by index). */
|
||||
public static final class Row {
|
||||
|
||||
private final Map<String, Integer> headerIndex;
|
||||
private final List<String> cells;
|
||||
|
||||
private Row(Map<String, Integer> headerIndex, List<String> cells) {
|
||||
this.headerIndex = headerIndex;
|
||||
this.cells = cells;
|
||||
}
|
||||
|
||||
/** Trimmed cell value for the named header, or "" when absent/blank. */
|
||||
public String get(String header) {
|
||||
Integer index = headerIndex.get(header);
|
||||
if (index == null || index >= cells.size()) return "";
|
||||
String value = cells.get(index);
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads all data rows from the first sheet, validating that every required header is
|
||||
* present. Throws a fail-closed {@link DomainException} on a missing header so a
|
||||
* loader never silently maps the wrong column.
|
||||
*/
|
||||
public static List<Row> readRows(File file, List<String> requiredHeaders) {
|
||||
try (FileInputStream fis = new FileInputStream(file);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
org.apache.poi.ss.usermodel.Row headerRow = sheet.getRow(sheet.getFirstRowNum());
|
||||
Map<String, Integer> headerIndex = mapHeaders(headerRow);
|
||||
requireHeaders(file, headerIndex, requiredHeaders);
|
||||
|
||||
List<Row> rows = new ArrayList<>();
|
||||
for (int i = sheet.getFirstRowNum() + 1; i <= sheet.getLastRowNum(); i++) {
|
||||
org.apache.poi.ss.usermodel.Row poiRow = sheet.getRow(i);
|
||||
if (poiRow == null) continue;
|
||||
rows.add(new Row(headerIndex, readCells(poiRow, headerIndex.size())));
|
||||
}
|
||||
return rows;
|
||||
} catch (DomainException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw DomainException.badRequest(ErrorCode.IMPORT_ARTIFACT_INVALID,
|
||||
"Unreadable canonical artifact: " + file.getName());
|
||||
}
|
||||
}
|
||||
|
||||
/** Splits a pipe-delimited list column into trimmed, non-empty segments. */
|
||||
public static List<String> splitList(String raw) {
|
||||
if (raw == null || raw.isBlank()) return List.of();
|
||||
return Arrays.stream(raw.split("\\|"))
|
||||
.map(String::trim)
|
||||
.filter(s -> !s.isEmpty())
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static Map<String, Integer> mapHeaders(org.apache.poi.ss.usermodel.Row headerRow) {
|
||||
if (headerRow == null) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, Integer> headerIndex = new HashMap<>();
|
||||
for (int c = 0; c < headerRow.getLastCellNum(); c++) {
|
||||
String name = cellToString(headerRow.getCell(c)).trim();
|
||||
if (!name.isEmpty()) headerIndex.putIfAbsent(name, c);
|
||||
}
|
||||
return headerIndex;
|
||||
}
|
||||
|
||||
private static void requireHeaders(File file, Map<String, Integer> headerIndex, List<String> requiredHeaders) {
|
||||
for (String header : requiredHeaders) {
|
||||
if (!headerIndex.containsKey(header)) {
|
||||
throw DomainException.badRequest(ErrorCode.IMPORT_ARTIFACT_INVALID,
|
||||
"Missing required header '" + header + "' in artifact " + file.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static List<String> readCells(org.apache.poi.ss.usermodel.Row poiRow, int columnCount) {
|
||||
int width = Math.max(columnCount, poiRow.getLastCellNum());
|
||||
List<String> cells = new ArrayList<>(width);
|
||||
for (int c = 0; c < width; c++) {
|
||||
cells.add(cellToString(poiRow.getCell(c)));
|
||||
}
|
||||
return cells;
|
||||
}
|
||||
|
||||
private static String cellToString(Cell cell) {
|
||||
if (cell == null) return "";
|
||||
return switch (cell.getCellType()) {
|
||||
case STRING -> cell.getStringCellValue();
|
||||
case NUMERIC -> {
|
||||
if (DateUtil.isCellDateFormatted(cell)) {
|
||||
yield cell.getLocalDateTimeCellValue().toLocalDate().toString();
|
||||
}
|
||||
yield String.valueOf((long) cell.getNumericCellValue());
|
||||
}
|
||||
case BOOLEAN -> String.valueOf(cell.getBooleanCellValue());
|
||||
default -> "";
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.raddatz.familienarchiv.document.DatePrecision;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
import org.raddatz.familienarchiv.document.DocumentService;
|
||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.raddatz.familienarchiv.document.ThumbnailAsyncRunner;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.raddatz.familienarchiv.person.PersonType;
|
||||
import org.raddatz.familienarchiv.person.PersonUpsertCommand;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
|
||||
import org.raddatz.familienarchiv.tag.TagService;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Loads {@code canonical-documents.xlsx} into the document domain. Java performs no
|
||||
* semantic transformation: the normalizer already resolved people to slugs and dates to
|
||||
* ISO values. This loader maps columns by header name, routes each attribution
|
||||
* register-first (always retaining the raw cell in {@code sender_text}/{@code receiver_text}),
|
||||
* parses clean dates, and keeps the S3/thumbnail plumbing.
|
||||
*
|
||||
* <p>The import corpus is uniform — every PDF is named {@code <index>.pdf} flat in the import
|
||||
* dir — so a document's PDF is resolved <em>directly by its index</em>:
|
||||
* {@code importDir.resolve(index + ".pdf")}. The {@code index} is still hostile input
|
||||
* regardless of upstream trust (CWE-22 does not care it came from our Python tool): it is
|
||||
* validated against a strict catalog pattern with {@link #isValidImportIndex} (no path
|
||||
* separators, no {@code .}/{@code ..}, no absolute path, no slash homoglyphs) and the
|
||||
* resolved path is asserted to stay inside the import dir in {@link #resolvePdfByIndex} as
|
||||
* defense-in-depth. The {@code %PDF} magic-byte check still gates upload.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class DocumentImporter {
|
||||
|
||||
static final List<String> REQUIRED_HEADERS = List.of(
|
||||
"index", "sender_person_id", "sender_name",
|
||||
"receiver_person_ids", "receiver_names", "date_iso", "date_raw", "date_precision");
|
||||
|
||||
// Catalog index shape: 1–4 letters (ASCII + Latin-1 letters, e.g. the German "ü" in
|
||||
// "Mü-0001"), one or more hyphens (the corpus has a few "C--0029" data-entry artefacts),
|
||||
// digits, and an optional trailing "x" the normalizer recognises. Anchored, with no
|
||||
// separator / dot / slash characters in the class, so "<index>.pdf" can never traverse.
|
||||
// NOTE: `\d` here is intentionally ASCII-only ([0-9]). Java's java.util.regex matches `\d`
|
||||
// against [0-9] unless Pattern.UNICODE_CHARACTER_CLASS is set — do NOT add that flag, or
|
||||
// Arabic-Indic / fullwidth digits would silently widen the accepted set.
|
||||
private static final Pattern INDEX_PATTERN =
|
||||
Pattern.compile("[A-Za-z\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u00FF]{1,4}-+\\d+x?");
|
||||
|
||||
private final DocumentService documentService;
|
||||
private final PersonService personService;
|
||||
private final TagService tagService;
|
||||
private final S3Client s3Client;
|
||||
private final ThumbnailAsyncRunner thumbnailAsyncRunner;
|
||||
|
||||
@Value("${app.s3.bucket:familienarchiv}")
|
||||
private String bucketName;
|
||||
|
||||
@Value("${app.import.dir:/import}")
|
||||
private String importDir;
|
||||
|
||||
/** Outcome of loading the document sheet: processed count + per-file skips. */
|
||||
public record LoadResult(int processed, List<ImportStatus.SkippedFile> skippedFiles) {}
|
||||
|
||||
// One transaction for the whole sheet keeps the Hibernate session open so an existing
|
||||
// document's lazy receivers collection initialises during an idempotent re-import.
|
||||
// Invoked cross-bean from the orchestrator, so the @Transactional proxy applies.
|
||||
@Transactional
|
||||
public LoadResult load(File artifact) {
|
||||
List<CanonicalSheetReader.Row> rows = CanonicalSheetReader.readRows(artifact, REQUIRED_HEADERS);
|
||||
int processed = 0;
|
||||
List<ImportStatus.SkippedFile> skipped = new ArrayList<>();
|
||||
// 1-based source row number for ops triage breadcrumbs (the spreadsheet header is row 1,
|
||||
// so the first data row is row 2 — matches what an operator sees in the .xlsx).
|
||||
int rowNumber = 1;
|
||||
for (CanonicalSheetReader.Row row : rows) {
|
||||
rowNumber++;
|
||||
String index = row.get("index");
|
||||
if (index.isBlank()) continue;
|
||||
Optional<ImportStatus.SkipReason> skipReason = importRow(row, index, rowNumber);
|
||||
if (skipReason.isPresent()) {
|
||||
skipped.add(new ImportStatus.SkippedFile(index, skipReason.get()));
|
||||
} else {
|
||||
processed++;
|
||||
}
|
||||
}
|
||||
log.info("Imported {} documents from {} ({} skipped)", processed, artifact.getName(), skipped.size());
|
||||
return new LoadResult(processed, skipped);
|
||||
}
|
||||
|
||||
private Optional<ImportStatus.SkipReason> importRow(CanonicalSheetReader.Row row, String index, int rowNumber) {
|
||||
if (!isValidImportIndex(index)) {
|
||||
// Breadcrumb is the source row number, NOT the raw (possibly-hostile) index — an
|
||||
// operator triaging the import can find the offending row in the .xlsx without us
|
||||
// echoing attacker-controlled input into the log.
|
||||
log.warn("Skipping import row {}: index rejected (fails catalog-shape validation)", rowNumber);
|
||||
return Optional.of(ImportStatus.SkipReason.INVALID_FILENAME_PATH_TRAVERSAL);
|
||||
}
|
||||
Optional<File> resolved = resolvePdfByIndex(index, rowNumber);
|
||||
if (resolved.isEmpty()) {
|
||||
// Distinct from the "index rejected" skip above: the index is VALID but no
|
||||
// <index>.pdf is on disk, so the row becomes a normal PLACEHOLDER (not skipped). The
|
||||
// index is a validated catalog id (no hostile content), so it is safe to log here —
|
||||
// this surfaces a corpus that drifts from the "<index>.pdf" assumption (e.g. a file
|
||||
// that arrived under a different name) rather than dropping it silently.
|
||||
log.info("Import row {}: index {} is valid but {}.pdf is absent — creating PLACEHOLDER",
|
||||
rowNumber, index, index);
|
||||
} else {
|
||||
try {
|
||||
if (!isPdfMagicBytes(resolved.get())) {
|
||||
return Optional.of(ImportStatus.SkipReason.INVALID_PDF_SIGNATURE);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Magic-byte check failed for row {}", index, e);
|
||||
return Optional.of(ImportStatus.SkipReason.FILE_READ_ERROR);
|
||||
}
|
||||
}
|
||||
return persist(row, index, resolved);
|
||||
}
|
||||
|
||||
private Optional<ImportStatus.SkipReason> persist(CanonicalSheetReader.Row row, String index, Optional<File> file) {
|
||||
Document existing = documentService.findByOriginalFilename(index).orElse(null);
|
||||
if (existing != null && existing.getStatus() != DocumentStatus.PLACEHOLDER) {
|
||||
return Optional.of(ImportStatus.SkipReason.ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
String s3Key = null;
|
||||
String contentType = null;
|
||||
DocumentStatus status = DocumentStatus.PLACEHOLDER;
|
||||
if (file.isPresent()) {
|
||||
contentType = probeContentType(file.get());
|
||||
s3Key = "documents/" + UUID.randomUUID() + "_" + file.get().getName();
|
||||
try {
|
||||
uploadToS3(file.get(), s3Key, contentType);
|
||||
status = DocumentStatus.UPLOADED;
|
||||
} catch (Exception e) {
|
||||
log.error("S3 upload failed for {}", file.get().getName(), e);
|
||||
return Optional.of(ImportStatus.SkipReason.S3_UPLOAD_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
Document doc = buildDocument(row, index, existing, s3Key, contentType, status);
|
||||
Document saved = documentService.save(doc);
|
||||
if (file.isPresent()) {
|
||||
thumbnailAsyncRunner.dispatchAfterCommit(saved.getId());
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
private Document buildDocument(CanonicalSheetReader.Row row, String index, Document existing,
|
||||
String s3Key, String contentType, DocumentStatus status) {
|
||||
Document doc = existing != null ? existing
|
||||
: Document.builder().originalFilename(index).build();
|
||||
|
||||
String senderName = row.get("sender_name");
|
||||
String receiverNames = row.get("receiver_names");
|
||||
Person sender = resolveSender(row.get("sender_person_id"), senderName);
|
||||
Set<Person> receivers = resolveReceivers(row.get("receiver_person_ids"));
|
||||
|
||||
LocalDate date = parseIsoDate(row.get("date_iso"));
|
||||
DatePrecision precision = parsePrecision(row.get("date_precision"));
|
||||
LocalDate dateEnd = parseIsoDate(row.get("date_end"));
|
||||
String dateRaw = blankToNull(row.get("date_raw"));
|
||||
String location = blankToNull(row.get("location"));
|
||||
|
||||
doc.setTitle(buildTitle(index, date, precision, dateEnd, dateRaw, location));
|
||||
doc.setStatus(status);
|
||||
doc.setFilePath(s3Key);
|
||||
doc.setContentType(contentType);
|
||||
doc.setSender(sender);
|
||||
doc.setSenderText(blankToNull(senderName));
|
||||
// The canonical row is authoritative for receivers/tags (ADR-025): clear then
|
||||
// re-populate so a shrunk set on re-import prunes stale links rather than
|
||||
// accumulating them. The raw sender_text/receiver_text retention is separate.
|
||||
doc.getReceivers().clear();
|
||||
doc.getReceivers().addAll(receivers);
|
||||
doc.setReceiverText(blankToNull(receiverNames));
|
||||
doc.setDocumentDate(date);
|
||||
doc.setMetaDatePrecision(precision);
|
||||
doc.setMetaDateEnd(dateEnd);
|
||||
doc.setMetaDateRaw(dateRaw);
|
||||
doc.setLocation(location);
|
||||
doc.setSummary(blankToNull(row.get("summary")));
|
||||
attachTag(doc, row.get("tags"));
|
||||
doc.setMetadataComplete(doc.getDocumentDate() != null || sender != null || !receivers.isEmpty());
|
||||
return doc;
|
||||
}
|
||||
|
||||
// The title carries the date at the HONEST precision (never a fabricated day) via the
|
||||
// shared DocumentTitleFormatter, plus the location — kept under 20 lines by delegating.
|
||||
private static String buildTitle(String index, LocalDate date, DatePrecision precision,
|
||||
LocalDate end, String raw, String location) {
|
||||
StringBuilder title = new StringBuilder(index);
|
||||
if (date != null && precision != DatePrecision.UNKNOWN) {
|
||||
title.append(" – ").append(DocumentTitleFormatter.formatTitleDate(date, precision, end, raw));
|
||||
}
|
||||
if (location != null && !location.isBlank()) {
|
||||
title.append(" – ").append(location);
|
||||
}
|
||||
return title.toString();
|
||||
}
|
||||
|
||||
// ─── attribution routing — register-first, always retain raw ─────────────────────
|
||||
|
||||
private Person resolveSender(String slug, String rawName) {
|
||||
if (slug.isBlank()) return null;
|
||||
return resolvePerson(slug, rawName);
|
||||
}
|
||||
|
||||
private Set<Person> resolveReceivers(String slugs) {
|
||||
Set<Person> receivers = new LinkedHashSet<>();
|
||||
for (String slug : CanonicalSheetReader.splitList(slugs)) {
|
||||
receivers.add(resolvePerson(slug, slug));
|
||||
}
|
||||
return receivers;
|
||||
}
|
||||
|
||||
private Person resolvePerson(String slug, String rawName) {
|
||||
return personService.findBySourceRef(slug)
|
||||
.orElseGet(() -> personService.upsertBySourceRef(PersonUpsertCommand.builder()
|
||||
.sourceRef(slug)
|
||||
.lastName(blankToNull(rawName) == null ? slug : rawName)
|
||||
.personType(PersonType.PERSON)
|
||||
.provisional(true)
|
||||
.build()));
|
||||
}
|
||||
|
||||
// Authoritative: the canonical row defines the document's tags exactly. Clearing first
|
||||
// means a tag removed from the row is pruned on re-import (ADR-025).
|
||||
private void attachTag(Document doc, String tagPath) {
|
||||
doc.getTags().clear();
|
||||
if (tagPath.isBlank()) return;
|
||||
tagService.findBySourceRef(tagPath).ifPresent(tag -> doc.getTags().add(tag));
|
||||
}
|
||||
|
||||
// ─── clean-value parsing (no semantic logic) ─────────────────────────────────────
|
||||
|
||||
private static LocalDate parseIsoDate(String value) {
|
||||
if (value == null || value.isBlank()) return null;
|
||||
try {
|
||||
return LocalDate.parse(value.trim());
|
||||
} catch (DateTimeParseException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static DatePrecision parsePrecision(String value) {
|
||||
if (value == null || value.isBlank()) return DatePrecision.UNKNOWN;
|
||||
try {
|
||||
return DatePrecision.valueOf(value.trim());
|
||||
} catch (IllegalArgumentException e) {
|
||||
return DatePrecision.UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── file handling + S3 (small ≤20-line methods) ─────────────────────────────────
|
||||
|
||||
private String probeContentType(File file) {
|
||||
try {
|
||||
String probed = Files.probeContentType(file.toPath());
|
||||
return probed != null ? probed : "application/octet-stream";
|
||||
} catch (IOException e) {
|
||||
return "application/octet-stream";
|
||||
}
|
||||
}
|
||||
|
||||
private void uploadToS3(File file, String s3Key, String contentType) {
|
||||
s3Client.putObject(PutObjectRequest.builder()
|
||||
.bucket(bucketName)
|
||||
.key(s3Key)
|
||||
.contentType(contentType)
|
||||
.build(),
|
||||
RequestBody.fromFile(file));
|
||||
}
|
||||
|
||||
// ─── index validation + containment — defense-in-depth, do not weaken ────────────
|
||||
|
||||
// The index is the only thing that drives the on-disk lookup, so it must never contain a
|
||||
// path separator, traversal token, slash homoglyph, null byte, or absolute-path marker —
|
||||
// each guard mirrors the filename guards ported from MassImportService — and it must match
|
||||
// the strict catalog shape so anything unexpected is skipped loudly rather than read.
|
||||
private boolean isValidImportIndex(String index) {
|
||||
if (index == null || index.isBlank()) return false;
|
||||
if (index.contains("/")) return false;
|
||||
if (index.contains("\\")) return false;
|
||||
if (index.contains("∕")) return false; // U+2215 DIVISION SLASH
|
||||
if (index.contains("/")) return false; // U+FF0F FULLWIDTH SOLIDUS
|
||||
if (index.contains("⧵")) return false; // U+29F5 REVERSE SOLIDUS OPERATOR
|
||||
if (index.contains(".")) return false; // no dots — "<index>.pdf" is the only extension
|
||||
if (index.contains("\0")) return false;
|
||||
if (Paths.get(index).isAbsolute()) return false;
|
||||
return INDEX_PATTERN.matcher(index).matches();
|
||||
}
|
||||
|
||||
// package-private: a Mockito spy in tests can override to inject IOException
|
||||
InputStream openFileStream(File file) throws IOException {
|
||||
return new FileInputStream(file);
|
||||
}
|
||||
|
||||
private boolean isPdfMagicBytes(File file) throws IOException {
|
||||
try (InputStream is = openFileStream(file)) {
|
||||
byte[] header = is.readNBytes(4);
|
||||
return header.length == 4
|
||||
&& header[0] == 0x25 // %
|
||||
&& header[1] == 0x50 // P
|
||||
&& header[2] == 0x44 // D
|
||||
&& header[3] == 0x46; // F
|
||||
}
|
||||
}
|
||||
|
||||
// O(1) direct lookup: the PDF is exactly importDir/<index>.pdf. The caller has already
|
||||
// validated the index shape; the canonical-path containment assertion below is
|
||||
// defense-in-depth so even a symlinked <index>.pdf cannot read outside importDir.
|
||||
private Optional<File> resolvePdfByIndex(String index, int rowNumber) {
|
||||
File baseDir = new File(importDir);
|
||||
File candidate = baseDir.toPath().resolve(index + ".pdf").toFile();
|
||||
try {
|
||||
if (!candidate.isFile()) return Optional.empty();
|
||||
String baseDirCanonical = baseDir.getCanonicalPath();
|
||||
if (!candidate.getCanonicalPath().startsWith(baseDirCanonical + File.separator)) {
|
||||
throw DomainException.internal(ErrorCode.INTERNAL_ERROR, "Path escape detected: " + candidate);
|
||||
}
|
||||
return Optional.of(candidate);
|
||||
} catch (IOException e) {
|
||||
// Distinct from the deliberate symlink-escape abort above (which throws): canonical
|
||||
// resolution itself failed (e.g. the OS rejected the path mid-resolution). We fail
|
||||
// safe to a PLACEHOLDER, but never silently — log it so the asymmetry surfaces in ops.
|
||||
log.warn("Canonical path resolution failed for import row {}: treating {}.pdf as absent",
|
||||
rowNumber, index, e);
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
private static String blankToNull(String s) {
|
||||
return (s == null || s.isBlank()) ? null : s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.raddatz.familienarchiv.document.DatePrecision;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Produces the honest German date label baked into an import title — at exactly
|
||||
* the precision the data claims, never finer. This is the Java half of the
|
||||
* single source of truth shared with the frontend {@code formatDocumentDate}
|
||||
* (TypeScript): both are asserted against {@code docs/date-label-fixtures.json}
|
||||
* so the two implementations cannot drift (see #666).
|
||||
*
|
||||
* <p>Import titles are always German, so the labels here are the German
|
||||
* canonical form (mirroring the {@code de} Paraglide messages used by the UI).
|
||||
*/
|
||||
final class DocumentTitleFormatter {
|
||||
|
||||
private static final DateTimeFormatter LONG = DateTimeFormatter.ofPattern("d. MMMM yyyy", Locale.GERMAN);
|
||||
private static final DateTimeFormatter MONTH_YEAR = DateTimeFormatter.ofPattern("MMMM yyyy", Locale.GERMAN);
|
||||
private static final DateTimeFormatter MEDIUM = DateTimeFormatter.ofPattern("d. MMM yyyy", Locale.GERMAN);
|
||||
private static final DateTimeFormatter DAY_MONTH = DateTimeFormatter.ofPattern("d. MMM", Locale.GERMAN);
|
||||
|
||||
private static final String UNKNOWN = "Datum unbekannt";
|
||||
private static final String APPROX_PREFIX = "ca.";
|
||||
private static final String OPEN_RANGE_PREFIX = "ab";
|
||||
|
||||
private DocumentTitleFormatter() {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param date the sort/filter anchor day; null for UNKNOWN rows
|
||||
* @param precision descriptive precision metadata
|
||||
* @param end the RANGE end day; null means an open-ended range
|
||||
* @param raw the verbatim spreadsheet cell, used only to pick a season word
|
||||
* @return the honest German label
|
||||
*/
|
||||
static String formatTitleDate(LocalDate date, DatePrecision precision, LocalDate end, String raw) {
|
||||
if (precision == DatePrecision.UNKNOWN || date == null) {
|
||||
return UNKNOWN;
|
||||
}
|
||||
return switch (precision) {
|
||||
case DAY -> LONG.format(date);
|
||||
case MONTH -> MONTH_YEAR.format(date);
|
||||
case SEASON -> seasonLabel(date, raw);
|
||||
case YEAR -> String.valueOf(date.getYear());
|
||||
case APPROX -> APPROX_PREFIX + " " + date.getYear();
|
||||
case RANGE -> rangeLabel(date, end);
|
||||
case UNKNOWN -> UNKNOWN;
|
||||
};
|
||||
}
|
||||
|
||||
private static String seasonLabel(LocalDate date, String raw) {
|
||||
Season season = seasonFromRaw(raw);
|
||||
if (season == null) {
|
||||
season = seasonOfMonth(date.getMonthValue());
|
||||
}
|
||||
return season.german + " " + date.getYear();
|
||||
}
|
||||
|
||||
private static String rangeLabel(LocalDate start, LocalDate end) {
|
||||
if (end == null) {
|
||||
return OPEN_RANGE_PREFIX + " " + MEDIUM.format(start);
|
||||
}
|
||||
if (end.equals(start)) {
|
||||
return MEDIUM.format(start);
|
||||
}
|
||||
if (start.getYear() != end.getYear()) {
|
||||
return MEDIUM.format(start) + " – " + MEDIUM.format(end);
|
||||
}
|
||||
if (start.getMonthValue() == end.getMonthValue()) {
|
||||
return start.getDayOfMonth() + ".–" + MEDIUM.format(end);
|
||||
}
|
||||
return DAY_MONTH.format(start) + " – " + MEDIUM.format(end);
|
||||
}
|
||||
|
||||
// ─── season mapping — mirrors the normalizer's representative months ─────────────
|
||||
|
||||
private enum Season {
|
||||
SPRING("Frühling"),
|
||||
SUMMER("Sommer"),
|
||||
AUTUMN("Herbst"),
|
||||
WINTER("Winter");
|
||||
|
||||
private final String german;
|
||||
|
||||
Season(String german) {
|
||||
this.german = german;
|
||||
}
|
||||
}
|
||||
|
||||
private static Season seasonOfMonth(int month) {
|
||||
if (month >= 3 && month <= 5) return Season.SPRING;
|
||||
if (month >= 6 && month <= 8) return Season.SUMMER;
|
||||
if (month >= 9 && month <= 11) return Season.AUTUMN;
|
||||
return Season.WINTER;
|
||||
}
|
||||
|
||||
private static Season seasonFromRaw(String raw) {
|
||||
if (raw == null || raw.isBlank()) return null;
|
||||
String token = raw.trim().split("\\s+")[0].toLowerCase(Locale.GERMAN);
|
||||
return switch (token) {
|
||||
case "frühling", "frühjahr" -> Season.SPRING;
|
||||
case "sommer" -> Season.SUMMER;
|
||||
case "herbst" -> Season.AUTUMN;
|
||||
case "winter" -> Season.WINTER;
|
||||
default -> null;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Async import state surfaced to {@code admin/system/ImportStatusCard.svelte} via the
|
||||
* generated types. The shape ({@code state, statusCode, processed, skippedFiles, skipped})
|
||||
* is kept verbatim from the retired MassImportService so the admin UI keeps working.
|
||||
*/
|
||||
public record ImportStatus(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) State state,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) String statusCode,
|
||||
@JsonIgnore String message,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) int processed,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) List<SkippedFile> skippedFiles,
|
||||
LocalDateTime startedAt
|
||||
) {
|
||||
|
||||
public enum State { IDLE, RUNNING, DONE, FAILED }
|
||||
|
||||
public enum SkipReason {
|
||||
INVALID_FILENAME_PATH_TRAVERSAL,
|
||||
INVALID_PDF_SIGNATURE,
|
||||
FILE_READ_ERROR,
|
||||
ALREADY_EXISTS,
|
||||
S3_UPLOAD_FAILED
|
||||
}
|
||||
|
||||
public record SkippedFile(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) String filename,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) SkipReason reason
|
||||
) {}
|
||||
|
||||
// Note: @Schema on a record accessor method is not picked up by SpringDoc; the
|
||||
// "skipped" count is a computed convenience field derived from skippedFiles.size().
|
||||
@JsonProperty("skipped")
|
||||
public int skipped() {
|
||||
return skippedFiles.size();
|
||||
}
|
||||
|
||||
/** Defensive-copy constructor — callers cannot mutate the stored list after construction. */
|
||||
public ImportStatus {
|
||||
skippedFiles = List.copyOf(skippedFiles);
|
||||
}
|
||||
}
|
||||
@@ -1,509 +0,0 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.*;
|
||||
import java.util.Objects;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
import org.raddatz.familienarchiv.document.DocumentService;
|
||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.raddatz.familienarchiv.document.ThumbnailAsyncRunner;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonNameParser;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.raddatz.familienarchiv.tag.TagService;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.zip.ZipFile;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class MassImportService {
|
||||
|
||||
public enum State { IDLE, RUNNING, DONE, FAILED }
|
||||
|
||||
public enum SkipReason {
|
||||
INVALID_FILENAME_PATH_TRAVERSAL,
|
||||
INVALID_PDF_SIGNATURE,
|
||||
FILE_READ_ERROR,
|
||||
ALREADY_EXISTS,
|
||||
S3_UPLOAD_FAILED
|
||||
}
|
||||
|
||||
public record SkippedFile(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) String filename,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) SkipReason reason
|
||||
) {}
|
||||
|
||||
public record ImportStatus(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) State state,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) String statusCode,
|
||||
@JsonIgnore String message,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) int processed,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) List<SkippedFile> skippedFiles,
|
||||
LocalDateTime startedAt
|
||||
) {
|
||||
// Note: @Schema on a record accessor method is not picked up by SpringDoc; the
|
||||
// "skipped" count is a computed convenience field derived from skippedFiles.size().
|
||||
@JsonProperty("skipped")
|
||||
public int skipped() { return skippedFiles.size(); }
|
||||
|
||||
/** Defensive-copy constructor — callers cannot mutate the stored list after construction. */
|
||||
public ImportStatus {
|
||||
skippedFiles = List.copyOf(skippedFiles);
|
||||
}
|
||||
}
|
||||
|
||||
record ProcessResult(int processed, List<SkippedFile> skippedFiles) {}
|
||||
|
||||
private volatile ImportStatus currentStatus = new ImportStatus(State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
|
||||
public ImportStatus getStatus() {
|
||||
return currentStatus;
|
||||
}
|
||||
|
||||
private final DocumentService documentService;
|
||||
private final PersonService personService;
|
||||
private final TagService tagService;
|
||||
private final S3Client s3Client;
|
||||
private final ThumbnailAsyncRunner thumbnailAsyncRunner;
|
||||
|
||||
@Value("${app.s3.bucket}")
|
||||
private String bucketName;
|
||||
|
||||
@Value("${app.import.col.index:0}")
|
||||
private int colIndex;
|
||||
|
||||
@Value("${app.import.col.box:1}")
|
||||
private int colBox;
|
||||
|
||||
@Value("${app.import.col.folder:2}")
|
||||
private int colFolder;
|
||||
|
||||
@Value("${app.import.col.sender:3}")
|
||||
private int colSender;
|
||||
|
||||
@Value("${app.import.col.receivers:5}")
|
||||
private int colReceivers;
|
||||
|
||||
@Value("${app.import.col.date:7}")
|
||||
private int colDate;
|
||||
|
||||
@Value("${app.import.col.location:9}")
|
||||
private int colLocation;
|
||||
|
||||
@Value("${app.import.col.tags:10}")
|
||||
private int colTags;
|
||||
|
||||
@Value("${app.import.col.summary:11}")
|
||||
private int colSummary;
|
||||
|
||||
@Value("${app.import.col.transcription:13}")
|
||||
private int colTranscription;
|
||||
|
||||
@Value("${app.import.dir:/import}")
|
||||
private String importDir;
|
||||
|
||||
private static final DateTimeFormatter GERMAN_DATE = DateTimeFormatter.ofPattern("d. MMMM yyyy", Locale.GERMAN);
|
||||
|
||||
// ODS XML namespaces
|
||||
private static final String NS_TABLE = "urn:oasis:names:tc:opendocument:xmlns:table:1.0";
|
||||
private static final String NS_TEXT = "urn:oasis:names:tc:opendocument:xmlns:text:1.0";
|
||||
|
||||
// We only need up to this many columns; caps repeated-empty-cell expansion
|
||||
private static final int MAX_COLS = 20;
|
||||
|
||||
@Async
|
||||
public void runImportAsync() {
|
||||
if (currentStatus.state() == State.RUNNING) {
|
||||
throw DomainException.conflict(ErrorCode.IMPORT_ALREADY_RUNNING, "A mass import is already in progress");
|
||||
}
|
||||
currentStatus = new ImportStatus(State.RUNNING, "IMPORT_RUNNING", "Import läuft...", 0, List.of(), LocalDateTime.now());
|
||||
try {
|
||||
File spreadsheet = findSpreadsheetFile();
|
||||
log.info("Starte Massenimport aus: {}", spreadsheet.getAbsolutePath());
|
||||
ProcessResult result = processRows(readSpreadsheet(spreadsheet));
|
||||
currentStatus = new ImportStatus(State.DONE, "IMPORT_DONE",
|
||||
"Import abgeschlossen. " + result.processed() + " Dokumente verarbeitet.",
|
||||
result.processed(), result.skippedFiles(), currentStatus.startedAt());
|
||||
} catch (NoSpreadsheetException e) {
|
||||
log.error("Massenimport fehlgeschlagen: keine Tabellendatei", e);
|
||||
currentStatus = new ImportStatus(State.FAILED, "IMPORT_FAILED_NO_SPREADSHEET",
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
} catch (Exception e) {
|
||||
log.error("Massenimport fehlgeschlagen", e);
|
||||
currentStatus = new ImportStatus(State.FAILED, "IMPORT_FAILED_INTERNAL",
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
}
|
||||
}
|
||||
|
||||
private static class NoSpreadsheetException extends RuntimeException {
|
||||
NoSpreadsheetException(String message) { super(message); }
|
||||
}
|
||||
|
||||
private File findSpreadsheetFile() throws IOException {
|
||||
try (Stream<Path> files = Files.list(Paths.get(importDir))) {
|
||||
return files
|
||||
.filter(p -> {
|
||||
String name = p.toString().toLowerCase();
|
||||
return name.endsWith(".ods") || name.endsWith(".xlsx") || name.endsWith(".xls");
|
||||
})
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new NoSpreadsheetException(
|
||||
"Keine Tabellendatei (.ods/.xlsx/.xls) in " + importDir + " gefunden!"))
|
||||
.toFile();
|
||||
}
|
||||
}
|
||||
|
||||
// --- Spreadsheet reading (format-specific, produces neutral List<List<String>>) ---
|
||||
|
||||
private List<List<String>> readSpreadsheet(File file) throws Exception {
|
||||
String name = file.getName().toLowerCase();
|
||||
if (name.endsWith(".ods")) {
|
||||
return readOds(file);
|
||||
}
|
||||
return readXlsx(file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an ODS file by parsing its content.xml directly (no extra library needed).
|
||||
* ODS is a ZIP archive; content.xml holds the spreadsheet data as XML.
|
||||
*/
|
||||
List<List<String>> readOds(File file) throws Exception {
|
||||
List<List<String>> result = new ArrayList<>();
|
||||
|
||||
try (ZipFile zip = new ZipFile(file)) {
|
||||
var entry = zip.getEntry("content.xml");
|
||||
if (entry == null) throw new RuntimeException("Ungültige ODS-Datei: content.xml fehlt");
|
||||
|
||||
var factory = XxeSafeXmlParser.hardenedFactory();
|
||||
factory.setNamespaceAware(true);
|
||||
var builder = factory.newDocumentBuilder();
|
||||
var doc = builder.parse(zip.getInputStream(entry));
|
||||
|
||||
NodeList tables = doc.getElementsByTagNameNS(NS_TABLE, "table");
|
||||
if (tables.getLength() == 0) return result;
|
||||
|
||||
var table = (Element) tables.item(0);
|
||||
NodeList rows = table.getElementsByTagNameNS(NS_TABLE, "table-row");
|
||||
|
||||
for (int i = 0; i < rows.getLength(); i++) {
|
||||
var row = (Element) rows.item(i);
|
||||
List<String> rowData = new ArrayList<>();
|
||||
NodeList cells = row.getElementsByTagNameNS(NS_TABLE, "table-cell");
|
||||
|
||||
for (int j = 0; j < cells.getLength() && rowData.size() < MAX_COLS; j++) {
|
||||
var cell = (Element) cells.item(j);
|
||||
|
||||
// Read the display text (first <text:p>)
|
||||
String value = "";
|
||||
NodeList textNodes = cell.getElementsByTagNameNS(NS_TEXT, "p");
|
||||
if (textNodes.getLength() > 0) {
|
||||
value = textNodes.item(0).getTextContent().trim();
|
||||
}
|
||||
|
||||
// Expand number-columns-repeated (capped at MAX_COLS)
|
||||
String repeatAttr = cell.getAttributeNS(NS_TABLE, "number-columns-repeated");
|
||||
int repeat = repeatAttr.isEmpty() ? 1 : Integer.parseInt(repeatAttr);
|
||||
repeat = Math.min(repeat, MAX_COLS - rowData.size());
|
||||
|
||||
for (int r = 0; r < repeat; r++) {
|
||||
rowData.add(value);
|
||||
}
|
||||
}
|
||||
result.add(rowData);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Reads an XLSX/XLS file using Apache POI. Converts all cells to strings. */
|
||||
private List<List<String>> readXlsx(File file) throws Exception {
|
||||
List<List<String>> result = new ArrayList<>();
|
||||
try (FileInputStream fis = new FileInputStream(file);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
for (int i = 0; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
List<String> rowData = new ArrayList<>();
|
||||
if (row != null) {
|
||||
for (int j = 0; j < MAX_COLS; j++) {
|
||||
rowData.add(xlsxCellToString(row.getCell(j)));
|
||||
}
|
||||
}
|
||||
result.add(rowData);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String xlsxCellToString(Cell cell) {
|
||||
if (cell == null) return "";
|
||||
return switch (cell.getCellType()) {
|
||||
case STRING -> cell.getStringCellValue();
|
||||
case NUMERIC -> {
|
||||
if (DateUtil.isCellDateFormatted(cell)) {
|
||||
yield cell.getLocalDateTimeCellValue().toLocalDate().toString(); // ISO
|
||||
}
|
||||
yield String.valueOf((int) cell.getNumericCellValue());
|
||||
}
|
||||
case BOOLEAN -> String.valueOf(cell.getBooleanCellValue());
|
||||
default -> "";
|
||||
};
|
||||
}
|
||||
|
||||
// --- Import logic (works on neutral List<String> rows) ---
|
||||
|
||||
private ProcessResult processRows(List<List<String>> rows) {
|
||||
int processed = 0;
|
||||
List<SkippedFile> skippedFiles = new ArrayList<>();
|
||||
|
||||
for (int i = 1; i < rows.size(); i++) { // skip header row
|
||||
List<String> cells = rows.get(i);
|
||||
String index = getCell(cells, colIndex);
|
||||
if (index.isBlank()) continue;
|
||||
|
||||
String filename = index.contains(".") ? index : index + ".pdf";
|
||||
if (!isValidImportFilename(filename)) {
|
||||
log.warn("Skipping import row {}: filename rejected — {}", i, filename);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.INVALID_FILENAME_PATH_TRAVERSAL));
|
||||
continue;
|
||||
}
|
||||
Optional<File> fileOnDisk = findFileRecursive(filename);
|
||||
if (fileOnDisk.isEmpty()) {
|
||||
log.warn("Datei nicht gefunden, importiere nur Metadaten: {}", filename);
|
||||
}
|
||||
|
||||
if (fileOnDisk.isPresent()) {
|
||||
try {
|
||||
if (!isPdfMagicBytes(fileOnDisk.get())) {
|
||||
log.warn("Überspringe {}: Datei beginnt nicht mit %PDF-Signatur", filename);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.INVALID_PDF_SIGNATURE));
|
||||
continue;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Fehler beim Prüfen der Magic-Bytes für {}", filename, e);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.FILE_READ_ERROR));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Optional<SkipReason> skipReason = importSingleDocument(cells, fileOnDisk, filename, index);
|
||||
if (skipReason.isPresent()) {
|
||||
skippedFiles.add(new SkippedFile(filename, skipReason.get()));
|
||||
} else {
|
||||
processed++;
|
||||
}
|
||||
}
|
||||
return new ProcessResult(processed, skippedFiles);
|
||||
}
|
||||
|
||||
private boolean isValidImportFilename(String filename) {
|
||||
if (filename == null || filename.isBlank()) return false;
|
||||
if (filename.contains("/")) return false;
|
||||
if (filename.contains("\\")) return false;
|
||||
if (filename.contains("∕")) return false; // U+2215 DIVISION SLASH
|
||||
if (filename.contains("/")) return false; // U+FF0F FULLWIDTH SOLIDUS
|
||||
if (filename.contains("⧵")) return false; // U+29F5 REVERSE SOLIDUS OPERATOR
|
||||
if (filename.contains("..")) return false;
|
||||
if (filename.equals(".")) return false;
|
||||
if (filename.contains("\0")) return false;
|
||||
// Paths.get() is safe here on Linux for all inputs that passed the checks above;
|
||||
// it may throw InvalidPathException for OS-specific illegal chars on Windows,
|
||||
// but those are not reachable in production.
|
||||
if (Paths.get(filename).isAbsolute()) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// package-private: Mockito spy in tests can override to inject IOException
|
||||
InputStream openFileStream(File file) throws IOException {
|
||||
return new FileInputStream(file);
|
||||
}
|
||||
|
||||
private boolean isPdfMagicBytes(File file) throws IOException {
|
||||
try (InputStream is = openFileStream(file)) {
|
||||
byte[] header = is.readNBytes(4);
|
||||
return header.length == 4
|
||||
&& header[0] == 0x25 // %
|
||||
&& header[1] == 0x50 // P
|
||||
&& header[2] == 0x44 // D
|
||||
&& header[3] == 0x46; // F
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports a single document row.
|
||||
*
|
||||
* @return empty Optional on success; an Optional containing the skip reason on failure/skip.
|
||||
*/
|
||||
@Transactional
|
||||
protected Optional<SkipReason> importSingleDocument(List<String> cells, Optional<File> file, String originalFilename, String index) {
|
||||
Optional<Document> existing = documentService.findByOriginalFilename(originalFilename);
|
||||
if (existing.isPresent() && existing.get().getStatus() != DocumentStatus.PLACEHOLDER) {
|
||||
log.info("Dokument {} existiert bereits, überspringe.", originalFilename);
|
||||
return Optional.of(SkipReason.ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
String archiveBox = getCell(cells, colBox);
|
||||
String archiveFolder = getCell(cells, colFolder);
|
||||
String senderRaw = getCell(cells, colSender);
|
||||
String receiversRaw = getCell(cells, colReceivers);
|
||||
LocalDate date = parseDate(getCell(cells, colDate));
|
||||
String location = getCell(cells, colLocation);
|
||||
String tagRaw = getCell(cells, colTags);
|
||||
String summary = getCell(cells, colSummary);
|
||||
String transcription = getCell(cells, colTranscription);
|
||||
|
||||
String s3Key = null;
|
||||
String contentType = null;
|
||||
DocumentStatus status = DocumentStatus.PLACEHOLDER;
|
||||
|
||||
if (file.isPresent()) {
|
||||
try {
|
||||
contentType = Files.probeContentType(file.get().toPath());
|
||||
} catch (IOException e) {
|
||||
contentType = null;
|
||||
}
|
||||
if (contentType == null) contentType = "application/octet-stream";
|
||||
|
||||
s3Key = "documents/" + UUID.randomUUID() + "_" + file.get().getName();
|
||||
try {
|
||||
s3Client.putObject(PutObjectRequest.builder()
|
||||
.bucket(bucketName)
|
||||
.key(s3Key)
|
||||
.contentType(contentType)
|
||||
.build(),
|
||||
RequestBody.fromFile(file.get()));
|
||||
status = DocumentStatus.UPLOADED;
|
||||
} catch (Exception e) {
|
||||
log.error("S3 Upload Fehler für {}", file.get().getName(), e);
|
||||
return Optional.of(SkipReason.S3_UPLOAD_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
Person sender = senderRaw.isBlank() ? null : findOrCreatePerson(senderRaw);
|
||||
List<Person> receivers = PersonNameParser.parseReceivers(receiversRaw).stream()
|
||||
.map(this::findOrCreatePerson)
|
||||
.filter(Objects::nonNull)
|
||||
.toList();
|
||||
|
||||
Tag tag = null;
|
||||
if (!tagRaw.isBlank()) {
|
||||
tag = tagService.findOrCreate(tagRaw);
|
||||
}
|
||||
|
||||
Document doc = existing.orElse(Document.builder()
|
||||
.originalFilename(originalFilename)
|
||||
.build());
|
||||
|
||||
// Heuristic: mark as complete if at least one key field is present in the spreadsheet row
|
||||
boolean metadataComplete = date != null || !senderRaw.isBlank() || !receiversRaw.isBlank();
|
||||
|
||||
doc.setTitle(buildTitle(index, date, location));
|
||||
doc.setFilePath(s3Key);
|
||||
doc.setContentType(contentType);
|
||||
doc.setStatus(status);
|
||||
doc.setArchiveBox(archiveBox.isBlank() ? null : archiveBox);
|
||||
doc.setArchiveFolder(archiveFolder.isBlank() ? null : archiveFolder);
|
||||
doc.setDocumentDate(date);
|
||||
doc.setLocation(location.isBlank() ? null : location);
|
||||
doc.setSummary(summary.isBlank() ? null : summary);
|
||||
doc.setTranscription(transcription.isBlank() ? null : transcription);
|
||||
doc.setSender(sender);
|
||||
doc.getReceivers().addAll(receivers);
|
||||
if (tag != null) doc.getTags().add(tag);
|
||||
doc.setMetadataComplete(metadataComplete);
|
||||
|
||||
Document saved = documentService.save(doc);
|
||||
if (file.isPresent()) {
|
||||
thumbnailAsyncRunner.dispatchAfterCommit(saved.getId());
|
||||
}
|
||||
log.info("Importiert{}: {}", file.isEmpty() ? " (nur Metadaten)" : "", originalFilename);
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
private String getCell(List<String> cells, int col) {
|
||||
if (col >= cells.size()) return "";
|
||||
String val = cells.get(col);
|
||||
return val == null ? "" : val.trim();
|
||||
}
|
||||
|
||||
private LocalDate parseDate(String value) {
|
||||
if (value == null || value.isBlank()) return null;
|
||||
try {
|
||||
return LocalDate.parse(value.trim());
|
||||
} catch (DateTimeParseException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String buildTitle(String index, LocalDate date, String location) {
|
||||
StringBuilder sb = new StringBuilder(index);
|
||||
if (date != null) {
|
||||
sb.append(" \u2013 ").append(date.format(GERMAN_DATE));
|
||||
}
|
||||
if (location != null && !location.isBlank()) {
|
||||
sb.append(" \u2013 ").append(location);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private Person findOrCreatePerson(String rawName) {
|
||||
return personService.findOrCreateByAlias(rawName);
|
||||
}
|
||||
|
||||
private Optional<File> findFileRecursive(String filename) {
|
||||
File baseDir = new File(importDir);
|
||||
try (Stream<Path> walk = Files.walk(baseDir.toPath())) {
|
||||
Optional<Path> match = walk.filter(p -> !Files.isDirectory(p))
|
||||
.filter(p -> p.getFileName().toString().equals(filename))
|
||||
.findFirst();
|
||||
if (match.isEmpty()) return Optional.empty();
|
||||
File candidate = match.get().toFile();
|
||||
String baseDirCanonical = baseDir.getCanonicalPath();
|
||||
if (!candidate.getCanonicalPath().startsWith(baseDirCanonical + File.separator)) {
|
||||
throw DomainException.internal(ErrorCode.INTERNAL_ERROR, "Path escape detected: " + candidate);
|
||||
}
|
||||
return Optional.of(candidate);
|
||||
} catch (IOException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.raddatz.familienarchiv.person.PersonType;
|
||||
import org.raddatz.familienarchiv.person.PersonUpsertCommand;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.File;
|
||||
import java.time.LocalDate;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Loads {@code canonical-persons.xlsx} (the register) into the person domain via
|
||||
* {@link PersonService}, upserting each person by the normalizer {@code person_id}
|
||||
* (source_ref). Register persons are confident identities, so {@code provisional} is
|
||||
* driven by the sheet's already-clean value (normally {@code False}).
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PersonRegisterImporter {
|
||||
|
||||
static final List<String> REQUIRED_HEADERS = List.of("person_id", "last_name", "first_name", "provisional");
|
||||
|
||||
private final PersonService personService;
|
||||
|
||||
public int load(File artifact) {
|
||||
List<CanonicalSheetReader.Row> rows = CanonicalSheetReader.readRows(artifact, REQUIRED_HEADERS);
|
||||
int processed = 0;
|
||||
for (CanonicalSheetReader.Row row : rows) {
|
||||
String personId = row.get("person_id");
|
||||
if (personId.isBlank()) continue;
|
||||
personService.upsertBySourceRef(toCommand(row, personId));
|
||||
processed++;
|
||||
}
|
||||
log.info("Imported {} register persons from {}", processed, artifact.getName());
|
||||
return processed;
|
||||
}
|
||||
|
||||
private PersonUpsertCommand toCommand(CanonicalSheetReader.Row row, String personId) {
|
||||
return PersonUpsertCommand.builder()
|
||||
.sourceRef(personId)
|
||||
.lastName(blankToNull(row.get("last_name")))
|
||||
.firstName(blankToNull(row.get("first_name")))
|
||||
.maidenName(blankToNull(row.get("maiden_name")))
|
||||
.notes(blankToNull(row.get("notes")))
|
||||
.birthYear(yearOf(row.get("birth_date")))
|
||||
.deathYear(yearOf(row.get("death_date")))
|
||||
.personType(PersonType.PERSON)
|
||||
.provisional(Boolean.parseBoolean(row.get("provisional")))
|
||||
.build();
|
||||
}
|
||||
|
||||
private static Integer yearOf(String isoDate) {
|
||||
if (isoDate == null || isoDate.isBlank()) return null;
|
||||
try {
|
||||
return LocalDate.parse(isoDate.trim()).getYear();
|
||||
} catch (DateTimeParseException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String blankToNull(String s) {
|
||||
return (s == null || s.isBlank()) ? null : s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.raddatz.familienarchiv.person.PersonType;
|
||||
import org.raddatz.familienarchiv.person.PersonUpsertCommand;
|
||||
import org.raddatz.familienarchiv.person.relationship.RelationType;
|
||||
import org.raddatz.familienarchiv.person.relationship.RelationshipService;
|
||||
import org.raddatz.familienarchiv.person.relationship.dto.CreateRelationshipRequest;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* Loads {@code canonical-persons-tree.json} into the person + relationship domains.
|
||||
* Tree persons are upserted via {@link PersonService} keyed on the shared
|
||||
* {@code personId} slug (which Phase 1 #670 now emits into the tree), so they reconcile
|
||||
* with the register rather than duplicating it. Relationships reference persons by the
|
||||
* tree's local {@code rowId}; each side is mapped to the upserted person's UUID and
|
||||
* created through {@link RelationshipService} (never the relationship repository —
|
||||
* layering rule). A duplicate relationship on re-import is swallowed for idempotency.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class PersonTreeImporter {
|
||||
|
||||
// The tree JSON is a local implementation detail, not a shared API payload, so the
|
||||
// importer owns its own mapper rather than depending on the web ObjectMapper bean.
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private final PersonService personService;
|
||||
private final RelationshipService relationshipService;
|
||||
|
||||
public int load(File artifact) {
|
||||
JsonNode root = readTree(artifact);
|
||||
Map<String, UUID> idByRowId = upsertPersons(root.path("persons"));
|
||||
int relationships = createRelationships(root.path("relationships"), idByRowId);
|
||||
log.info("Imported {} tree persons and {} relationships from {}",
|
||||
idByRowId.size(), relationships, artifact.getName());
|
||||
return idByRowId.size();
|
||||
}
|
||||
|
||||
private JsonNode readTree(File artifact) {
|
||||
try {
|
||||
return OBJECT_MAPPER.readTree(artifact);
|
||||
} catch (Exception e) {
|
||||
throw DomainException.badRequest(ErrorCode.IMPORT_ARTIFACT_INVALID,
|
||||
"Unreadable canonical artifact: " + artifact.getName());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, UUID> upsertPersons(JsonNode persons) {
|
||||
Map<String, UUID> idByRowId = new HashMap<>();
|
||||
for (JsonNode node : persons) {
|
||||
String personId = text(node, "personId");
|
||||
if (personId.isBlank()) continue;
|
||||
Person person = personService.upsertBySourceRef(toCommand(node, personId));
|
||||
idByRowId.put(text(node, "rowId"), person.getId());
|
||||
}
|
||||
return idByRowId;
|
||||
}
|
||||
|
||||
private PersonUpsertCommand toCommand(JsonNode node, String personId) {
|
||||
return PersonUpsertCommand.builder()
|
||||
.sourceRef(personId)
|
||||
.lastName(blankToNull(text(node, "lastName")))
|
||||
.firstName(blankToNull(text(node, "firstName")))
|
||||
.maidenName(blankToNull(text(node, "maidenName")))
|
||||
.notes(blankToNull(text(node, "notes")))
|
||||
.birthYear(intOrNull(node, "birthYear"))
|
||||
.deathYear(intOrNull(node, "deathYear"))
|
||||
.familyMember(node.path("familyMember").asBoolean(false))
|
||||
.personType(PersonType.PERSON)
|
||||
.provisional(false)
|
||||
.build();
|
||||
}
|
||||
|
||||
private int createRelationships(JsonNode relationships, Map<String, UUID> idByRowId) {
|
||||
int created = 0;
|
||||
for (JsonNode node : relationships) {
|
||||
// Trap: a relationship node's personId / relatedPersonId fields carry the tree's
|
||||
// local rowId (e.g. "row_a"), NOT a person slug. They are resolved through
|
||||
// idByRowId to the upserted person's UUID.
|
||||
UUID person = idByRowId.get(text(node, "personId"));
|
||||
UUID related = idByRowId.get(text(node, "relatedPersonId"));
|
||||
if (person == null || related == null) {
|
||||
log.warn("Skipping tree relationship with unresolved rowId: {} -> {}",
|
||||
text(node, "personId"), text(node, "relatedPersonId"));
|
||||
continue;
|
||||
}
|
||||
if (addRelationshipIdempotently(person, related, text(node, "type"))) {
|
||||
created++;
|
||||
}
|
||||
}
|
||||
return created;
|
||||
}
|
||||
|
||||
private boolean addRelationshipIdempotently(UUID person, UUID related, String type) {
|
||||
try {
|
||||
relationshipService.addRelationship(person,
|
||||
new CreateRelationshipRequest(related, RelationType.valueOf(type), null, null, null));
|
||||
return true;
|
||||
} catch (DomainException e) {
|
||||
if (e.getCode() == ErrorCode.DUPLICATE_RELATIONSHIP
|
||||
|| e.getCode() == ErrorCode.CIRCULAR_RELATIONSHIP) {
|
||||
return false;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private static String text(JsonNode node, String field) {
|
||||
JsonNode value = node.get(field);
|
||||
return value == null || value.isNull() ? "" : value.asText();
|
||||
}
|
||||
|
||||
private static Integer intOrNull(JsonNode node, String field) {
|
||||
JsonNode value = node.get(field);
|
||||
return value == null || value.isNull() ? null : value.asInt();
|
||||
}
|
||||
|
||||
private static String blankToNull(String s) {
|
||||
return (s == null || s.isBlank()) ? null : s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
import org.raddatz.familienarchiv.tag.TagService;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* Loads {@code canonical-tag-tree.xlsx} into the tag domain via {@link TagService},
|
||||
* upserting each tag by its canonical {@code tag_path} (the source_ref). Parent links are
|
||||
* resolved by the parent's path, which is the child path with its last {@code /segment}
|
||||
* stripped. Rows are emitted parents-first by the normalizer, so a parent is always
|
||||
* resolved before any child references it.
|
||||
*/
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class TagTreeImporter {
|
||||
|
||||
static final List<String> REQUIRED_HEADERS = List.of("tag_path", "parent_name", "tag_name");
|
||||
private static final String PATH_SEPARATOR = "/";
|
||||
|
||||
private final TagService tagService;
|
||||
|
||||
public int load(File artifact) {
|
||||
List<CanonicalSheetReader.Row> rows = CanonicalSheetReader.readRows(artifact, REQUIRED_HEADERS);
|
||||
Map<String, UUID> idByPath = new HashMap<>();
|
||||
int processed = 0;
|
||||
for (CanonicalSheetReader.Row row : rows) {
|
||||
String path = row.get("tag_path");
|
||||
if (path.isBlank()) continue;
|
||||
UUID parentId = resolveParentId(path, idByPath);
|
||||
Tag tag = tagService.upsertBySourceRef(path, row.get("tag_name"), parentId);
|
||||
idByPath.put(path, tag.getId());
|
||||
processed++;
|
||||
}
|
||||
log.info("Imported {} tags from {}", processed, artifact.getName());
|
||||
return processed;
|
||||
}
|
||||
|
||||
private UUID resolveParentId(String path, Map<String, UUID> idByPath) {
|
||||
int lastSeparator = path.lastIndexOf(PATH_SEPARATOR);
|
||||
if (lastSeparator < 0) return null;
|
||||
String parentPath = path.substring(0, lastSeparator);
|
||||
return idByPath.get(parentPath);
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
|
||||
class XxeSafeXmlParser {
|
||||
|
||||
private XxeSafeXmlParser() {}
|
||||
|
||||
static DocumentBuilderFactory hardenedFactory() throws ParserConfigurationException {
|
||||
var factory = DocumentBuilderFactory.newInstance();
|
||||
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
|
||||
factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
|
||||
factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
|
||||
factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
|
||||
factory.setXIncludeAware(false);
|
||||
factory.setExpandEntityReferences(false);
|
||||
return factory;
|
||||
}
|
||||
}
|
||||
@@ -57,6 +57,18 @@ public class Person {
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private boolean familyMember = false;
|
||||
|
||||
// The normalizer person_id — join key and re-import idempotency key. Null for manually
|
||||
// created persons; unique among non-null values (see ADR-025).
|
||||
@Column(name = "source_ref")
|
||||
private String sourceRef;
|
||||
|
||||
// A provisional person is one the importer inferred but could not confidently identify.
|
||||
// Distinct from familyMember (a genealogical fact); set true only by the importer (Phase 3).
|
||||
@Column(name = "provisional", nullable = false)
|
||||
@Builder.Default
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
private boolean provisional = false;
|
||||
|
||||
// Entity-graph navigation for JPA JOIN queries (e.g. DocumentSpecifications.hasText).
|
||||
// Uses entity relationship rather than cross-domain repository access, avoiding a
|
||||
// separate DB roundtrip while respecting domain boundaries.
|
||||
|
||||
@@ -22,12 +22,15 @@ import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
|
||||
import jakarta.validation.Valid;
|
||||
import jakarta.validation.constraints.Max;
|
||||
import jakarta.validation.constraints.Min;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/persons")
|
||||
@RequiredArgsConstructor
|
||||
@Validated
|
||||
public class PersonController {
|
||||
|
||||
private final PersonService personService;
|
||||
@@ -35,15 +38,37 @@ public class PersonController {
|
||||
|
||||
@GetMapping
|
||||
@RequirePermission(Permission.READ_ALL)
|
||||
public ResponseEntity<List<PersonSummaryDTO>> getPersons(
|
||||
public ResponseEntity<PersonSearchResult> getPersons(
|
||||
@RequestParam(required = false) String q,
|
||||
@RequestParam(required = false, defaultValue = "0") int size,
|
||||
@RequestParam(required = false) String sort) {
|
||||
if ("documentCount".equals(sort) && size > 0 && q == null) {
|
||||
@RequestParam(required = false) PersonType type,
|
||||
@RequestParam(required = false) Boolean familyOnly,
|
||||
@RequestParam(required = false) Boolean hasDocuments,
|
||||
@RequestParam(required = false) Boolean provisional,
|
||||
// review=true reveals the import noise (transcriber view); absent/false keeps the
|
||||
// clean reader default (familyMember OR documentCount > 0). The explicit filters AND
|
||||
// within whichever base the review flag selects.
|
||||
@RequestParam(required = false, defaultValue = "false") boolean review,
|
||||
@RequestParam(required = false) String sort,
|
||||
@RequestParam(defaultValue = "0") @Min(0) int page,
|
||||
@RequestParam(defaultValue = "50") @Min(1) @Max(100) int size) {
|
||||
// Legacy top-N-by-document-count path (reader dashboard): preserved, wrapped in the
|
||||
// same envelope so /api/persons always returns one shape. It is explicitly NON-paged —
|
||||
// the top-N query returns the complete result, so PersonSearchResult.topN reports an
|
||||
// honest totalElements (= returned count) instead of pretending to be a page slice.
|
||||
if ("documentCount".equals(sort) && q == null) {
|
||||
int safeSize = Math.min(size, 50);
|
||||
return ResponseEntity.ok(personService.findTopByDocumentCount(safeSize));
|
||||
List<PersonSummaryDTO> top = personService.findTopByDocumentCount(safeSize);
|
||||
return ResponseEntity.ok(PersonSearchResult.topN(top));
|
||||
}
|
||||
return ResponseEntity.ok(personService.findAll(q));
|
||||
|
||||
PersonFilter filter = PersonFilter.builder()
|
||||
.type(type)
|
||||
.familyOnly(familyOnly)
|
||||
.hasDocuments(hasDocuments)
|
||||
.provisional(provisional)
|
||||
.readerDefault(!review)
|
||||
.build();
|
||||
return ResponseEntity.ok(personService.search(filter, page, size, q));
|
||||
}
|
||||
|
||||
@GetMapping("/{id}")
|
||||
@@ -110,6 +135,21 @@ public class PersonController {
|
||||
personService.mergePersons(id, UUID.fromString(targetIdStr));
|
||||
}
|
||||
|
||||
// Dedicated state transition that clears the provisional flag. A separate verb (not a
|
||||
// mass-assignable DTO field) so provisional can never be smuggled in via create/update.
|
||||
@PatchMapping("/{id}/confirm")
|
||||
@RequirePermission(Permission.WRITE_ALL)
|
||||
public ResponseEntity<Person> confirmPerson(@PathVariable UUID id) {
|
||||
return ResponseEntity.ok(personService.confirmPerson(id));
|
||||
}
|
||||
|
||||
@DeleteMapping("/{id}")
|
||||
@ResponseStatus(HttpStatus.NO_CONTENT)
|
||||
@RequirePermission(Permission.WRITE_ALL)
|
||||
public void deletePerson(@PathVariable UUID id) {
|
||||
personService.deletePerson(id);
|
||||
}
|
||||
|
||||
// ─── Alias endpoints ────────────────────────────────────────────────────
|
||||
|
||||
@GetMapping("/{id}/aliases")
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package org.raddatz.familienarchiv.person;
|
||||
|
||||
import lombok.Builder;
|
||||
|
||||
/**
|
||||
* The reader/triage filter set for the persons directory, threaded as one value through
|
||||
* {@code PersonController -> PersonService -> PersonRepository}. Each field is nullable:
|
||||
* null means "do not constrain on this dimension".
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code type} — restrict to a single {@link PersonType}.</li>
|
||||
* <li>{@code familyOnly} — when true, only {@code familyMember} persons.</li>
|
||||
* <li>{@code hasDocuments} — when true, only persons with documentCount > 0.</li>
|
||||
* <li>{@code provisional} — match the {@code Person.provisional} flag exactly.</li>
|
||||
* <li>{@code readerDefault} — when true, restrict to {@code familyMember OR documentCount > 0}
|
||||
* (the clean reader view). The explicit filters above AND with this restriction.</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Builder
|
||||
public record PersonFilter(
|
||||
PersonType type,
|
||||
Boolean familyOnly,
|
||||
Boolean hasDocuments,
|
||||
Boolean provisional,
|
||||
boolean readerDefault
|
||||
) {
|
||||
/** The unconstrained "show all" filter (transcriber view, no reader restriction). */
|
||||
public static PersonFilter showAll() {
|
||||
return PersonFilter.builder().readerDefault(false).build();
|
||||
}
|
||||
|
||||
/** The clean reader default: familyMember OR documentCount > 0, no other constraints. */
|
||||
public static PersonFilter cleanDefault() {
|
||||
return PersonFilter.builder().readerDefault(true).build();
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,9 @@ public interface PersonRepository extends JpaRepository<Person, UUID> {
|
||||
// Lookup by full alias string, used during ODS mass import
|
||||
Optional<Person> findByAliasIgnoreCase(String alias);
|
||||
|
||||
// Lookup by the normalizer person_id, used for idempotent canonical re-import (Phase 3).
|
||||
Optional<Person> findBySourceRef(String sourceRef);
|
||||
|
||||
// Exact first+last name match, used for filename-based sender lookup
|
||||
Optional<Person> findByFirstNameIgnoreCaseAndLastNameIgnoreCase(String firstName, String lastName);
|
||||
|
||||
@@ -41,7 +44,7 @@ public interface PersonRepository extends JpaRepository<Person, UUID> {
|
||||
SELECT p.id, p.title, p.first_name AS firstName, p.last_name AS lastName,
|
||||
p.person_type AS personType,
|
||||
p.alias, p.birth_year AS birthYear, p.death_year AS deathYear, p.notes,
|
||||
p.family_member AS familyMember,
|
||||
p.family_member AS familyMember, p.provisional AS provisional,
|
||||
(SELECT COUNT(*) FROM documents d WHERE d.sender_id = p.id)
|
||||
+ (SELECT COUNT(*) FROM document_receivers dr WHERE dr.person_id = p.id) AS documentCount
|
||||
FROM persons p
|
||||
@@ -54,7 +57,7 @@ public interface PersonRepository extends JpaRepository<Person, UUID> {
|
||||
SELECT p.id, p.title, p.first_name AS firstName, p.last_name AS lastName,
|
||||
p.person_type AS personType,
|
||||
p.alias, p.birth_year AS birthYear, p.death_year AS deathYear, p.notes,
|
||||
p.family_member AS familyMember,
|
||||
p.family_member AS familyMember, p.provisional AS provisional,
|
||||
(SELECT COUNT(*) FROM documents d WHERE d.sender_id = p.id)
|
||||
+ (SELECT COUNT(*) FROM document_receivers dr WHERE dr.person_id = p.id) AS documentCount
|
||||
FROM persons p
|
||||
@@ -63,7 +66,7 @@ public interface PersonRepository extends JpaRepository<Person, UUID> {
|
||||
OR LOWER(CONCAT(p.last_name,' ',COALESCE(p.first_name,''))) LIKE LOWER(CONCAT('%',:query,'%'))
|
||||
OR LOWER(p.alias) LIKE LOWER(CONCAT('%',:query,'%'))
|
||||
OR LOWER(a.last_name) LIKE LOWER(CONCAT('%',:query,'%'))
|
||||
GROUP BY p.id, p.title, p.first_name, p.last_name, p.person_type, p.alias, p.birth_year, p.death_year, p.notes, p.family_member
|
||||
GROUP BY p.id, p.title, p.first_name, p.last_name, p.person_type, p.alias, p.birth_year, p.death_year, p.notes, p.family_member, p.provisional
|
||||
ORDER BY p.last_name ASC, p.first_name ASC
|
||||
""",
|
||||
nativeQuery = true)
|
||||
@@ -75,7 +78,7 @@ public interface PersonRepository extends JpaRepository<Person, UUID> {
|
||||
SELECT p.id, p.title, p.first_name AS firstName, p.last_name AS lastName,
|
||||
p.person_type AS personType,
|
||||
p.alias, p.birth_year AS birthYear, p.death_year AS deathYear, p.notes,
|
||||
p.family_member AS familyMember,
|
||||
p.family_member AS familyMember, p.provisional AS provisional,
|
||||
(SELECT COUNT(*) FROM documents d WHERE d.sender_id = p.id)
|
||||
+ (SELECT COUNT(*) FROM document_receivers dr WHERE dr.person_id = p.id) AS documentCount
|
||||
FROM persons p
|
||||
@@ -85,6 +88,61 @@ public interface PersonRepository extends JpaRepository<Person, UUID> {
|
||||
nativeQuery = true)
|
||||
List<PersonSummaryDTO> findTopByDocumentCount(@Param("limit") int limit);
|
||||
|
||||
// --- #667: filter-aware paged directory ---
|
||||
//
|
||||
// The slice query and the count query below MUST keep an IDENTICAL WHERE clause so the
|
||||
// rendered page and totalElements can never drift. Every filter is nullable: a null param
|
||||
// disables that predicate via the `:param IS NULL OR …` idiom. `readerDefault` (a plain
|
||||
// boolean) restricts to "familyMember OR has documents"; the explicit filters AND on top.
|
||||
// documentCount is recomputed inline (not via the SELECT alias) because WHERE cannot
|
||||
// reference a computed alias. All params are named — no string concatenation, no injection.
|
||||
String FILTER_WHERE = """
|
||||
WHERE (CAST(:type AS text) IS NULL OR p.person_type = CAST(:type AS text))
|
||||
AND (:familyOnly = FALSE OR :familyOnly IS NULL OR p.family_member = TRUE)
|
||||
AND (:hasDocuments = FALSE OR :hasDocuments IS NULL OR (
|
||||
(SELECT COUNT(*) FROM documents d WHERE d.sender_id = p.id)
|
||||
+ (SELECT COUNT(*) FROM document_receivers dr WHERE dr.person_id = p.id)) > 0)
|
||||
AND (:provisional IS NULL OR p.provisional = :provisional)
|
||||
AND (:readerDefault = FALSE OR (
|
||||
p.family_member = TRUE OR (
|
||||
(SELECT COUNT(*) FROM documents d WHERE d.sender_id = p.id)
|
||||
+ (SELECT COUNT(*) FROM document_receivers dr WHERE dr.person_id = p.id)) > 0))
|
||||
AND (CAST(:query AS text) IS NULL OR
|
||||
LOWER(CONCAT(COALESCE(p.first_name,''),' ',p.last_name)) LIKE LOWER(CONCAT('%',CAST(:query AS text),'%'))
|
||||
OR LOWER(CONCAT(p.last_name,' ',COALESCE(p.first_name,''))) LIKE LOWER(CONCAT('%',CAST(:query AS text),'%'))
|
||||
OR LOWER(p.alias) LIKE LOWER(CONCAT('%',CAST(:query AS text),'%')))
|
||||
""";
|
||||
|
||||
@Query(value = """
|
||||
SELECT p.id, p.title, p.first_name AS firstName, p.last_name AS lastName,
|
||||
p.person_type AS personType,
|
||||
p.alias, p.birth_year AS birthYear, p.death_year AS deathYear, p.notes,
|
||||
p.family_member AS familyMember, p.provisional AS provisional,
|
||||
(SELECT COUNT(*) FROM documents d WHERE d.sender_id = p.id)
|
||||
+ (SELECT COUNT(*) FROM document_receivers dr WHERE dr.person_id = p.id) AS documentCount
|
||||
FROM persons p
|
||||
""" + FILTER_WHERE + """
|
||||
ORDER BY p.last_name ASC, p.first_name ASC
|
||||
LIMIT :limit OFFSET :offset
|
||||
""",
|
||||
nativeQuery = true)
|
||||
List<PersonSummaryDTO> findByFilter(@Param("type") String type,
|
||||
@Param("familyOnly") Boolean familyOnly,
|
||||
@Param("hasDocuments") Boolean hasDocuments,
|
||||
@Param("provisional") Boolean provisional,
|
||||
@Param("readerDefault") boolean readerDefault,
|
||||
@Param("query") String query,
|
||||
@Param("limit") int limit,
|
||||
@Param("offset") int offset);
|
||||
|
||||
@Query(value = "SELECT COUNT(*) FROM persons p " + FILTER_WHERE, nativeQuery = true)
|
||||
long countByFilter(@Param("type") String type,
|
||||
@Param("familyOnly") Boolean familyOnly,
|
||||
@Param("hasDocuments") Boolean hasDocuments,
|
||||
@Param("provisional") Boolean provisional,
|
||||
@Param("readerDefault") boolean readerDefault,
|
||||
@Param("query") String query);
|
||||
|
||||
// --- Correspondent queries ---
|
||||
|
||||
@Query(value = """
|
||||
@@ -136,6 +194,12 @@ public interface PersonRepository extends JpaRepository<Person, UUID> {
|
||||
@Query(value = "UPDATE documents SET sender_id = :target WHERE sender_id = :source", nativeQuery = true)
|
||||
void reassignSender(@Param("source") UUID source, @Param("target") UUID target);
|
||||
|
||||
// Used by deletePerson: detach a deleted person from documents they sent, so the hard
|
||||
// delete cannot orphan a documents.sender_id FK (the column is nullable).
|
||||
@Modifying
|
||||
@Query(value = "UPDATE documents SET sender_id = NULL WHERE sender_id = :source", nativeQuery = true)
|
||||
void reassignSenderToNull(@Param("source") UUID source);
|
||||
|
||||
@Modifying
|
||||
@Query(value = """
|
||||
INSERT INTO document_receivers (document_id, person_id)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package org.raddatz.familienarchiv.person;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Paged result for the /api/persons list endpoint.
|
||||
*
|
||||
* <p>Hand-written to mirror {@code document/DocumentSearchResult} field-for-field so the
|
||||
* frontend sees one paged shape across the app. Deliberately NOT Spring {@code Page<T>}
|
||||
* (unstable serialized shape across Spring versions, noisy in OpenAPI) and deliberately
|
||||
* NOT a reuse of the document DTO (would couple two feature modules — duplication beats
|
||||
* coupling here).
|
||||
*/
|
||||
public record PersonSearchResult(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
List<PersonSummaryDTO> items,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
long totalElements,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int pageNumber,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int pageSize,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||
int totalPages
|
||||
) {
|
||||
/**
|
||||
* Paged factory: derives {@code totalPages} from the full match count and the page size.
|
||||
* A zero count yields zero pages so the frontend hides the pagination control.
|
||||
*/
|
||||
public static PersonSearchResult paged(List<PersonSummaryDTO> slice, int pageNumber, int pageSize, long totalElements) {
|
||||
int totalPages = pageSize == 0 ? 0 : (int) ((totalElements + pageSize - 1) / pageSize);
|
||||
return new PersonSearchResult(slice, totalElements, pageNumber, pageSize, totalPages);
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-paged factory for the legacy {@code sort=documentCount} top-N dashboard path.
|
||||
* That query returns the <em>complete</em> result in one shot — there is no further page
|
||||
* to fetch — so the envelope reports reality rather than pretending to be a slice of a
|
||||
* larger set: {@code totalElements} equals the number of rows actually returned,
|
||||
* {@code pageSize} equals that same count, and {@code totalPages} is 1 (or 0 when empty).
|
||||
* This avoids the earlier ambiguity where {@code totalElements} looked like a paged total.
|
||||
*/
|
||||
public static PersonSearchResult topN(List<PersonSummaryDTO> all) {
|
||||
int count = all.size();
|
||||
int totalPages = count == 0 ? 0 : 1;
|
||||
return new PersonSearchResult(all, count, 0, count, totalPages);
|
||||
}
|
||||
}
|
||||
@@ -31,20 +31,55 @@ public class PersonService {
|
||||
private final PersonRepository personRepository;
|
||||
private final PersonNameAliasRepository aliasRepository;
|
||||
|
||||
public List<PersonSummaryDTO> findAll(String q) {
|
||||
if (q == null) {
|
||||
return personRepository.findAllWithDocumentCount();
|
||||
}
|
||||
if (q.isBlank()) {
|
||||
return List.of();
|
||||
}
|
||||
return personRepository.searchWithDocumentCount(q.trim());
|
||||
}
|
||||
|
||||
public List<PersonSummaryDTO> findTopByDocumentCount(int limit) {
|
||||
return personRepository.findTopByDocumentCount(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filtered, paginated directory query. The slice and the total are derived from one
|
||||
* shared WHERE clause (see {@link PersonRepository#FILTER_WHERE}) so totalElements can
|
||||
* never drift from the rendered page. {@code type} is passed as the enum name because the
|
||||
* native query compares against the string column.
|
||||
*/
|
||||
public PersonSearchResult search(PersonFilter filter, int page, int size, String q) {
|
||||
String type = filter.type() == null ? null : filter.type().name();
|
||||
String query = (q == null || q.isBlank()) ? null : q.trim();
|
||||
int offset = page * size;
|
||||
|
||||
List<PersonSummaryDTO> items = personRepository.findByFilter(
|
||||
type, filter.familyOnly(), filter.hasDocuments(), filter.provisional(),
|
||||
filter.readerDefault(), query, size, offset);
|
||||
long total = personRepository.countByFilter(
|
||||
type, filter.familyOnly(), filter.hasDocuments(), filter.provisional(),
|
||||
filter.readerDefault(), query);
|
||||
|
||||
return PersonSearchResult.paged(items, page, size, total);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears the {@code provisional} flag — a deliberate state transition exposed as
|
||||
* {@code PATCH /api/persons/{id}/confirm}, never as a mass-assignable DTO field (CWE-915).
|
||||
*/
|
||||
@Transactional
|
||||
public Person confirmPerson(UUID id) {
|
||||
Person person = getById(id);
|
||||
person.setProvisional(false);
|
||||
return personRepository.save(person);
|
||||
}
|
||||
|
||||
/**
|
||||
* Hard-deletes a person used by triage. Detaches the person from any documents they
|
||||
* sent (nulls sender_id) and from any received-document references first, so the delete
|
||||
* cannot orphan an FK and fail with a 500.
|
||||
*/
|
||||
@Transactional
|
||||
public void deletePerson(UUID id) {
|
||||
getById(id);
|
||||
personRepository.reassignSenderToNull(id);
|
||||
personRepository.deleteReceiverReferences(id);
|
||||
personRepository.deleteById(id);
|
||||
}
|
||||
|
||||
public Person getById(UUID id) {
|
||||
return personRepository.findById(id)
|
||||
.orElseThrow(() -> DomainException.notFound(ErrorCode.PERSON_NOT_FOUND, "Person not found: " + id));
|
||||
@@ -80,6 +115,11 @@ public class PersonService {
|
||||
return personRepository.findByFirstNameIgnoreCaseAndLastNameIgnoreCase(firstName, lastName);
|
||||
}
|
||||
|
||||
/** Lookup by the normalizer person_id — used by the canonical importer for register-first matching. */
|
||||
public Optional<Person> findBySourceRef(String sourceRef) {
|
||||
return personRepository.findBySourceRef(sourceRef);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Transactional
|
||||
public Person findOrCreateByAlias(String rawName) {
|
||||
@@ -115,6 +155,80 @@ public class PersonService {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent upsert keyed on {@code sourceRef} (the normalizer person_id) for the
|
||||
* canonical importer (Phase 3, ADR-025). On first import the canonical fields are
|
||||
* written verbatim. On re-import the human-edit-preserve precedence applies:
|
||||
* a non-blank existing field is never overwritten, and {@code provisional} never
|
||||
* flips back to true once a human has confirmed the person.
|
||||
*/
|
||||
@Transactional
|
||||
public Person upsertBySourceRef(PersonUpsertCommand cmd) {
|
||||
return personRepository.findBySourceRef(cmd.sourceRef())
|
||||
.map(existing -> personRepository.save(mergeCanonical(existing, cmd)))
|
||||
.orElseGet(() -> fromCanonical(cmd));
|
||||
}
|
||||
|
||||
private Person fromCanonical(PersonUpsertCommand cmd) {
|
||||
Person person = personRepository.save(Person.builder()
|
||||
.sourceRef(cmd.sourceRef())
|
||||
.firstName(blankToNull(cmd.firstName()))
|
||||
.lastName(cmd.lastName())
|
||||
.notes(blankToNull(cmd.notes()))
|
||||
.birthYear(cmd.birthYear())
|
||||
.deathYear(cmd.deathYear())
|
||||
.familyMember(cmd.familyMember())
|
||||
.personType(cmd.personType() == null ? PersonType.PERSON : cmd.personType())
|
||||
.provisional(cmd.provisional())
|
||||
.build());
|
||||
String maiden = blankToNull(cmd.maidenName());
|
||||
if (maiden != null) {
|
||||
int nextSortOrder = aliasRepository.findMaxSortOrder(person.getId()) + 1;
|
||||
aliasRepository.save(PersonNameAlias.builder()
|
||||
.person(person)
|
||||
.lastName(maiden)
|
||||
.type(PersonNameAliasType.MAIDEN_NAME)
|
||||
.sortOrder(nextSortOrder)
|
||||
.build());
|
||||
}
|
||||
return person;
|
||||
}
|
||||
|
||||
private Person mergeCanonical(Person existing, PersonUpsertCommand cmd) {
|
||||
existing.setFirstName(preferHuman(existing.getFirstName(), cmd.firstName()));
|
||||
existing.setLastName(preferHuman(existing.getLastName(), cmd.lastName()));
|
||||
existing.setNotes(preferHuman(existing.getNotes(), cmd.notes()));
|
||||
existing.setBirthYear(preferHuman(existing.getBirthYear(), cmd.birthYear()));
|
||||
existing.setDeathYear(preferHuman(existing.getDeathYear(), cmd.deathYear()));
|
||||
if (cmd.personType() != null && existing.getPersonType() == PersonType.PERSON) {
|
||||
existing.setPersonType(cmd.personType());
|
||||
}
|
||||
// provisional is monotonic-downward: once it is false it never reverts to true.
|
||||
// This also pins the cross-loader precedence (ADR-025): a register/tree person is
|
||||
// loaded before documents and already false, so a later document row that references
|
||||
// the same source_ref (provisional=true) can never flip it provisional — the guard
|
||||
// below only fires while existing is still provisional. Order of document rows is
|
||||
// therefore irrelevant.
|
||||
if (existing.isProvisional()) {
|
||||
existing.setProvisional(cmd.provisional());
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
// preferHuman keeps an existing human-entered value and only falls back to the canonical
|
||||
// value when the existing one is absent — the single idiom for every fill-blank field.
|
||||
private static String preferHuman(String existing, String canonical) {
|
||||
return (existing == null || existing.isBlank()) ? blankToNull(canonical) : existing;
|
||||
}
|
||||
|
||||
private static Integer preferHuman(Integer existing, Integer canonical) {
|
||||
return existing != null ? existing : canonical;
|
||||
}
|
||||
|
||||
private static String blankToNull(String s) {
|
||||
return (s == null || s.isBlank()) ? null : s.trim();
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Person createPerson(String firstName, String lastName, String alias) {
|
||||
Person person = Person.builder()
|
||||
|
||||
@@ -18,6 +18,7 @@ public interface PersonSummaryDTO {
|
||||
Integer getDeathYear();
|
||||
String getNotes();
|
||||
boolean isFamilyMember();
|
||||
boolean isProvisional();
|
||||
long getDocumentCount();
|
||||
|
||||
default String getDisplayName() {
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package org.raddatz.familienarchiv.person;
|
||||
|
||||
import lombok.Builder;
|
||||
|
||||
/**
|
||||
* Importer → {@link PersonService} command for an idempotent upsert keyed on
|
||||
* {@code sourceRef} (the normalizer's stable person_id). Carries only the canonical
|
||||
* fields the importer owns; the service applies the human-edit-preserve precedence
|
||||
* (see ADR-025): non-blank existing fields are never overwritten, and {@code provisional}
|
||||
* never flips back to true once a human has confirmed a person.
|
||||
*/
|
||||
@Builder
|
||||
public record PersonUpsertCommand(
|
||||
String sourceRef,
|
||||
String firstName,
|
||||
String lastName,
|
||||
String maidenName,
|
||||
String notes,
|
||||
Integer birthYear,
|
||||
Integer deathYear,
|
||||
boolean familyMember,
|
||||
PersonType personType,
|
||||
boolean provisional
|
||||
) {}
|
||||
@@ -30,4 +30,11 @@ public class Tag {
|
||||
|
||||
/** Color token name (e.g. "sage"), only set on root-level tags. Null means no color. */
|
||||
private String color;
|
||||
|
||||
/**
|
||||
* Import identity key, keyed on the canonical tag_path. Null for manually created tags;
|
||||
* unique among non-null values. The importer (Phase 3) uses it for idempotent re-import.
|
||||
*/
|
||||
@Column(name = "source_ref")
|
||||
private String sourceRef;
|
||||
}
|
||||
|
||||
@@ -22,6 +22,9 @@ public interface TagRepository extends JpaRepository<Tag, UUID> {
|
||||
|
||||
Optional<Tag> findByNameIgnoreCase(String name);
|
||||
|
||||
// Lookup by the canonical tag_path, used for idempotent canonical re-import (Phase 3).
|
||||
Optional<Tag> findBySourceRef(String sourceRef);
|
||||
|
||||
List<Tag> findByNameContainingIgnoreCase(String name);
|
||||
|
||||
/**
|
||||
|
||||
@@ -7,6 +7,7 @@ import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
@@ -49,12 +50,37 @@ public class TagService {
|
||||
.orElseThrow(() -> DomainException.notFound(ErrorCode.TAG_NOT_FOUND, "Tag not found: " + id));
|
||||
}
|
||||
|
||||
/** Lookup by the canonical tag_path — used by the canonical importer to attach a document's tag. */
|
||||
public Optional<Tag> findBySourceRef(String sourceRef) {
|
||||
return tagRepository.findBySourceRef(sourceRef);
|
||||
}
|
||||
|
||||
public Tag findOrCreate(String name) {
|
||||
String cleanName = name.trim();
|
||||
return tagRepository.findByNameIgnoreCase(cleanName)
|
||||
.orElseGet(() -> tagRepository.save(Tag.builder().name(cleanName).build()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent upsert keyed on {@code sourceRef} (the canonical tag_path) for the
|
||||
* Phase-3 importer (ADR-025). On first import the canonical name and parent are
|
||||
* written; on re-import a human-renamed tag name is preserved (the source_ref is the
|
||||
* stable identity, the name is a human-editable label).
|
||||
*/
|
||||
@Transactional
|
||||
public Tag upsertBySourceRef(String sourceRef, String name, UUID parentId) {
|
||||
return tagRepository.findBySourceRef(sourceRef)
|
||||
.map(existing -> {
|
||||
existing.setParentId(parentId);
|
||||
return tagRepository.save(existing);
|
||||
})
|
||||
.orElseGet(() -> tagRepository.save(Tag.builder()
|
||||
.sourceRef(sourceRef)
|
||||
.name(name)
|
||||
.parentId(parentId)
|
||||
.build()));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Tag update(UUID id, TagUpdateDTO dto) {
|
||||
Tag tag = getById(id);
|
||||
|
||||
@@ -5,7 +5,8 @@ import org.raddatz.familienarchiv.security.Permission;
|
||||
import org.raddatz.familienarchiv.security.RequirePermission;
|
||||
import org.raddatz.familienarchiv.document.DocumentService;
|
||||
import org.raddatz.familienarchiv.document.DocumentVersionService;
|
||||
import org.raddatz.familienarchiv.importing.MassImportService;
|
||||
import org.raddatz.familienarchiv.importing.CanonicalImportOrchestrator;
|
||||
import org.raddatz.familienarchiv.importing.ImportStatus;
|
||||
import org.raddatz.familienarchiv.document.ThumbnailBackfillService;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
@@ -21,20 +22,20 @@ import lombok.RequiredArgsConstructor;
|
||||
@RequiredArgsConstructor
|
||||
public class AdminController {
|
||||
|
||||
private final MassImportService massImportService;
|
||||
private final CanonicalImportOrchestrator importOrchestrator;
|
||||
private final DocumentService documentService;
|
||||
private final DocumentVersionService documentVersionService;
|
||||
private final ThumbnailBackfillService thumbnailBackfillService;
|
||||
|
||||
@PostMapping("/trigger-import")
|
||||
public ResponseEntity<MassImportService.ImportStatus> triggerMassImport() {
|
||||
massImportService.runImportAsync();
|
||||
return ResponseEntity.accepted().body(massImportService.getStatus());
|
||||
public ResponseEntity<ImportStatus> triggerMassImport() {
|
||||
importOrchestrator.runImportAsync();
|
||||
return ResponseEntity.accepted().body(importOrchestrator.getStatus());
|
||||
}
|
||||
|
||||
@GetMapping("/import-status")
|
||||
public ResponseEntity<MassImportService.ImportStatus> importStatus() {
|
||||
return ResponseEntity.ok(massImportService.getStatus());
|
||||
public ResponseEntity<ImportStatus> importStatus() {
|
||||
return ResponseEntity.ok(importOrchestrator.getStatus());
|
||||
}
|
||||
|
||||
@PostMapping("/backfill-versions")
|
||||
|
||||
@@ -125,17 +125,10 @@ app:
|
||||
password: ${APP_ADMIN_PASSWORD:admin123}
|
||||
|
||||
import:
|
||||
col:
|
||||
index: 0
|
||||
box: 1
|
||||
folder: 2
|
||||
sender: 3
|
||||
receivers: 5
|
||||
date: 7
|
||||
location: 9
|
||||
tags: 10
|
||||
summary: 11
|
||||
transcription: 13
|
||||
# Directory holding the normalizer's committed canonical artifacts
|
||||
# (canonical-{documents,persons,tag-tree}.xlsx + canonical-persons-tree.json).
|
||||
# The loader maps columns by header name — no positional indices (see ADR-025).
|
||||
dir: ${IMPORT_DIR:/import}
|
||||
|
||||
ocr:
|
||||
sender-model:
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
-- Phase 2 of "Handling the Unknowns": the schema foundation.
|
||||
-- Consolidates every new import/precision/attribution/identity column into ONE
|
||||
-- migration with a single owner so downstream phases (importer, rendering, persons
|
||||
-- directory) compile against a finished, collision-free schema. See ADR-025.
|
||||
--
|
||||
-- This file is forward-only and immutable once shipped (Flyway checksum model):
|
||||
-- any fix goes in a later version, never an edit here.
|
||||
|
||||
-- ─── documents: date precision, range end, raw date, raw attribution ──────────
|
||||
|
||||
-- Range end is only set for RANGE precision (open-ended ranges allowed → end may be null).
|
||||
ALTER TABLE documents ADD COLUMN meta_date_end date;
|
||||
|
||||
-- Original date cell, verbatim, for provenance and "as written" display (Phase 4).
|
||||
ALTER TABLE documents ADD COLUMN meta_date_raw text;
|
||||
|
||||
-- Raw attribution preserved even when a person is linked.
|
||||
ALTER TABLE documents ADD COLUMN sender_text text;
|
||||
ALTER TABLE documents ADD COLUMN receiver_text text;
|
||||
|
||||
-- Bound user-influenced spreadsheet text at the DB layer (mirrors transcription_blocks
|
||||
-- length cap in V18). Defense in depth against malformed/huge import cells.
|
||||
ALTER TABLE documents ADD CONSTRAINT chk_meta_date_raw_length CHECK (length(meta_date_raw) <= 10000);
|
||||
ALTER TABLE documents ADD CONSTRAINT chk_sender_text_length CHECK (length(sender_text) <= 10000);
|
||||
ALTER TABLE documents ADD CONSTRAINT chk_receiver_text_length CHECK (length(receiver_text) <= 10000);
|
||||
|
||||
-- Precision enum — added with a DB default of 'UNKNOWN', backfilled, then made NOT NULL.
|
||||
-- The DEFAULT serves two purposes: (1) existing rows get 'UNKNOWN' immediately, and
|
||||
-- (2) raw-SQL inserts that omit the column (test fixtures, ad-hoc data loads) get a sane,
|
||||
-- CHECK-valid value instead of violating the NOT NULL constraint. JPA saves still set it
|
||||
-- explicitly via the entity's @Builder.Default = DatePrecision.UNKNOWN.
|
||||
ALTER TABLE documents ADD COLUMN meta_date_precision varchar(16) DEFAULT 'UNKNOWN';
|
||||
|
||||
UPDATE documents
|
||||
SET meta_date_precision = CASE WHEN meta_date IS NOT NULL THEN 'DAY' ELSE 'UNKNOWN' END;
|
||||
|
||||
ALTER TABLE documents ALTER COLUMN meta_date_precision SET NOT NULL;
|
||||
|
||||
-- Fail-closed allowlist of the seven precision values (verbatim mirror of the
|
||||
-- normalizer's Precision enum). The DB enforces validity independent of the Java enum.
|
||||
ALTER TABLE documents ADD CONSTRAINT chk_meta_date_precision
|
||||
CHECK (meta_date_precision IN ('DAY', 'MONTH', 'SEASON', 'YEAR', 'RANGE', 'APPROX', 'UNKNOWN'));
|
||||
|
||||
-- A non-null range end is permitted only when precision = RANGE. A RANGE row MAY have a
|
||||
-- null end (open-ended range), so the rule is one-directional, not biconditional.
|
||||
ALTER TABLE documents ADD CONSTRAINT chk_meta_date_end_only_for_range
|
||||
CHECK (meta_date_end IS NULL OR meta_date_precision = 'RANGE');
|
||||
|
||||
-- For ranges with both endpoints, the end must not precede the start.
|
||||
ALTER TABLE documents ADD CONSTRAINT chk_meta_date_end_after_start
|
||||
CHECK (meta_date_end IS NULL OR meta_date IS NULL OR meta_date_end >= meta_date);
|
||||
|
||||
-- ─── persons: source_ref (import identity) + provisional flag ─────────────────
|
||||
|
||||
-- The normalizer person_id: join key for documents → persons and idempotency key for
|
||||
-- re-import. Nullable (manually created persons never have one); unique among non-nulls.
|
||||
ALTER TABLE persons ADD COLUMN source_ref varchar(255);
|
||||
CREATE UNIQUE INDEX idx_persons_source_ref ON persons (source_ref);
|
||||
|
||||
-- A provisional person is one the importer inferred but could not confidently identify.
|
||||
-- Stays false until Phase 3 (importer) sets it; no code path writes true in this phase.
|
||||
ALTER TABLE persons ADD COLUMN provisional boolean NOT NULL DEFAULT false;
|
||||
|
||||
-- ─── tag: source_ref (import identity, keyed on canonical tag_path) ───────────
|
||||
|
||||
ALTER TABLE tag ADD COLUMN source_ref varchar(255);
|
||||
CREATE UNIQUE INDEX idx_tag_source_ref ON tag (source_ref);
|
||||
@@ -479,6 +479,191 @@ class MigrationIntegrationTest {
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
// ─── V69: import/precision/attribution/identity schema foundation ────────
|
||||
|
||||
@Test
|
||||
void v69_metaDatePrecisionColumn_isNotNull() {
|
||||
Integer count = jdbc.queryForObject(
|
||||
"""
|
||||
SELECT COUNT(*) FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'documents'
|
||||
AND column_name = 'meta_date_precision'
|
||||
AND is_nullable = 'NO'
|
||||
""",
|
||||
Integer.class);
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_backfillSql_setsDatedRowsToDayPrecision() {
|
||||
// Re-run the migration's backfill UPDATE on a freshly dated row to prove the rule.
|
||||
UUID docId = createDocumentWithDate("1943-05-12");
|
||||
|
||||
jdbc.update(V69_BACKFILL_PRECISION_SQL);
|
||||
|
||||
String precision = jdbc.queryForObject(
|
||||
"SELECT meta_date_precision FROM documents WHERE id = ?", String.class, docId);
|
||||
assertThat(precision).isEqualTo("DAY");
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_backfillSql_setsUndatedRowsToUnknownPrecision() {
|
||||
UUID docId = createDocument(); // no meta_date
|
||||
|
||||
jdbc.update(V69_BACKFILL_PRECISION_SQL);
|
||||
|
||||
String precision = jdbc.queryForObject(
|
||||
"SELECT meta_date_precision FROM documents WHERE id = ?", String.class, docId);
|
||||
assertThat(precision).isEqualTo("UNKNOWN");
|
||||
}
|
||||
|
||||
// Mirrors the backfill UPDATE shipped in V69; idempotent for verification.
|
||||
private static final String V69_BACKFILL_PRECISION_SQL = """
|
||||
UPDATE documents
|
||||
SET meta_date_precision = CASE WHEN meta_date IS NOT NULL THEN 'DAY' ELSE 'UNKNOWN' END
|
||||
""";
|
||||
|
||||
@Test
|
||||
void v69_precisionCheck_rejectsValueOutsideEnum() {
|
||||
UUID docId = createDocument();
|
||||
|
||||
assertThatThrownBy(() ->
|
||||
jdbc.update("UPDATE documents SET meta_date_precision = 'BOGUS' WHERE id = ?", docId)
|
||||
).isInstanceOf(DataIntegrityViolationException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_metaDateEndCheck_rejectsNonNullEndWhenPrecisionNotRange() {
|
||||
UUID docId = createDocumentWithDate("1943-05-12"); // precision DAY
|
||||
|
||||
assertThatThrownBy(() ->
|
||||
jdbc.update("UPDATE documents SET meta_date_end = '1943-06-01' WHERE id = ?", docId)
|
||||
).isInstanceOf(DataIntegrityViolationException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_metaDateEndCheck_allowsNonNullEndWhenPrecisionRange() {
|
||||
UUID docId = createDocumentWithDate("1943-05-12");
|
||||
|
||||
int rows = jdbc.update(
|
||||
"UPDATE documents SET meta_date_precision = 'RANGE', meta_date_end = '1943-06-01' WHERE id = ?",
|
||||
docId);
|
||||
assertThat(rows).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_metaDateEndCheck_allowsRangeWithNullEnd() {
|
||||
// Loose semantics: the normalizer may emit an open-ended RANGE (start only).
|
||||
UUID docId = createDocumentWithDate("1943-05-12");
|
||||
|
||||
int rows = jdbc.update(
|
||||
"UPDATE documents SET meta_date_precision = 'RANGE' WHERE id = ?", docId);
|
||||
assertThat(rows).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_metaDateEndCheck_allowsRangeWithBothEndpointsNull() {
|
||||
// Fully-open RANGE: neither start (meta_date) nor end (meta_date_end) is set.
|
||||
// Both CHECKs hold (end IS NULL passes chk_meta_date_end_only_for_range; both-null
|
||||
// passes chk_meta_date_end_after_start), so the row survives. This locks the actual
|
||||
// DB behavior so a future tightening to a biconditional rule is a deliberate change.
|
||||
UUID docId = createDocument(); // null meta_date
|
||||
|
||||
int rows = jdbc.update(
|
||||
"UPDATE documents SET meta_date_precision = 'RANGE' WHERE id = ?", docId);
|
||||
assertThat(rows).isEqualTo(1);
|
||||
|
||||
Object metaDate = jdbc.queryForObject("SELECT meta_date FROM documents WHERE id = ?", Object.class, docId);
|
||||
Object metaDateEnd = jdbc.queryForObject(
|
||||
"SELECT meta_date_end FROM documents WHERE id = ?", Object.class, docId);
|
||||
assertThat(metaDate).isNull();
|
||||
assertThat(metaDateEnd).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_rangeOrderCheck_rejectsEndBeforeStart() {
|
||||
UUID docId = createDocumentWithDate("1943-05-12");
|
||||
|
||||
assertThatThrownBy(() ->
|
||||
jdbc.update(
|
||||
"UPDATE documents SET meta_date_precision = 'RANGE', meta_date_end = '1943-01-01' WHERE id = ?",
|
||||
docId)
|
||||
).isInstanceOf(DataIntegrityViolationException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_metaDateRawCheck_rejectsOverlongText() {
|
||||
UUID docId = createDocument();
|
||||
String tooLong = "x".repeat(10001);
|
||||
|
||||
assertThatThrownBy(() ->
|
||||
jdbc.update("UPDATE documents SET meta_date_raw = ? WHERE id = ?", tooLong, docId)
|
||||
).isInstanceOf(DataIntegrityViolationException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_senderTextAndReceiverText_storeRawAttribution() {
|
||||
UUID docId = createDocument();
|
||||
|
||||
int rows = jdbc.update(
|
||||
"UPDATE documents SET sender_text = 'Oma Anna', receiver_text = 'Tante Grete' WHERE id = ?",
|
||||
docId);
|
||||
assertThat(rows).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
void v69_personsSourceRef_uniqueIndexRejectsDuplicate() {
|
||||
jdbc.update(
|
||||
"INSERT INTO persons (id, last_name, source_ref) VALUES (gen_random_uuid(), 'A', 'person:dup')");
|
||||
try {
|
||||
assertThatThrownBy(() ->
|
||||
jdbc.update(
|
||||
"INSERT INTO persons (id, last_name, source_ref) VALUES (gen_random_uuid(), 'B', 'person:dup')")
|
||||
).isInstanceOf(DataIntegrityViolationException.class);
|
||||
} finally {
|
||||
jdbc.update("DELETE FROM persons WHERE source_ref = 'person:dup'");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
void v69_personsSourceRef_allowsMultipleNulls() {
|
||||
UUID a = createPerson("Null", "RefA");
|
||||
UUID b = createPerson("Null", "RefB");
|
||||
try {
|
||||
String refA = jdbc.queryForObject("SELECT source_ref FROM persons WHERE id = ?", String.class, a);
|
||||
String refB = jdbc.queryForObject("SELECT source_ref FROM persons WHERE id = ?", String.class, b);
|
||||
assertThat(refA).isNull();
|
||||
assertThat(refB).isNull();
|
||||
} finally {
|
||||
jdbc.update("DELETE FROM persons WHERE id IN (?, ?)", a, b);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void v69_personsProvisional_defaultsToFalse() {
|
||||
UUID id = createPerson("Provisional", "Default");
|
||||
|
||||
Boolean provisional = jdbc.queryForObject(
|
||||
"SELECT provisional FROM persons WHERE id = ?", Boolean.class, id);
|
||||
assertThat(provisional).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@Transactional(propagation = Propagation.NOT_SUPPORTED)
|
||||
void v69_tagSourceRef_uniqueIndexRejectsDuplicate() {
|
||||
jdbc.update("INSERT INTO tag (id, name, source_ref) VALUES (gen_random_uuid(), 'TagDupA', 'tag:dup')");
|
||||
try {
|
||||
assertThatThrownBy(() ->
|
||||
jdbc.update("INSERT INTO tag (id, name, source_ref) VALUES (gen_random_uuid(), 'TagDupB', 'tag:dup')")
|
||||
).isInstanceOf(DataIntegrityViolationException.class);
|
||||
} finally {
|
||||
jdbc.update("DELETE FROM tag WHERE source_ref = 'tag:dup'");
|
||||
}
|
||||
}
|
||||
|
||||
// ─── helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
private UUID createPerson(String firstName, String lastName) {
|
||||
@@ -504,6 +689,12 @@ class MigrationIntegrationTest {
|
||||
return doc.getId();
|
||||
}
|
||||
|
||||
private UUID createDocumentWithDate(String isoDate) {
|
||||
UUID id = createDocument();
|
||||
jdbc.update("UPDATE documents SET meta_date = ?::date WHERE id = ?", isoDate, id);
|
||||
return id;
|
||||
}
|
||||
|
||||
private UUID insertAnnotation(UUID docId) {
|
||||
UUID id = UUID.randomUUID();
|
||||
jdbc.update("""
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package org.raddatz.familienarchiv.document;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.raddatz.familienarchiv.document.DocumentBatchMetadataDTO;
|
||||
import org.raddatz.familienarchiv.document.DocumentSearchResult;
|
||||
import org.raddatz.familienarchiv.document.DocumentVersionSummary;
|
||||
@@ -35,7 +36,9 @@ import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.verify;
|
||||
@@ -73,23 +76,69 @@ class DocumentControllerTest {
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_returns200_whenAuthenticated() throws Exception {
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of()));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search"))
|
||||
.andExpect(status().isOk());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_undatedTrue_isReachableByAuthenticatedUser() throws Exception {
|
||||
// The read GET must stay reachable for READ_ALL users — guards against a
|
||||
// future refactor accidentally write-guarding the undated triage path (#668).
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of()));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search").param("undated", "true"))
|
||||
.andExpect(status().isOk());
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_undatedTrue_returns401_whenUnauthenticated() throws Exception {
|
||||
mockMvc.perform(get("/api/documents/search").param("undated", "true"))
|
||||
.andExpect(status().isUnauthorized());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_undatedTrue_isForwardedToServiceAsTrue() throws Exception {
|
||||
ArgumentCaptor<Boolean> undatedCaptor = ArgumentCaptor.forClass(Boolean.class);
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of()));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search").param("undated", "true"))
|
||||
.andExpect(status().isOk());
|
||||
|
||||
verify(documentService).searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), undatedCaptor.capture(), any());
|
||||
assertThat(undatedCaptor.getValue()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_withoutUndatedParam_forwardsFalseToService() throws Exception {
|
||||
ArgumentCaptor<Boolean> undatedCaptor = ArgumentCaptor.forClass(Boolean.class);
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of()));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search"))
|
||||
.andExpect(status().isOk());
|
||||
|
||||
verify(documentService).searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), undatedCaptor.capture(), any());
|
||||
assertThat(undatedCaptor.getValue()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_withStatusParam_passesItToService() throws Exception {
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), eq(DocumentStatus.REVIEWED), any(), any(), any(), any()))
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), eq(DocumentStatus.REVIEWED), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of()));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search").param("status", "REVIEWED"))
|
||||
.andExpect(status().isOk());
|
||||
|
||||
verify(documentService).searchDocuments(any(), any(), any(), any(), any(), any(), any(), eq(DocumentStatus.REVIEWED), any(), any(), any(), any());
|
||||
verify(documentService).searchDocuments(any(), any(), any(), any(), any(), any(), any(), eq(DocumentStatus.REVIEWED), any(), any(), any(), anyBoolean(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -116,7 +165,7 @@ class DocumentControllerTest {
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_responseContainsTotalCount() throws Exception {
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of()));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search"))
|
||||
@@ -131,9 +180,10 @@ class DocumentControllerTest {
|
||||
UUID docId = UUID.randomUUID();
|
||||
var matchData = new SearchMatchData(
|
||||
"Er schrieb einen langen Brief", List.of(), false, List.of(), List.of(), List.of(), null, List.of());
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of(new DocumentListItem(
|
||||
docId, "Brief an Anna", "brief.pdf", null, null, null,
|
||||
docId, "Brief an Anna", "brief.pdf", null, null,
|
||||
DatePrecision.UNKNOWN, null, null,
|
||||
List.of(), List.of(), null, null, null, null,
|
||||
0, List.of(), matchData,
|
||||
LocalDateTime.of(2026, 1, 15, 10, 0), LocalDateTime.of(2026, 1, 15, 10, 0)))));
|
||||
@@ -150,9 +200,10 @@ class DocumentControllerTest {
|
||||
void search_returns_flat_item_with_id_and_without_sensitive_fields() throws Exception {
|
||||
UUID docId = UUID.randomUUID();
|
||||
var matchData = new SearchMatchData(null, List.of(), false, List.of(), List.of(), List.of(), null, List.of());
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of(new DocumentListItem(
|
||||
docId, "Brief an Anna", "brief.pdf", null, null, null,
|
||||
docId, "Brief an Anna", "brief.pdf", null, null,
|
||||
DatePrecision.UNKNOWN, null, null,
|
||||
List.of(), List.of(), null, null, null, null,
|
||||
0, List.of(), matchData,
|
||||
LocalDateTime.of(2026, 1, 15, 10, 0), LocalDateTime.of(2026, 1, 15, 10, 0)))));
|
||||
@@ -172,7 +223,7 @@ class DocumentControllerTest {
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_responseExposesPagingFields() throws Exception {
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of()));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search"))
|
||||
@@ -217,7 +268,7 @@ class DocumentControllerTest {
|
||||
@Test
|
||||
@WithMockUser
|
||||
void search_passesPageRequestToService() throws Exception {
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any()))
|
||||
.thenReturn(DocumentSearchResult.of(List.of()));
|
||||
|
||||
mockMvc.perform(get("/api/documents/search").param("page", "2").param("size", "25"))
|
||||
@@ -225,7 +276,7 @@ class DocumentControllerTest {
|
||||
|
||||
org.mockito.ArgumentCaptor<org.springframework.data.domain.Pageable> captor =
|
||||
org.mockito.ArgumentCaptor.forClass(org.springframework.data.domain.Pageable.class);
|
||||
verify(documentService).searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), captor.capture());
|
||||
verify(documentService).searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), captor.capture());
|
||||
org.springframework.data.domain.Pageable pageable = captor.getValue();
|
||||
org.assertj.core.api.Assertions.assertThat(pageable.getPageNumber()).isEqualTo(2);
|
||||
org.assertj.core.api.Assertions.assertThat(pageable.getPageSize()).isEqualTo(25);
|
||||
@@ -294,6 +345,34 @@ class DocumentControllerTest {
|
||||
.andExpect(status().isOk());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "WRITE_ALL")
|
||||
void updateDocument_bindsPrecisionFormFields_toDTO() throws Exception {
|
||||
// Pins the wire contract: the edit form's metaDatePrecision / metaDateEnd /
|
||||
// metaDateRaw multipart field names must bind to DocumentUpdateDTO. A rename
|
||||
// on either side silently drops the precision edit; this captures the DTO.
|
||||
UUID id = UUID.randomUUID();
|
||||
Document doc = Document.builder().id(id).title("Brief").originalFilename("brief.pdf").build();
|
||||
when(userService.findByEmail(any())).thenReturn(AppUser.builder().id(UUID.randomUUID()).build());
|
||||
|
||||
org.mockito.ArgumentCaptor<DocumentUpdateDTO> captor =
|
||||
org.mockito.ArgumentCaptor.forClass(DocumentUpdateDTO.class);
|
||||
when(documentService.updateDocument(eq(id), captor.capture(), any(), any())).thenReturn(doc);
|
||||
|
||||
mockMvc.perform(multipart("/api/documents/" + id)
|
||||
.param("metaDatePrecision", "RANGE")
|
||||
.param("metaDateEnd", "1917-01-11")
|
||||
.param("metaDateRaw", "10.–11. Januar 1917")
|
||||
.with(req -> { req.setMethod("PUT"); return req; }).with(csrf()))
|
||||
.andExpect(status().isOk());
|
||||
|
||||
DocumentUpdateDTO bound = captor.getValue();
|
||||
org.assertj.core.api.Assertions.assertThat(bound.getMetaDatePrecision()).isEqualTo(DatePrecision.RANGE);
|
||||
org.assertj.core.api.Assertions.assertThat(bound.getMetaDateEnd())
|
||||
.isEqualTo(java.time.LocalDate.of(1917, 1, 11));
|
||||
org.assertj.core.api.Assertions.assertThat(bound.getMetaDateRaw()).isEqualTo("10.–11. Januar 1917");
|
||||
}
|
||||
|
||||
// ─── DELETE /api/documents/{id} ──────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
@@ -1115,7 +1194,7 @@ class DocumentControllerTest {
|
||||
void getDocumentIds_returns200_andDelegatesToService() throws Exception {
|
||||
when(userService.findByEmail(any())).thenReturn(AppUser.builder().id(UUID.randomUUID()).build());
|
||||
UUID id = UUID.randomUUID();
|
||||
when(documentService.findIdsForFilter(any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
when(documentService.findIdsForFilter(any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean()))
|
||||
.thenReturn(List.of(id));
|
||||
|
||||
mockMvc.perform(get("/api/documents/ids"))
|
||||
@@ -1128,13 +1207,13 @@ class DocumentControllerTest {
|
||||
void getDocumentIds_passesSenderIdParamToService() throws Exception {
|
||||
when(userService.findByEmail(any())).thenReturn(AppUser.builder().id(UUID.randomUUID()).build());
|
||||
UUID senderId = UUID.randomUUID();
|
||||
when(documentService.findIdsForFilter(any(), any(), any(), eq(senderId), any(), any(), any(), any(), any()))
|
||||
when(documentService.findIdsForFilter(any(), any(), any(), eq(senderId), any(), any(), any(), any(), any(), anyBoolean()))
|
||||
.thenReturn(List.of());
|
||||
|
||||
mockMvc.perform(get("/api/documents/ids").param("senderId", senderId.toString()))
|
||||
.andExpect(status().isOk());
|
||||
|
||||
verify(documentService).findIdsForFilter(any(), any(), any(), eq(senderId), any(), any(), any(), any(), any());
|
||||
verify(documentService).findIdsForFilter(any(), any(), any(), eq(senderId), any(), any(), any(), any(), any(), anyBoolean());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -1144,7 +1223,7 @@ class DocumentControllerTest {
|
||||
// Service returns 5001 IDs — one over BULK_EDIT_FILTER_MAX_IDS (5000).
|
||||
java.util.List<UUID> tooMany = new java.util.ArrayList<>(5001);
|
||||
for (int i = 0; i < 5001; i++) tooMany.add(UUID.randomUUID());
|
||||
when(documentService.findIdsForFilter(any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||
when(documentService.findIdsForFilter(any(), any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean()))
|
||||
.thenReturn(tooMany);
|
||||
|
||||
mockMvc.perform(get("/api/documents/ids"))
|
||||
|
||||
@@ -123,8 +123,7 @@ class DocumentLazyLoadingTest {
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.RECEIVER, "asc", null,
|
||||
PageRequest.of(0, 20));
|
||||
DocumentSort.RECEIVER, "asc", null, false, PageRequest.of(0, 20));
|
||||
assertThat(result.totalElements()).isGreaterThan(0);
|
||||
assertThatCode(() ->
|
||||
result.items().forEach(i -> { if (i.sender() != null) i.sender().getLastName(); }))
|
||||
@@ -139,8 +138,7 @@ class DocumentLazyLoadingTest {
|
||||
|
||||
assertThatCode(() -> documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.SENDER, "asc", null,
|
||||
PageRequest.of(0, 20)))
|
||||
DocumentSort.SENDER, "asc", null, false, PageRequest.of(0, 20)))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
|
||||
@@ -56,8 +56,7 @@ class DocumentListItemIntegrationTest {
|
||||
|
||||
assertThatCode(() -> documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(0, 50)))
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(0, 50)))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@@ -72,8 +71,7 @@ class DocumentListItemIntegrationTest {
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(0, 50));
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(0, 50));
|
||||
|
||||
assertThat(result.totalElements()).isGreaterThan(0);
|
||||
DocumentListItem item = result.items().get(0);
|
||||
@@ -81,6 +79,27 @@ class DocumentListItemIntegrationTest {
|
||||
assertThat(item.title()).isEqualTo("Kurrent Brief");
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_listItem_carriesMetaDatePrecisionAndEnd() {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("Range Brief")
|
||||
.originalFilename("range.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.documentDate(java.time.LocalDate.of(1943, 1, 1))
|
||||
.metaDatePrecision(DatePrecision.RANGE)
|
||||
.metaDateEnd(java.time.LocalDate.of(1943, 12, 31))
|
||||
.build());
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(0, 50));
|
||||
|
||||
DocumentListItem item = result.items().stream()
|
||||
.filter(i -> i.title().equals("Range Brief")).findFirst().orElseThrow();
|
||||
assertThat(item.metaDatePrecision()).isEqualTo(DatePrecision.RANGE);
|
||||
assertThat(item.metaDateEnd()).isEqualTo(java.time.LocalDate.of(1943, 12, 31));
|
||||
}
|
||||
|
||||
@Test
|
||||
void detail_stillReturnsTrainingLabels() {
|
||||
Document saved = documentRepository.save(Document.builder()
|
||||
|
||||
@@ -62,8 +62,7 @@ class DocumentSearchPagedIntegrationTest {
|
||||
void search_firstPage_returnsExactlyPageSizeItems_andCorrectTotalElements() {
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(0, 50));
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(0, 50));
|
||||
|
||||
assertThat(result.items()).hasSize(50);
|
||||
assertThat(result.totalElements()).isEqualTo(FIXTURE_SIZE);
|
||||
@@ -76,8 +75,7 @@ class DocumentSearchPagedIntegrationTest {
|
||||
void search_lastPartialPage_returnsRemainingItems() {
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(2, 50));
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(2, 50));
|
||||
|
||||
// Page 2 (offset 100) of 120 docs → exactly 20 items on the tail.
|
||||
assertThat(result.items()).hasSize(20);
|
||||
@@ -89,8 +87,7 @@ class DocumentSearchPagedIntegrationTest {
|
||||
void search_pageBeyondLast_returnsEmptyContent_totalElementsStillCorrect() {
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(99, 50));
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(99, 50));
|
||||
|
||||
assertThat(result.items()).isEmpty();
|
||||
assertThat(result.totalElements()).isEqualTo(FIXTURE_SIZE);
|
||||
@@ -103,8 +100,7 @@ class DocumentSearchPagedIntegrationTest {
|
||||
// returns the correct total from a real repository fetch.
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.SENDER, "asc", null,
|
||||
PageRequest.of(1, 50));
|
||||
DocumentSort.SENDER, "asc", null, false, PageRequest.of(1, 50));
|
||||
|
||||
assertThat(result.items()).hasSize(50);
|
||||
assertThat(result.totalElements()).isEqualTo(FIXTURE_SIZE);
|
||||
@@ -112,16 +108,91 @@ class DocumentSearchPagedIntegrationTest {
|
||||
assertThat(result.totalPages()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_undatedCount_isGlobalFilteredTotal_notPageSlice() {
|
||||
// Seed 70 undated docs on top of the 120 dated ones. With a 50-per-page
|
||||
// window the undated rows span multiple pages, so a page-local count could
|
||||
// never exceed 50 — the global count must be the full 70 (issue #668).
|
||||
int undatedTotal = 70;
|
||||
for (int i = 0; i < undatedTotal; i++) {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("Undatiert-" + String.format("%03d", i))
|
||||
.originalFilename("undatiert-" + i + ".pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.metaDatePrecision(DatePrecision.UNKNOWN)
|
||||
.documentDate(null)
|
||||
.build());
|
||||
}
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(0, 50));
|
||||
|
||||
// Global undated count is the full undated total, independent of page size.
|
||||
assertThat(result.undatedCount()).isEqualTo(undatedTotal);
|
||||
// Total matches both dated + undated (no undated-only filter applied).
|
||||
assertThat(result.totalElements()).isEqualTo(FIXTURE_SIZE + undatedTotal);
|
||||
// The first DATE-DESC page is all dated rows (nulls last), so a page-local
|
||||
// tally would report 0 undated — proving the count is not page-derived.
|
||||
assertThat(result.items()).allMatch(item -> item.documentDate() != null);
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_undatedCount_ignoresUndatedOnlyToggle() {
|
||||
// The "Nur undatierte" toggle must not skew the count: whether undated=true or
|
||||
// false, the global undated count for the same filter is identical (issue #668).
|
||||
int undatedTotal = 12;
|
||||
for (int i = 0; i < undatedTotal; i++) {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("U-" + i)
|
||||
.originalFilename("u-" + i + ".pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.metaDatePrecision(DatePrecision.UNKNOWN)
|
||||
.documentDate(null)
|
||||
.build());
|
||||
}
|
||||
|
||||
DocumentSearchResult unfiltered = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(0, 50));
|
||||
DocumentSearchResult undatedOnly = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null, true, PageRequest.of(0, 50));
|
||||
|
||||
assertThat(unfiltered.undatedCount()).isEqualTo(undatedTotal);
|
||||
assertThat(undatedOnly.undatedCount()).isEqualTo(undatedTotal);
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_undatedCount_isZero_insideDateRange() {
|
||||
// A from/to range excludes undated rows by the collision rule (#668), so the
|
||||
// global undated count inside a range is legitimately 0 even when undated docs exist.
|
||||
for (int i = 0; i < 5; i++) {
|
||||
documentRepository.save(Document.builder()
|
||||
.title("U-range-" + i)
|
||||
.originalFilename("u-range-" + i + ".pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.metaDatePrecision(DatePrecision.UNKNOWN)
|
||||
.documentDate(null)
|
||||
.build());
|
||||
}
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, LocalDate.of(1900, 1, 1), LocalDate.of(2000, 12, 31),
|
||||
null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(0, 50));
|
||||
|
||||
assertThat(result.undatedCount()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_differentPagesReturnDisjointSlices() {
|
||||
DocumentSearchResult page0 = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(0, 50));
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(0, 50));
|
||||
DocumentSearchResult page1 = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null,
|
||||
PageRequest.of(1, 50));
|
||||
DocumentSort.DATE, "DESC", null, false, PageRequest.of(1, 50));
|
||||
|
||||
// No document id should appear on both pages — slicing must be exclusive.
|
||||
var idsOnPage0 = page0.items().stream()
|
||||
|
||||
@@ -15,7 +15,8 @@ class DocumentSearchResultTest {
|
||||
|
||||
private DocumentListItem item(UUID docId) {
|
||||
return new DocumentListItem(
|
||||
docId, "Test", "test.pdf", null, null, null,
|
||||
docId, "Test", "test.pdf", null, null,
|
||||
DatePrecision.UNKNOWN, null, null,
|
||||
List.of(), List.of(), null, null, null, null,
|
||||
0, List.of(), SearchMatchData.empty(),
|
||||
LocalDateTime.of(2026, 1, 15, 10, 0), LocalDateTime.of(2026, 1, 15, 10, 0));
|
||||
@@ -66,7 +67,8 @@ class DocumentSearchResultTest {
|
||||
UUID id = UUID.randomUUID();
|
||||
ActivityActorDTO actor = new ActivityActorDTO("AB", "#f00", "Anna Braun");
|
||||
DocumentListItem item = new DocumentListItem(
|
||||
id, "T", "t.pdf", null, null, null,
|
||||
id, "T", "t.pdf", null, null,
|
||||
DatePrecision.UNKNOWN, null, null,
|
||||
List.of(), List.of(), null, null, null, null,
|
||||
75, List.of(actor), SearchMatchData.empty(),
|
||||
LocalDateTime.of(2026, 1, 15, 10, 0), LocalDateTime.of(2026, 1, 15, 10, 0));
|
||||
@@ -100,4 +102,32 @@ class DocumentSearchResultTest {
|
||||
assertThat(schema.requiredMode()).isEqualTo(Schema.RequiredMode.REQUIRED);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void undatedCount_component_is_annotated_as_required_in_openapi_schema() throws NoSuchFieldException {
|
||||
Schema schema = DocumentSearchResult.class.getDeclaredField("undatedCount").getAnnotation(Schema.class);
|
||||
assertThat(schema).isNotNull();
|
||||
assertThat(schema.requiredMode()).isEqualTo(Schema.RequiredMode.REQUIRED);
|
||||
}
|
||||
|
||||
@Test
|
||||
void factories_default_undatedCount_to_zero() {
|
||||
assertThat(DocumentSearchResult.of(List.of()).undatedCount()).isZero();
|
||||
assertThat(DocumentSearchResult.paged(List.of(), PageRequest.of(0, 50), 0L).undatedCount()).isZero();
|
||||
}
|
||||
|
||||
@Test
|
||||
void withUndatedCount_overlays_count_and_preserves_other_fields() {
|
||||
DocumentSearchResult base = DocumentSearchResult.paged(
|
||||
List.of(item(UUID.randomUUID())), PageRequest.of(1, 50), 120L);
|
||||
|
||||
DocumentSearchResult withCount = base.withUndatedCount(7L);
|
||||
|
||||
assertThat(withCount.undatedCount()).isEqualTo(7L);
|
||||
assertThat(withCount.items()).isEqualTo(base.items());
|
||||
assertThat(withCount.totalElements()).isEqualTo(120L);
|
||||
assertThat(withCount.pageNumber()).isEqualTo(1);
|
||||
assertThat(withCount.pageSize()).isEqualTo(50);
|
||||
assertThat(withCount.totalPages()).isEqualTo(3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,7 +67,7 @@ class DocumentServiceSortTest {
|
||||
.thenReturn(new PageImpl<>(List.of(newer, older)));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.DATE, "DESC", null, PAGE);
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.DATE, "DESC", null, false, PAGE);
|
||||
|
||||
assertThat(result.items()).hasSize(2);
|
||||
assertThat(result.items().get(0).id()).isEqualTo(id2); // newer first
|
||||
@@ -84,7 +84,7 @@ class DocumentServiceSortTest {
|
||||
.thenReturn(List.of(doc(id1)));
|
||||
|
||||
documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, PAGE);
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, false, PAGE);
|
||||
|
||||
verify(documentRepository).findFtsPageRaw(anyString(), anyInt(), anyInt());
|
||||
verify(documentRepository, never()).findAllMatchingIdsByFts(anyString());
|
||||
@@ -102,7 +102,7 @@ class DocumentServiceSortTest {
|
||||
when(documentRepository.findAllById(any())).thenReturn(List.of(doc(id2), doc(id1))); // unordered from JPA
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, PAGE);
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, false, PAGE);
|
||||
|
||||
assertThat(result.items().get(0).id()).isEqualTo(id1);
|
||||
}
|
||||
@@ -119,7 +119,7 @@ class DocumentServiceSortTest {
|
||||
when(documentRepository.findAllById(any())).thenReturn(List.of(doc(id2), doc(id1)));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null, null, null, null, PAGE);
|
||||
"Brief", null, null, null, null, null, null, null, null, null, null, false, PAGE);
|
||||
|
||||
assertThat(result.items().get(0).id()).isEqualTo(id1);
|
||||
}
|
||||
@@ -133,7 +133,7 @@ class DocumentServiceSortTest {
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null,
|
||||
DocumentSort.RELEVANCE, null, null, hugePage);
|
||||
DocumentSort.RELEVANCE, null, null, false, hugePage);
|
||||
|
||||
assertThat(result.items()).isEmpty();
|
||||
verify(documentRepository, never()).findFtsPageRaw(anyString(), anyInt(), anyInt());
|
||||
@@ -153,7 +153,7 @@ class DocumentServiceSortTest {
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null,
|
||||
DocumentSort.RELEVANCE, null, null, PAGE);
|
||||
DocumentSort.RELEVANCE, null, null, false, PAGE);
|
||||
|
||||
assertThat(result.items()).hasSize(1);
|
||||
assertThat(result.items().get(0).id()).isEqualTo(uuidId);
|
||||
@@ -173,7 +173,7 @@ class DocumentServiceSortTest {
|
||||
// sender filter is active → triggers in-memory path, not findFtsPageRaw
|
||||
LocalDate from = LocalDate.of(1900, 1, 1);
|
||||
documentService.searchDocuments(
|
||||
"Brief", from, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, PAGE);
|
||||
"Brief", from, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, false, PAGE);
|
||||
|
||||
verify(documentRepository, never()).findFtsPageRaw(anyString(), anyInt(), anyInt());
|
||||
verify(documentRepository).findAllMatchingIdsByFts("Brief");
|
||||
|
||||
@@ -47,6 +47,8 @@ import java.util.UUID;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.*;
|
||||
@@ -144,6 +146,53 @@ class DocumentServiceTest {
|
||||
assertThat(doc.getArchiveFolder()).isEqualTo("Mappe B");
|
||||
}
|
||||
|
||||
@Test
|
||||
void updateDocument_persistsDatePrecisionEndAndRaw() throws Exception {
|
||||
UUID id = UUID.randomUUID();
|
||||
Document doc = Document.builder().id(id).receivers(new HashSet<>()).tags(new HashSet<>()).build();
|
||||
when(documentRepository.findById(id)).thenReturn(Optional.of(doc));
|
||||
when(documentRepository.save(any())).thenReturn(doc);
|
||||
|
||||
DocumentUpdateDTO dto = new DocumentUpdateDTO();
|
||||
dto.setDocumentDate(LocalDate.of(1917, 1, 10));
|
||||
dto.setMetaDatePrecision(DatePrecision.RANGE);
|
||||
dto.setMetaDateEnd(LocalDate.of(1917, 1, 11));
|
||||
dto.setMetaDateRaw("10.–11. Januar 1917");
|
||||
|
||||
documentService.updateDocument(id, dto, null, null);
|
||||
|
||||
assertThat(doc.getMetaDatePrecision()).isEqualTo(DatePrecision.RANGE);
|
||||
assertThat(doc.getMetaDateEnd()).isEqualTo(LocalDate.of(1917, 1, 11));
|
||||
assertThat(doc.getMetaDateRaw()).isEqualTo("10.–11. Januar 1917");
|
||||
}
|
||||
|
||||
@Test
|
||||
void updateDocument_preservesStoredPrecision_whenDtoOmitsIt() throws Exception {
|
||||
// Editing a doc (e.g. fixing a location typo) without touching the precision
|
||||
// controls must NOT fabricate a precision. The form omits the three precision
|
||||
// fields → they arrive null on the DTO → the stored values must be preserved.
|
||||
UUID id = UUID.randomUUID();
|
||||
Document doc = Document.builder()
|
||||
.id(id)
|
||||
.metaDatePrecision(DatePrecision.MONTH)
|
||||
.metaDateEnd(LocalDate.of(1916, 6, 30))
|
||||
.metaDateRaw("Juni 1916")
|
||||
.receivers(new HashSet<>())
|
||||
.tags(new HashSet<>())
|
||||
.build();
|
||||
when(documentRepository.findById(id)).thenReturn(Optional.of(doc));
|
||||
when(documentRepository.save(any())).thenReturn(doc);
|
||||
|
||||
DocumentUpdateDTO dto = new DocumentUpdateDTO();
|
||||
dto.setLocation("Berlin"); // unrelated edit; precision fields left null
|
||||
|
||||
documentService.updateDocument(id, dto, null, null);
|
||||
|
||||
assertThat(doc.getMetaDatePrecision()).isEqualTo(DatePrecision.MONTH);
|
||||
assertThat(doc.getMetaDateEnd()).isEqualTo(LocalDate.of(1916, 6, 30));
|
||||
assertThat(doc.getMetaDateRaw()).isEqualTo("Juni 1916");
|
||||
}
|
||||
|
||||
// ─── deleteTagCascading ───────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
@@ -1362,8 +1411,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(new PageImpl<>(List.of()));
|
||||
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, null,
|
||||
org.raddatz.familienarchiv.document.DocumentSort.DATE, "DESC", null,
|
||||
org.springframework.data.domain.PageRequest.of(1, 50));
|
||||
org.raddatz.familienarchiv.document.DocumentSort.DATE, "DESC", null, false, org.springframework.data.domain.PageRequest.of(1, 50));
|
||||
|
||||
verify(documentRepository).findAll(any(org.springframework.data.jpa.domain.Specification.class), any(Pageable.class));
|
||||
verify(documentRepository, never()).findAll(any(org.springframework.data.jpa.domain.Specification.class), any(Sort.class));
|
||||
@@ -1376,8 +1424,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(new PageImpl<>(List.of()));
|
||||
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, null,
|
||||
org.raddatz.familienarchiv.document.DocumentSort.DATE, "DESC", null,
|
||||
org.springframework.data.domain.PageRequest.of(3, 25));
|
||||
org.raddatz.familienarchiv.document.DocumentSort.DATE, "DESC", null, false, org.springframework.data.domain.PageRequest.of(3, 25));
|
||||
|
||||
verify(documentRepository).findAll(any(org.springframework.data.jpa.domain.Specification.class), captor.capture());
|
||||
assertThat(captor.getValue().getPageNumber()).isEqualTo(3);
|
||||
@@ -1393,8 +1440,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(new PageImpl<>(List.of(d), org.springframework.data.domain.PageRequest.of(0, 50), 120L));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(null, null, null, null, null, null, null, null,
|
||||
org.raddatz.familienarchiv.document.DocumentSort.DATE, "DESC", null,
|
||||
org.springframework.data.domain.PageRequest.of(0, 50));
|
||||
org.raddatz.familienarchiv.document.DocumentSort.DATE, "DESC", null, false, org.springframework.data.domain.PageRequest.of(0, 50));
|
||||
|
||||
assertThat(result.totalElements()).isEqualTo(120L);
|
||||
assertThat(result.pageNumber()).isZero();
|
||||
@@ -1403,6 +1449,50 @@ class DocumentServiceTest {
|
||||
assertThat(result.items()).hasSize(1); // only the slice is enriched
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_dateSort_DESC_ordersUndatedLast() {
|
||||
ArgumentCaptor<Pageable> captor = ArgumentCaptor.forClass(Pageable.class);
|
||||
when(documentRepository.findAll(any(org.springframework.data.jpa.domain.Specification.class), any(Pageable.class)))
|
||||
.thenReturn(new PageImpl<>(List.of()));
|
||||
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "DESC", null, false, org.springframework.data.domain.PageRequest.of(0, 5));
|
||||
|
||||
verify(documentRepository).findAll(any(org.springframework.data.jpa.domain.Specification.class), captor.capture());
|
||||
Sort.Order dateOrder = captor.getValue().getSort().getOrderFor("documentDate");
|
||||
assertThat(dateOrder).isNotNull();
|
||||
assertThat(dateOrder.getDirection()).isEqualTo(Sort.Direction.DESC);
|
||||
assertThat(dateOrder.getNullHandling()).isEqualTo(Sort.NullHandling.NULLS_LAST);
|
||||
// Owner-decided tiebreaker (#668): title ASC, not createdAt.
|
||||
Sort.Order tiebreak = captor.getValue().getSort().getOrderFor("title");
|
||||
assertThat(tiebreak).isNotNull();
|
||||
assertThat(tiebreak.getDirection()).isEqualTo(Sort.Direction.ASC);
|
||||
assertThat(captor.getValue().getSort().getOrderFor("createdAt")).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_dateSort_ASC_ordersUndatedLast() {
|
||||
// The ASC bug: Postgres puts NULLs FIRST on ascending sort without explicit
|
||||
// NULLS LAST, surfacing undated documents at the top. This is the red.
|
||||
ArgumentCaptor<Pageable> captor = ArgumentCaptor.forClass(Pageable.class);
|
||||
when(documentRepository.findAll(any(org.springframework.data.jpa.domain.Specification.class), any(Pageable.class)))
|
||||
.thenReturn(new PageImpl<>(List.of()));
|
||||
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, null,
|
||||
DocumentSort.DATE, "ASC", null, false, org.springframework.data.domain.PageRequest.of(0, 5));
|
||||
|
||||
verify(documentRepository).findAll(any(org.springframework.data.jpa.domain.Specification.class), captor.capture());
|
||||
Sort.Order dateOrder = captor.getValue().getSort().getOrderFor("documentDate");
|
||||
assertThat(dateOrder).isNotNull();
|
||||
assertThat(dateOrder.getDirection()).isEqualTo(Sort.Direction.ASC);
|
||||
assertThat(dateOrder.getNullHandling()).isEqualTo(Sort.NullHandling.NULLS_LAST);
|
||||
// Owner-decided tiebreaker (#668): title ASC, not createdAt.
|
||||
Sort.Order tiebreak = captor.getValue().getSort().getOrderFor("title");
|
||||
assertThat(tiebreak).isNotNull();
|
||||
assertThat(tiebreak.getDirection()).isEqualTo(Sort.Direction.ASC);
|
||||
assertThat(captor.getValue().getSort().getOrderFor("createdAt")).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_UPDATED_AT_sort_resolves_to_updatedAt_field() {
|
||||
ArgumentCaptor<Pageable> captor = ArgumentCaptor.forClass(Pageable.class);
|
||||
@@ -1410,8 +1500,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(new PageImpl<>(List.of()));
|
||||
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, null,
|
||||
DocumentSort.UPDATED_AT, "DESC", null,
|
||||
org.springframework.data.domain.PageRequest.of(0, 5));
|
||||
DocumentSort.UPDATED_AT, "DESC", null, false, org.springframework.data.domain.PageRequest.of(0, 5));
|
||||
|
||||
verify(documentRepository).findAll(any(org.springframework.data.jpa.domain.Specification.class), captor.capture());
|
||||
assertThat(captor.getValue().getSort())
|
||||
@@ -1435,8 +1524,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(all);
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(null, null, null, null, null, null, null, null,
|
||||
org.raddatz.familienarchiv.document.DocumentSort.SENDER, "asc", null,
|
||||
org.springframework.data.domain.PageRequest.of(1, 50));
|
||||
org.raddatz.familienarchiv.document.DocumentSort.SENDER, "asc", null, false, org.springframework.data.domain.PageRequest.of(1, 50));
|
||||
|
||||
assertThat(result.totalElements()).isEqualTo(120L);
|
||||
assertThat(result.pageNumber()).isEqualTo(1);
|
||||
@@ -1460,8 +1548,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(all);
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(null, null, null, null, null, null, null, null,
|
||||
org.raddatz.familienarchiv.document.DocumentSort.SENDER, "asc", null,
|
||||
org.springframework.data.domain.PageRequest.of(10, 50));
|
||||
org.raddatz.familienarchiv.document.DocumentSort.SENDER, "asc", null, false, org.springframework.data.domain.PageRequest.of(10, 50));
|
||||
|
||||
assertThat(result.items()).isEmpty();
|
||||
assertThat(result.totalElements()).isEqualTo(30L);
|
||||
@@ -1474,7 +1561,7 @@ class DocumentServiceTest {
|
||||
when(documentRepository.findAll(any(org.springframework.data.jpa.domain.Specification.class), any(Pageable.class)))
|
||||
.thenReturn(new PageImpl<>(List.of()));
|
||||
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, DocumentStatus.REVIEWED, null, null, null, UNPAGED);
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, DocumentStatus.REVIEWED, null, null, null, false, UNPAGED);
|
||||
|
||||
verify(documentRepository).findAll(any(org.springframework.data.jpa.domain.Specification.class), any(Pageable.class));
|
||||
}
|
||||
@@ -1484,7 +1571,7 @@ class DocumentServiceTest {
|
||||
when(documentRepository.findAll(any(org.springframework.data.jpa.domain.Specification.class), any(Pageable.class)))
|
||||
.thenReturn(new PageImpl<>(List.of()));
|
||||
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, null, null, null, null, UNPAGED);
|
||||
documentService.searchDocuments(null, null, null, null, null, null, null, null, null, null, null, false, UNPAGED);
|
||||
|
||||
verify(documentRepository).findAll(any(org.springframework.data.jpa.domain.Specification.class), any(Pageable.class));
|
||||
}
|
||||
@@ -1562,7 +1649,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(List.of(withSender, noSender));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, UNPAGED);
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, false, UNPAGED);
|
||||
|
||||
assertThat(result.items()).hasSize(2);
|
||||
assertThat(result.items()).extracting(DocumentListItem::title).containsExactly("Has Sender", "No Sender");
|
||||
@@ -1582,12 +1669,117 @@ class DocumentServiceTest {
|
||||
.thenReturn(List.of(noReceivers, withReceiver));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null, DocumentSort.RECEIVER, "asc", null, UNPAGED);
|
||||
null, null, null, null, null, null, null, null, DocumentSort.RECEIVER, "asc", null, false, UNPAGED);
|
||||
|
||||
assertThat(result.items()).extracting(DocumentListItem::title)
|
||||
.containsExactly("Has Receiver", "No Receivers");
|
||||
}
|
||||
|
||||
// ─── searchDocuments — undated docs stay in their person group (#668) ───────
|
||||
|
||||
@Test
|
||||
void searchDocuments_senderSort_asc_keepsUndatedInsideSenderGroupNotAtHead() {
|
||||
// Locking test (#668): the in-memory SENDER comparator orders by sender name,
|
||||
// not by date, so an undated (null documentDate) letter must stay WITHIN its
|
||||
// sender's group — it must NOT float to the head of a multi-sender page.
|
||||
// Two senders, each with a dated + an undated doc. ASC by "lastName firstName":
|
||||
// "Adler Bob" < "Ziegler Anna", so both of Bob's docs come before both of Anna's.
|
||||
// The undated doc supplied FIRST in the input proves grouping (not date) wins:
|
||||
// were it ordered by date, the two undated docs would clump together at one end.
|
||||
Person bobAdler = Person.builder().id(UUID.randomUUID()).firstName("Bob").lastName("Adler").build();
|
||||
Person annaZiegler = Person.builder().id(UUID.randomUUID()).firstName("Anna").lastName("Ziegler").build();
|
||||
Document undatedBob = Document.builder().id(UUID.randomUUID()).title("Bob undated")
|
||||
.sender(bobAdler).documentDate(null).build();
|
||||
Document datedBob = Document.builder().id(UUID.randomUUID()).title("Bob dated")
|
||||
.sender(bobAdler).documentDate(LocalDate.of(1916, 6, 15)).build();
|
||||
Document undatedAnna = Document.builder().id(UUID.randomUUID()).title("Anna undated")
|
||||
.sender(annaZiegler).documentDate(null).build();
|
||||
Document datedAnna = Document.builder().id(UUID.randomUUID()).title("Anna dated")
|
||||
.sender(annaZiegler).documentDate(LocalDate.of(1943, 12, 24)).build();
|
||||
|
||||
// Input order interleaves dated/undated so a date-based regression would reorder.
|
||||
when(documentRepository.findAll(any(org.springframework.data.jpa.domain.Specification.class)))
|
||||
.thenReturn(List.of(undatedBob, datedAnna, datedBob, undatedAnna));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, false, UNPAGED);
|
||||
|
||||
// Bob's group precedes Anna's group (ASC by sender). The sort is stable, so
|
||||
// within each group the input order is preserved (undatedBob, datedBob for Bob;
|
||||
// datedAnna, undatedAnna for Anna). The undated docs never jump to the head and
|
||||
// each stays inside its sender group — a date-based comparator would instead
|
||||
// clump the two undated docs together at one end.
|
||||
assertThat(result.items()).extracting(DocumentListItem::title)
|
||||
.containsExactly("Bob undated", "Bob dated", "Anna dated", "Anna undated");
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_senderSort_desc_keepsUndatedInsideSenderGroupNotAtHead() {
|
||||
// DESC symmetry for the in-memory path: sender order reverses ("Ziegler Anna"
|
||||
// before "Adler Bob"), but the undated doc still sorts by sender, never by date,
|
||||
// so it stays within its group and does not surface at the page head.
|
||||
Person bobAdler = Person.builder().id(UUID.randomUUID()).firstName("Bob").lastName("Adler").build();
|
||||
Person annaZiegler = Person.builder().id(UUID.randomUUID()).firstName("Anna").lastName("Ziegler").build();
|
||||
Document undatedBob = Document.builder().id(UUID.randomUUID()).title("Bob undated")
|
||||
.sender(bobAdler).documentDate(null).build();
|
||||
Document datedBob = Document.builder().id(UUID.randomUUID()).title("Bob dated")
|
||||
.sender(bobAdler).documentDate(LocalDate.of(1916, 6, 15)).build();
|
||||
Document undatedAnna = Document.builder().id(UUID.randomUUID()).title("Anna undated")
|
||||
.sender(annaZiegler).documentDate(null).build();
|
||||
Document datedAnna = Document.builder().id(UUID.randomUUID()).title("Anna dated")
|
||||
.sender(annaZiegler).documentDate(LocalDate.of(1943, 12, 24)).build();
|
||||
|
||||
when(documentRepository.findAll(any(org.springframework.data.jpa.domain.Specification.class)))
|
||||
.thenReturn(List.of(undatedBob, datedAnna, datedBob, undatedAnna));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "desc", null, false, UNPAGED);
|
||||
|
||||
// Anna's group precedes Bob's (DESC by sender); undated stays inside its group.
|
||||
assertThat(result.items()).extracting(DocumentListItem::title)
|
||||
.containsExactly("Anna dated", "Anna undated", "Bob undated", "Bob dated");
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_undatedTrue_withSenderSort_appliesUndatedSpecification() {
|
||||
// Reachable UI state: "Nur undatierte" toggled on while grouped by sender.
|
||||
// The SENDER sort takes the in-memory path, but the undatedOnly predicate must
|
||||
// still be composed into the Specification handed to the repository — proven by
|
||||
// capturing the spec passed to findAll and confirming it filters to null dates.
|
||||
Person alice = Person.builder().id(UUID.randomUUID()).firstName("Alice").lastName("Ziegler").build();
|
||||
Document undatedFromAlice = Document.builder().id(UUID.randomUUID()).title("Undated")
|
||||
.sender(alice).documentDate(null).build();
|
||||
|
||||
org.mockito.ArgumentCaptor<org.springframework.data.jpa.domain.Specification<Document>> specCaptor =
|
||||
org.mockito.ArgumentCaptor.forClass(org.springframework.data.jpa.domain.Specification.class);
|
||||
when(documentRepository.findAll(specCaptor.capture()))
|
||||
.thenReturn(List.of(undatedFromAlice));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, true, UNPAGED);
|
||||
|
||||
// The in-memory path queried via a Specification (built by buildSearchSpec with
|
||||
// undatedOnly(true)) rather than skipping straight to a sorted findAll.
|
||||
assertThat(specCaptor.getValue()).isNotNull();
|
||||
assertThat(result.items()).extracting(DocumentListItem::title).containsExactly("Undated");
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_undatedTrue_usesSpecificationPath_notPureTextRelevanceShortcut() {
|
||||
// undated=true must bypass the pure-text RELEVANCE SQL shortcut, which
|
||||
// skips buildSearchSpec and would silently drop the undatedOnly predicate.
|
||||
when(documentRepository.findAllMatchingIdsByFts("brief")).thenReturn(List.of(UUID.randomUUID()));
|
||||
when(documentRepository.findAll(any(org.springframework.data.jpa.domain.Specification.class)))
|
||||
.thenReturn(List.of());
|
||||
|
||||
documentService.searchDocuments("brief", null, null, null, null, null, null, null,
|
||||
DocumentSort.RELEVANCE, null, null, true, UNPAGED);
|
||||
|
||||
// The FTS-id path (buildSearchSpec) ran; the raw-page SQL shortcut did not.
|
||||
verify(documentRepository).findAllMatchingIdsByFts("brief");
|
||||
verify(documentRepository, never()).findFtsPageRaw(anyString(), anyInt(), anyInt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchDocuments_senderSort_nullLastNameSortsToEnd() {
|
||||
// Without fix: null lastName produces sort key "null Smith" which compares
|
||||
@@ -1604,7 +1796,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(List.of(docNullName, docSmith));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, UNPAGED);
|
||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, false, UNPAGED);
|
||||
|
||||
// null lastName should sort to end (treated as empty), not before "smith" (as "null")
|
||||
assertThat(result.items()).extracting(DocumentListItem::title)
|
||||
@@ -1627,7 +1819,7 @@ class DocumentServiceTest {
|
||||
when(documentRepository.findEnrichmentData(any(), eq("Brief"))).thenReturn(rows);
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, UNPAGED);
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, false, UNPAGED);
|
||||
|
||||
assertThat(result.items()).hasSize(1);
|
||||
SearchMatchData md = result.items().get(0).matchData();
|
||||
@@ -1641,8 +1833,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(new PageImpl<>(List.of()));
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
null, null, null, null, null, null, null, null, null, null, null,
|
||||
UNPAGED);
|
||||
null, null, null, null, null, null, null, null, null, null, null, false, UNPAGED);
|
||||
|
||||
assertThat(result.items()).isEmpty();
|
||||
}
|
||||
@@ -1662,7 +1853,7 @@ class DocumentServiceTest {
|
||||
when(documentRepository.findEnrichmentData(any(), eq("Brief"))).thenReturn(rows);
|
||||
|
||||
DocumentSearchResult result = documentService.searchDocuments(
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, UNPAGED);
|
||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, false, UNPAGED);
|
||||
|
||||
SearchMatchData md = result.items().get(0).matchData();
|
||||
assertThat(md.transcriptionSnippet()).isEqualTo("Hier ist der Brief aus Berlin");
|
||||
@@ -2179,7 +2370,7 @@ class DocumentServiceTest {
|
||||
.thenReturn(List.of(d1, d2));
|
||||
|
||||
List<UUID> result = documentService.findIdsForFilter(
|
||||
null, null, null, null, null, null, null, null, null);
|
||||
null, null, null, null, null, null, null, null, null, false);
|
||||
|
||||
assertThat(result).containsExactly(d1.getId(), d2.getId());
|
||||
}
|
||||
@@ -2194,7 +2385,7 @@ class DocumentServiceTest {
|
||||
when(tagService.expandTagNamesToDescendantIdSets(any())).thenReturn(List.of());
|
||||
|
||||
documentService.findIdsForFilter(
|
||||
null, null, null, null, null, List.of("Brief"), null, null, TagOperator.OR);
|
||||
null, null, null, null, null, List.of("Brief"), null, null, TagOperator.OR, false);
|
||||
|
||||
// Spec built without throwing → OR branch was exercised. Coverage gain
|
||||
// is in not-throwing on the OR-specific code path; the actual SQL is
|
||||
@@ -2207,7 +2398,7 @@ class DocumentServiceTest {
|
||||
when(documentRepository.findAllMatchingIdsByFts("xyz")).thenReturn(List.of());
|
||||
|
||||
List<UUID> result = documentService.findIdsForFilter(
|
||||
"xyz", null, null, null, null, null, null, null, null);
|
||||
"xyz", null, null, null, null, null, null, null, null, false);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
verify(documentRepository, never()).findAll(any(org.springframework.data.jpa.domain.Specification.class));
|
||||
|
||||
@@ -261,4 +261,21 @@ class DocumentSpecificationsTest {
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
// ─── undatedOnly ──────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void undatedOnly_false_returnsAllDocuments() {
|
||||
// false → no predicate (null), so the filter is a no-op (issue #668).
|
||||
List<Document> result = documentRepository.findAll(Specification.where(undatedOnly(false)));
|
||||
assertThat(result).hasSize(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void undatedOnly_true_returnsOnlyDocumentsWithoutADate() {
|
||||
// Only the placeholder photo has a null documentDate in the fixture.
|
||||
List<Document> result = documentRepository.findAll(Specification.where(undatedOnly(true)));
|
||||
assertThat(result).extracting(Document::getTitle).containsExactly("Familienfoto");
|
||||
assertThat(result).allMatch(d -> d.getDocumentDate() == null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package org.raddatz.familienarchiv.document;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||
import org.raddatz.familienarchiv.config.FlywayConfig;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
|
||||
import org.springframework.boot.jdbc.test.autoconfigure.AutoConfigureTestDatabase;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.data.domain.Sort;
|
||||
import org.springframework.data.jpa.domain.Specification;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.raddatz.familienarchiv.document.DocumentSpecifications.isBetween;
|
||||
import static org.raddatz.familienarchiv.document.DocumentSpecifications.undatedOnly;
|
||||
|
||||
/**
|
||||
* Real-Postgres assertions for issue #668. H2 disagrees with Postgres on
|
||||
* {@code NULLS FIRST/LAST} defaults and on whether {@code BETWEEN} excludes
|
||||
* NULL, so these guarantees MUST run against {@code postgres:16-alpine}, never
|
||||
* an in-memory database.
|
||||
*/
|
||||
@DataJpaTest
|
||||
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
|
||||
@Import({PostgresContainerConfig.class, FlywayConfig.class})
|
||||
class UndatedDocumentOrderingIntegrationTest {
|
||||
|
||||
@Autowired DocumentRepository documentRepository;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
documentRepository.deleteAll();
|
||||
save("1916", LocalDate.of(1916, 6, 15));
|
||||
save("1943", LocalDate.of(1943, 12, 24));
|
||||
save("undated-a", null);
|
||||
save("undated-b", null);
|
||||
}
|
||||
|
||||
private void save(String title, LocalDate date) {
|
||||
documentRepository.save(Document.builder()
|
||||
.title(title)
|
||||
.originalFilename(title + ".pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.metaDatePrecision(date == null ? DatePrecision.UNKNOWN : DatePrecision.DAY)
|
||||
.documentDate(date)
|
||||
.build());
|
||||
}
|
||||
|
||||
@Test
|
||||
void dateAscWithNullsLast_returnsDatedFirstUndatedLast() {
|
||||
Sort sort = Sort.by(new Sort.Order(Sort.Direction.ASC, "documentDate").nullsLast());
|
||||
|
||||
List<Document> result = documentRepository.findAll(sort);
|
||||
|
||||
assertThat(result).hasSize(4);
|
||||
assertThat(result.get(0).getDocumentDate()).isEqualTo(LocalDate.of(1916, 6, 15));
|
||||
assertThat(result.get(1).getDocumentDate()).isEqualTo(LocalDate.of(1943, 12, 24));
|
||||
assertThat(result.get(2).getDocumentDate()).isNull();
|
||||
assertThat(result.get(3).getDocumentDate()).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void sameDate_tiebreaksByTitleAsc_notCreatedAt_forBothDirections() throws Exception {
|
||||
// Owner decision (#668): equal-date rows tie-break by title ASC, NOT
|
||||
// createdAt. Insert two same-date docs so that createdAt order (insertion
|
||||
// order) is the OPPOSITE of title order: the first-saved doc gets the later
|
||||
// title ("zzz-first"), the second-saved doc gets the earlier title
|
||||
// ("aaa-second"). If the tiebreaker were still createdAt-asc the first-saved
|
||||
// row would lead; because it is title-asc the "aaa-second" row must lead —
|
||||
// and it must lead in BOTH ASC and DESC date directions, since the date is
|
||||
// equal so only the title tiebreaker decides.
|
||||
//
|
||||
// The Sort under test is built by the PRODUCTION resolveSort(DATE, dir) (via
|
||||
// reflection — it is private), not hand-rolled here, so this test proves the
|
||||
// real Postgres ordering that production emits, on real same-date rows.
|
||||
documentRepository.deleteAll();
|
||||
LocalDate sameDate = LocalDate.of(1920, 3, 3);
|
||||
save("zzz-first", sameDate); // saved first → earlier createdAt
|
||||
save("aaa-second", sameDate); // saved second → later createdAt
|
||||
|
||||
List<Document> asc = documentRepository.findAll(resolveProductionSort("ASC"));
|
||||
assertThat(asc).extracting(Document::getTitle)
|
||||
.containsExactly("aaa-second", "zzz-first");
|
||||
|
||||
List<Document> desc = documentRepository.findAll(resolveProductionSort("DESC"));
|
||||
assertThat(desc).extracting(Document::getTitle)
|
||||
.containsExactly("aaa-second", "zzz-first");
|
||||
}
|
||||
|
||||
/**
|
||||
* Invokes the production {@link DocumentService#resolveSort(DocumentSort, String)}
|
||||
* for the DATE sort so the integration assertions exercise the real tiebreaker
|
||||
* choice rather than a sort hand-built in the test.
|
||||
*/
|
||||
private Sort resolveProductionSort(String dir) throws Exception {
|
||||
// resolveSort is a pure function of its arguments (uses no instance state), so a
|
||||
// bean instance with null collaborators is sufficient to exercise it.
|
||||
var ctor = DocumentService.class.getDeclaredConstructors()[0];
|
||||
ctor.setAccessible(true);
|
||||
Object[] args = new Object[ctor.getParameterCount()];
|
||||
DocumentService service = (DocumentService) ctor.newInstance(args);
|
||||
var m = DocumentService.class.getDeclaredMethod("resolveSort", DocumentSort.class, String.class);
|
||||
m.setAccessible(true);
|
||||
return (Sort) m.invoke(service, DocumentSort.DATE, dir);
|
||||
}
|
||||
|
||||
@Test
|
||||
void undatedOnly_returnsExactlyTheNullDatedRows() {
|
||||
List<Document> result = documentRepository.findAll(undatedOnly(true));
|
||||
|
||||
assertThat(result).hasSize(2);
|
||||
assertThat(result).allMatch(d -> d.getDocumentDate() == null);
|
||||
}
|
||||
|
||||
@Test
|
||||
void undatedOnly_false_returnsAllRows() {
|
||||
Specification<Document> spec = Specification.where(undatedOnly(false));
|
||||
|
||||
List<Document> result = documentRepository.findAll(spec);
|
||||
|
||||
assertThat(result).hasSize(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
void dateRange_excludesUndatedRows() {
|
||||
List<Document> result = documentRepository.findAll(isBetween(
|
||||
LocalDate.of(1900, 1, 1), LocalDate.of(2000, 12, 31)));
|
||||
|
||||
assertThat(result).hasSize(2);
|
||||
assertThat(result).allMatch(d -> d.getDocumentDate() != null);
|
||||
}
|
||||
|
||||
@Test
|
||||
void undatedOnly_combinedWithDateRange_returnsEmpty() {
|
||||
// The collision rule (#668): a from/to range and undated=true are mutually
|
||||
// exclusive — a row cannot both have a null date and fall inside a range.
|
||||
Specification<Document> spec = Specification
|
||||
.where(undatedOnly(true))
|
||||
.and(isBetween(LocalDate.of(1900, 1, 1), LocalDate.of(2000, 12, 31)));
|
||||
|
||||
List<Document> result = documentRepository.findAll(spec);
|
||||
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
import org.raddatz.familienarchiv.document.DocumentRepository;
|
||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonRepository;
|
||||
import org.raddatz.familienarchiv.tag.TagRepository;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Real Postgres (Testcontainers) integration test for the canonical importer. The
|
||||
* {@code UNIQUE(source_ref)} constraint and the upsert-on-conflict behaviour only exist
|
||||
* in real Postgres (never H2), so idempotency is verified here. S3 is mocked — the
|
||||
* synthetic document rows carry no on-disk files, so every document is a PLACEHOLDER and
|
||||
* no upload is attempted.
|
||||
*/
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE)
|
||||
@ActiveProfiles("test")
|
||||
@Import(PostgresContainerConfig.class)
|
||||
class CanonicalImportIntegrationTest {
|
||||
|
||||
@MockitoBean S3Client s3Client;
|
||||
|
||||
@Autowired CanonicalImportOrchestrator orchestrator;
|
||||
@Autowired PersonRepository personRepository;
|
||||
@Autowired TagRepository tagRepository;
|
||||
@Autowired DocumentRepository documentRepository;
|
||||
|
||||
Path artifactDir;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
documentRepository.deleteAll();
|
||||
personRepository.deleteAll();
|
||||
tagRepository.deleteAll();
|
||||
artifactDir = Files.createTempDirectory("canonical-import-it");
|
||||
writeArtifacts(artifactDir);
|
||||
ReflectionTestUtils.setField(orchestrator, "canonicalDir", artifactDir.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
* The import commits through its own transactions (the orchestrator is not transactional),
|
||||
* so this test cannot rely on {@code @Transactional} rollback for isolation. Delete the
|
||||
* committed rows after each test — otherwise the last test's documents (dated 1888-02) and
|
||||
* persons/tags leak into the shared Testcontainers Postgres and pollute other integration
|
||||
* tests that assume a known seed (e.g. DocumentDensityIntegrationTest,
|
||||
* DocumentSearchPagedIntegrationTest). Mirrors the @AfterEach deleteAll convention used by
|
||||
* DocumentListItemIntegrationTest.
|
||||
*/
|
||||
@AfterEach
|
||||
void cleanup() {
|
||||
documentRepository.deleteAll();
|
||||
personRepository.deleteAll();
|
||||
tagRepository.deleteAll();
|
||||
}
|
||||
|
||||
@Test
|
||||
void reimport_isIdempotent_noDuplicatePersonsTagsOrDocuments() {
|
||||
orchestrator.runImport();
|
||||
long personsAfterFirst = personRepository.count();
|
||||
long tagsAfterFirst = tagRepository.count();
|
||||
long documentsAfterFirst = documentRepository.count();
|
||||
assertThat(orchestrator.getStatus().state()).isEqualTo(ImportStatus.State.DONE);
|
||||
assertThat(personsAfterFirst).isPositive();
|
||||
assertThat(tagsAfterFirst).isPositive();
|
||||
assertThat(documentsAfterFirst).isPositive();
|
||||
|
||||
orchestrator.runImport();
|
||||
|
||||
assertThat(personRepository.count()).isEqualTo(personsAfterFirst);
|
||||
assertThat(tagRepository.count()).isEqualTo(tagsAfterFirst);
|
||||
assertThat(documentRepository.count()).isEqualTo(documentsAfterFirst);
|
||||
}
|
||||
|
||||
@Test
|
||||
void reimport_preservesHumanEditedPersonField() {
|
||||
orchestrator.runImport();
|
||||
Person walter = personRepository.findBySourceRef("de-gruyter-walter").orElseThrow();
|
||||
walter.setNotes("Verified by archivist");
|
||||
walter.setFirstName("Walther");
|
||||
personRepository.save(walter);
|
||||
|
||||
orchestrator.runImport();
|
||||
|
||||
Person reimported = personRepository.findBySourceRef("de-gruyter-walter").orElseThrow();
|
||||
assertThat(reimported.getNotes()).isEqualTo("Verified by archivist");
|
||||
assertThat(reimported.getFirstName()).isEqualTo("Walther");
|
||||
}
|
||||
|
||||
@Test
|
||||
void import_linksDocumentSenderToRegisterPerson_andRetainsRawText() {
|
||||
orchestrator.runImport();
|
||||
|
||||
Person walter = personRepository.findBySourceRef("de-gruyter-walter").orElseThrow();
|
||||
Document doc = documentRepository.findByOriginalFilename("W-0001").orElseThrow();
|
||||
assertThat(doc.getSender()).isNotNull();
|
||||
assertThat(doc.getSender().getId()).isEqualTo(walter.getId());
|
||||
assertThat(doc.getSenderText()).isEqualTo("Walter de Gruyter");
|
||||
assertThat(doc.getStatus()).isEqualTo(DocumentStatus.PLACEHOLDER);
|
||||
}
|
||||
|
||||
@Test
|
||||
void import_provisionalFlag_trueForImporterCreated_falseForRegister() {
|
||||
orchestrator.runImport();
|
||||
|
||||
Optional<Person> register = personRepository.findBySourceRef("de-gruyter-walter");
|
||||
assertThat(register).get().extracting(Person::isProvisional).isEqualTo(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
void reimport_prunesRemovedReceiverAndTag_whenCanonicalRowShrinks() throws Exception {
|
||||
orchestrator.runImport();
|
||||
// findById uses the Document.full entity graph so receivers/tags initialise eagerly.
|
||||
Document before = documentRepository.findById(
|
||||
documentRepository.findByOriginalFilename("W-0001").orElseThrow().getId()).orElseThrow();
|
||||
assertThat(before.getReceivers()).isNotEmpty();
|
||||
assertThat(before.getTags()).isNotEmpty();
|
||||
|
||||
// Re-stage the document sheet with W-0001's receiver and tag removed.
|
||||
writeSheet(artifactDir.resolve("canonical-documents.xlsx"),
|
||||
List.of("index", "sender_person_id", "sender_name", "receiver_person_ids",
|
||||
"receiver_names", "date_iso", "date_raw", "date_precision", "date_end", "location", "tags", "summary"),
|
||||
List.of(
|
||||
List.of("W-0001", "de-gruyter-walter", "Walter de Gruyter",
|
||||
"", "", "1888-02-15", "15.2.1888", "DAY", "", "Rotterdam", "", "Geschäftsreise"),
|
||||
List.of("W-0002", "de-gruyter-eugenie", "Eugenie de Gruyter",
|
||||
"de-gruyter-walter", "Walter de Gruyter", "1888-02-16", "16.2.1888", "DAY", "",
|
||||
"Middelburg", "Themen/Brautbriefe", "Reisepläne")));
|
||||
|
||||
orchestrator.runImport();
|
||||
|
||||
Document after = documentRepository.findById(before.getId()).orElseThrow();
|
||||
assertThat(after.getReceivers()).isEmpty();
|
||||
assertThat(after.getTags()).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void import_neverFlipsRegisterPersonToProvisional_whenReferencedByDocumentRow() {
|
||||
// de-gruyter-walter is a register person (provisional=false) AND the sender of W-0001.
|
||||
// The orchestrator loads the register before documents, so the document loader's
|
||||
// register-first match links the existing person and never mints a provisional one.
|
||||
// A second run (documents reference the same person again) must not flip it true.
|
||||
orchestrator.runImport();
|
||||
orchestrator.runImport();
|
||||
|
||||
Person walter = personRepository.findBySourceRef("de-gruyter-walter").orElseThrow();
|
||||
assertThat(walter.isProvisional()).isFalse();
|
||||
Person eugenie = personRepository.findBySourceRef("de-gruyter-eugenie").orElseThrow();
|
||||
assertThat(eugenie.isProvisional()).isFalse();
|
||||
}
|
||||
|
||||
// ─── synthetic-but-real artifact set ─────────────────────────────────────────────
|
||||
|
||||
private void writeArtifacts(Path dir) throws Exception {
|
||||
writeSheet(dir.resolve("canonical-tag-tree.xlsx"),
|
||||
List.of("tag_path", "parent_name", "tag_name"),
|
||||
List.of(
|
||||
List.of("Themen", "", "Themen"),
|
||||
List.of("Themen/Brautbriefe", "Themen", "Brautbriefe")));
|
||||
|
||||
writeSheet(dir.resolve("canonical-persons.xlsx"),
|
||||
List.of("person_id", "last_name", "first_name", "maiden_name", "notes", "birth_date", "death_date", "provisional"),
|
||||
List.of(
|
||||
List.of("de-gruyter-walter", "de Gruyter", "Walter", "", "", "1865-01-01", "", "False"),
|
||||
List.of("de-gruyter-eugenie", "de Gruyter", "Eugenie", "Wöhler", "", "", "", "False")));
|
||||
|
||||
Files.writeString(dir.resolve("canonical-persons-tree.json"), """
|
||||
{"persons":[
|
||||
{"rowId":"row_1","firstName":"Walter","lastName":"de Gruyter","familyMember":true,"personId":"de-gruyter-walter"},
|
||||
{"rowId":"row_2","firstName":"Eugenie","lastName":"de Gruyter","maidenName":"Wöhler","familyMember":true,"personId":"de-gruyter-eugenie"}
|
||||
],"relationships":[
|
||||
{"personId":"row_1","relatedPersonId":"row_2","type":"SPOUSE_OF","source":"verheiratet_mit"}
|
||||
]}
|
||||
""");
|
||||
|
||||
writeSheet(dir.resolve("canonical-documents.xlsx"),
|
||||
List.of("index", "sender_person_id", "sender_name", "receiver_person_ids",
|
||||
"receiver_names", "date_iso", "date_raw", "date_precision", "date_end", "location", "tags", "summary"),
|
||||
List.of(
|
||||
List.of("W-0001", "de-gruyter-walter", "Walter de Gruyter",
|
||||
"de-gruyter-eugenie", "Eugenie de Gruyter", "1888-02-15", "15.2.1888", "DAY", "",
|
||||
"Rotterdam", "Themen/Brautbriefe", "Geschäftsreise"),
|
||||
List.of("W-0002", "de-gruyter-eugenie", "Eugenie de Gruyter",
|
||||
"de-gruyter-walter", "Walter de Gruyter", "1888-02-16", "16.2.1888", "DAY", "",
|
||||
"Middelburg", "Themen/Brautbriefe", "Reisepläne")));
|
||||
}
|
||||
|
||||
private void writeSheet(Path file, List<String> headers, List<List<String>> rows) throws Exception {
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
header.createCell(i).setCellValue(headers.get(i));
|
||||
}
|
||||
for (int r = 0; r < rows.size(); r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
List<String> values = rows.get(r);
|
||||
for (int c = 0; c < values.size(); c++) {
|
||||
row.createCell(c).setCellValue(values.get(c));
|
||||
}
|
||||
}
|
||||
try (OutputStream out = Files.newOutputStream(file)) {
|
||||
wb.write(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class CanonicalImportOrchestratorTest {
|
||||
|
||||
@Mock TagTreeImporter tagTreeImporter;
|
||||
@Mock PersonRegisterImporter personRegisterImporter;
|
||||
@Mock PersonTreeImporter personTreeImporter;
|
||||
@Mock DocumentImporter documentImporter;
|
||||
|
||||
private CanonicalImportOrchestrator orchestrator(Path dir) {
|
||||
CanonicalImportOrchestrator o = new CanonicalImportOrchestrator(
|
||||
tagTreeImporter, personRegisterImporter, personTreeImporter, documentImporter);
|
||||
ReflectionTestUtils.setField(o, "canonicalDir", dir.toString());
|
||||
return o;
|
||||
}
|
||||
|
||||
private void writeAllArtifacts(Path dir) throws Exception {
|
||||
Files.writeString(dir.resolve("canonical-tag-tree.xlsx"), "x");
|
||||
Files.writeString(dir.resolve("canonical-persons.xlsx"), "x");
|
||||
Files.writeString(dir.resolve("canonical-persons-tree.json"), "x");
|
||||
Files.writeString(dir.resolve("canonical-documents.xlsx"), "x");
|
||||
}
|
||||
|
||||
@Test
|
||||
void getStatus_isIdleByDefault(@TempDir Path dir) {
|
||||
assertThat(orchestrator(dir).getStatus().state()).isEqualTo(ImportStatus.State.IDLE);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImport_loadsTagsAndPersonsBeforeDocuments(@TempDir Path dir) throws Exception {
|
||||
writeAllArtifacts(dir);
|
||||
when(documentImporter.load(any())).thenReturn(new DocumentImporter.LoadResult(0, List.of()));
|
||||
CanonicalImportOrchestrator o = orchestrator(dir);
|
||||
|
||||
o.runImport();
|
||||
|
||||
InOrder order = inOrder(tagTreeImporter, personRegisterImporter, personTreeImporter, documentImporter);
|
||||
order.verify(tagTreeImporter).load(any());
|
||||
order.verify(personRegisterImporter).load(any());
|
||||
order.verify(personTreeImporter).load(any());
|
||||
order.verify(documentImporter).load(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImport_setsStatusDone_onSuccess(@TempDir Path dir) throws Exception {
|
||||
writeAllArtifacts(dir);
|
||||
when(documentImporter.load(any())).thenReturn(new DocumentImporter.LoadResult(3, List.of()));
|
||||
CanonicalImportOrchestrator o = orchestrator(dir);
|
||||
|
||||
o.runImport();
|
||||
|
||||
assertThat(o.getStatus().state()).isEqualTo(ImportStatus.State.DONE);
|
||||
assertThat(o.getStatus().processed()).isEqualTo(3);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImport_failsClosed_whenAnArtifactIsMissing(@TempDir Path dir) throws Exception {
|
||||
Files.writeString(dir.resolve("canonical-tag-tree.xlsx"), "x");
|
||||
// the other three artifacts are absent
|
||||
CanonicalImportOrchestrator o = orchestrator(dir);
|
||||
|
||||
o.runImport();
|
||||
|
||||
assertThat(o.getStatus().state()).isEqualTo(ImportStatus.State.FAILED);
|
||||
verify(tagTreeImporter, never()).load(any());
|
||||
verify(documentImporter, never()).load(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImport_setsStatusFailed_whenLoaderThrows(@TempDir Path dir) throws Exception {
|
||||
writeAllArtifacts(dir);
|
||||
when(tagTreeImporter.load(any())).thenThrow(DomainException.badRequest(
|
||||
org.raddatz.familienarchiv.exception.ErrorCode.IMPORT_ARTIFACT_INVALID, "bad"));
|
||||
CanonicalImportOrchestrator o = orchestrator(dir);
|
||||
|
||||
o.runImport();
|
||||
|
||||
assertThat(o.getStatus().state()).isEqualTo(ImportStatus.State.FAILED);
|
||||
verify(documentImporter, never()).load(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_throwsConflict_whenAlreadyRunning(@TempDir Path dir) {
|
||||
CanonicalImportOrchestrator o = orchestrator(dir);
|
||||
ReflectionTestUtils.setField(o, "currentStatus", new ImportStatus(
|
||||
ImportStatus.State.RUNNING, "IMPORT_RUNNING", "running", 0, List.of(), null));
|
||||
|
||||
assertThatThrownBy(o::runImportAsync)
|
||||
.isInstanceOf(DomainException.class)
|
||||
.hasMessageContaining("already in progress");
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImport_aggregatesDocumentSkips(@TempDir Path dir) throws Exception {
|
||||
writeAllArtifacts(dir);
|
||||
when(documentImporter.load(any())).thenReturn(new DocumentImporter.LoadResult(1,
|
||||
List.of(new ImportStatus.SkippedFile("fake.pdf", ImportStatus.SkipReason.INVALID_PDF_SIGNATURE))));
|
||||
CanonicalImportOrchestrator o = orchestrator(dir);
|
||||
|
||||
o.runImport();
|
||||
|
||||
assertThat(o.getStatus().skipped()).isEqualTo(1);
|
||||
assertThat(o.getStatus().skippedFiles())
|
||||
.extracting(ImportStatus.SkippedFile::filename)
|
||||
.containsExactly("fake.pdf");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
class CanonicalSheetReaderTest {
|
||||
|
||||
@Test
|
||||
void readRows_mapsCellsByHeaderName(@TempDir Path tempDir) throws Exception {
|
||||
Path xlsx = write(tempDir, List.of("index", "file"), List.of(List.of("W-0001", "scan.pdf")));
|
||||
|
||||
List<CanonicalSheetReader.Row> rows = CanonicalSheetReader.readRows(xlsx.toFile(), List.of("index", "file"));
|
||||
|
||||
assertThat(rows).hasSize(1);
|
||||
assertThat(rows.get(0).get("index")).isEqualTo("W-0001");
|
||||
assertThat(rows.get(0).get("file")).isEqualTo("scan.pdf");
|
||||
}
|
||||
|
||||
@Test
|
||||
void readRows_throwsBadRequest_whenRequiredHeaderMissing(@TempDir Path tempDir) throws Exception {
|
||||
Path xlsx = write(tempDir, List.of("index"), List.of(List.of("W-0001")));
|
||||
|
||||
assertThatThrownBy(() -> CanonicalSheetReader.readRows(xlsx.toFile(), List.of("index", "file")))
|
||||
.isInstanceOf(DomainException.class)
|
||||
.hasMessageContaining("file");
|
||||
}
|
||||
|
||||
@Test
|
||||
void get_returnsEmptyString_forBlankCell(@TempDir Path tempDir) throws Exception {
|
||||
Path xlsx = write(tempDir, List.of("index", "file"), List.of(List.of("W-0001", "")));
|
||||
|
||||
List<CanonicalSheetReader.Row> rows = CanonicalSheetReader.readRows(xlsx.toFile(), List.of("index", "file"));
|
||||
|
||||
assertThat(rows.get(0).get("file")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void get_returnsEmptyString_forUnknownColumn(@TempDir Path tempDir) throws Exception {
|
||||
Path xlsx = write(tempDir, List.of("index"), List.of(List.of("W-0001")));
|
||||
|
||||
List<CanonicalSheetReader.Row> rows = CanonicalSheetReader.readRows(xlsx.toFile(), List.of("index"));
|
||||
|
||||
assertThat(rows.get(0).get("does_not_exist")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void get_returnsEmptyString_forTrailingColumns_whenRowShorterThanHeader(@TempDir Path tempDir) throws Exception {
|
||||
// POI omits trailing empty cells, so a real-world artifact row can be narrower than
|
||||
// the header. The missing columns must read as "" rather than throwing.
|
||||
Path xlsx = write(tempDir,
|
||||
List.of("index", "file", "summary"),
|
||||
List.of(List.of("W-0001")));
|
||||
|
||||
List<CanonicalSheetReader.Row> rows = CanonicalSheetReader.readRows(xlsx.toFile(), List.of("index", "file", "summary"));
|
||||
|
||||
assertThat(rows.get(0).get("index")).isEqualTo("W-0001");
|
||||
assertThat(rows.get(0).get("file")).isEmpty();
|
||||
assertThat(rows.get(0).get("summary")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitList_splitsOnPipe() {
|
||||
assertThat(CanonicalSheetReader.splitList("a|b|c")).containsExactly("a", "b", "c");
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitList_returnsEmptyList_forBlank() {
|
||||
assertThat(CanonicalSheetReader.splitList("")).isEmpty();
|
||||
assertThat(CanonicalSheetReader.splitList(" ")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitList_returnsSingleElement_whenNoPipe() {
|
||||
assertThat(CanonicalSheetReader.splitList("solo")).containsExactly("solo");
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitList_trimsAndDropsEmptySegments() {
|
||||
assertThat(CanonicalSheetReader.splitList("a| |b")).containsExactly("a", "b");
|
||||
}
|
||||
|
||||
private Path write(Path dir, List<String> headers, List<List<String>> dataRows) throws Exception {
|
||||
Path xlsx = dir.resolve("sheet.xlsx");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
header.createCell(i).setCellValue(headers.get(i));
|
||||
}
|
||||
for (int r = 0; r < dataRows.size(); r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
List<String> values = dataRows.get(r);
|
||||
for (int c = 0; c < values.size(); c++) {
|
||||
row.createCell(c).setCellValue(values.get(c));
|
||||
}
|
||||
}
|
||||
try (OutputStream out = Files.newOutputStream(xlsx)) {
|
||||
wb.write(out);
|
||||
}
|
||||
}
|
||||
return xlsx;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
import org.raddatz.familienarchiv.document.DocumentService;
|
||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.raddatz.familienarchiv.document.ThumbnailAsyncRunner;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.raddatz.familienarchiv.person.PersonUpsertCommand;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
import org.raddatz.familienarchiv.tag.TagService;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDate;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class DocumentImporterTest {
|
||||
|
||||
@Mock DocumentService documentService;
|
||||
@Mock PersonService personService;
|
||||
@Mock TagService tagService;
|
||||
@Mock S3Client s3Client;
|
||||
@Mock ThumbnailAsyncRunner thumbnailAsyncRunner;
|
||||
|
||||
DocumentImporter importer;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
importer = new DocumentImporter(documentService, personService, tagService, s3Client, thumbnailAsyncRunner);
|
||||
ReflectionTestUtils.setField(importer, "bucketName", "test-bucket");
|
||||
}
|
||||
|
||||
// ─── index validation — a malicious/garbage index can never reach disk I/O ─────────
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenNull() {
|
||||
assertThat(validIndex(null)).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenBlank() {
|
||||
assertThat(validIndex(" ")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenForwardSlash() {
|
||||
assertThat(validIndex("etc/passwd")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenBackslash() {
|
||||
assertThat(validIndex("..\\etc\\passwd")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenDotDot() {
|
||||
assertThat(validIndex("W-..0001")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenIsDotDot() {
|
||||
assertThat(validIndex("..")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenSingleDot() {
|
||||
assertThat(validIndex(".")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenAbsolutePath() {
|
||||
assertThat(validIndex("/etc/passwd")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenNullByte() {
|
||||
assertThat(validIndex("W-0001\0")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenUnicodeDivisionSlash() {
|
||||
assertThat(validIndex("W∕0001")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenFullwidthSlash() {
|
||||
assertThat(validIndex("W/0001")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenReverseSolidusOperator() {
|
||||
assertThat(validIndex("W⧵0001")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenContainsDotPdfExtension() {
|
||||
// The index is the bare catalog id; appending ".pdf" is the importer's job. A dot in
|
||||
// the index would let "W-0001.pdf" become "W-0001.pdf.pdf" or smuggle an extension.
|
||||
assertThat(validIndex("W-0001.pdf")).isFalse();
|
||||
}
|
||||
|
||||
// ─── catalog-shape rejects — pass the char pre-checks but must fail INDEX_PATTERN ────
|
||||
// These pin the regex branch itself: each string contains no separator, dot, slash
|
||||
// homoglyph, null byte, or absolute marker, so it sails past every char guard and is
|
||||
// rejected *only* because INDEX_PATTERN.matches() returns false. A weaker pattern would
|
||||
// let them through — these tests would then go red.
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenSpaceInIndex() {
|
||||
// The real-world reject: "J 0070" is a space-typo with no PDF on disk.
|
||||
assertThat(validIndex("J 0070")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenFiveLetterPrefix() {
|
||||
// The catalog prefix is at most 4 letters; 5 must not match.
|
||||
assertThat(validIndex("WXYZA-0001")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenNoLetterPrefix() {
|
||||
// A digit-led id (no letter prefix) is not a catalog shape.
|
||||
assertThat(validIndex("12-0001")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsFalse_whenUppercaseXSuffix() {
|
||||
// Only a lowercase trailing "x" is allowed; an uppercase "X" suffix must fail.
|
||||
assertThat(validIndex("W-0001X")).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsTrue_whenPlainCatalogIndex() {
|
||||
assertThat(validIndex("W-0124")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsTrue_whenTwoLetterPrefix() {
|
||||
assertThat(validIndex("Al-0001")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsTrue_whenThreeLetterPrefix() {
|
||||
assertThat(validIndex("CuH-0010")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsTrue_whenUmlautPrefix() {
|
||||
// Real corpus indices carry a German umlaut, e.g. "Mü-0001.pdf" exists on disk.
|
||||
assertThat(validIndex("Mü-0001")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsTrue_whenDoubleHyphen() {
|
||||
// Real corpus: "C--0029" appears in the spreadsheet (a data-entry artefact, but a
|
||||
// legitimate catalog shape that must still resolve, not crash).
|
||||
assertThat(validIndex("C--0029")).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportIndex_returnsTrue_whenXSuffix() {
|
||||
// The normalizer recognises an x-suffix catalog id; allow it defensively.
|
||||
assertThat(validIndex("W-0001x")).isTrue();
|
||||
}
|
||||
|
||||
// ─── a valid index resolves to exactly importDir/<index>.pdf within containment ─────
|
||||
|
||||
@Test
|
||||
void load_resolvesPdfByIndex_uploadsToS3_andSetsStatusUploaded(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
byte[] pdf = {0x25, 0x50, 0x44, 0x46, 0x2D};
|
||||
Files.write(tempDir.resolve("W-0124.pdf"), pdf);
|
||||
when(documentService.findByOriginalFilename("W-0124")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0124", "", "", "", "", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
// exactly importDir/<index>.pdf was uploaded — the S3 key carries that basename
|
||||
org.mockito.ArgumentCaptor<RequestBody> bodyCaptor = org.mockito.ArgumentCaptor.forClass(RequestBody.class);
|
||||
verify(s3Client).putObject(any(PutObjectRequest.class), bodyCaptor.capture());
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
d.getStatus() == DocumentStatus.UPLOADED
|
||||
&& d.getFilePath() != null
|
||||
&& d.getFilePath().endsWith("_W-0124.pdf")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_yieldsPlaceholder_whenIndexedPdfMissing(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("X-9999")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("X-9999", "", "", "", "", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d -> d.getStatus() == DocumentStatus.PLACEHOLDER));
|
||||
verify(s3Client, never()).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_rejectsMaliciousIndex_neverReadsOutsideImportDir(@TempDir Path tempDir) throws Exception {
|
||||
// An index with a path separator must be skipped outright, never used for disk I/O.
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Path xlsx = writeDocs(tempDir, docRow("../../etc/cron.d/x", "", "", "", "", "", "", "", ""));
|
||||
|
||||
DocumentImporter.LoadResult result = importer.load(xlsx.toFile());
|
||||
|
||||
assertThat(result.skippedFiles())
|
||||
.extracting(ImportStatus.SkippedFile::reason)
|
||||
.containsExactly(ImportStatus.SkipReason.INVALID_FILENAME_PATH_TRAVERSAL);
|
||||
verify(documentService, never()).save(any());
|
||||
verify(s3Client, never()).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvePdfByIndex_throwsWhenResolvedPathEscapesImportDir_viaSymlink(
|
||||
@TempDir Path importDirPath, @TempDir Path outsideDir) throws Exception {
|
||||
// Containment defense-in-depth: even a syntactically valid index whose <index>.pdf is a
|
||||
// symlink pointing outside importDir must be refused — the resolved canonical path is
|
||||
// asserted to stay inside importDir.
|
||||
Path outsideFile = outsideDir.resolve("secret.pdf");
|
||||
Files.writeString(outsideFile, "sensitive");
|
||||
Files.createSymbolicLink(importDirPath.resolve("W-0001.pdf"), outsideFile);
|
||||
ReflectionTestUtils.setField(importer, "importDir", importDirPath.toString());
|
||||
|
||||
org.assertj.core.api.Assertions.assertThatThrownBy(
|
||||
() -> ReflectionTestUtils.invokeMethod(importer, "resolvePdfByIndex", "W-0001", 2))
|
||||
.isInstanceOf(org.raddatz.familienarchiv.exception.DomainException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvePdfByIndex_returnsExactlyImportDirIndexPdf_whenPresent(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Path expected = tempDir.resolve("Eu-0628.pdf");
|
||||
Files.writeString(expected, "%PDF-1.4");
|
||||
|
||||
Optional<File> resolved = ReflectionTestUtils.invokeMethod(importer, "resolvePdfByIndex", "Eu-0628", 2);
|
||||
|
||||
assertThat(resolved).isPresent();
|
||||
assertThat(resolved.get().getCanonicalFile()).isEqualTo(expected.toFile().getCanonicalFile());
|
||||
}
|
||||
|
||||
// NOTE (Sara, PR #687): the IOException branch of resolvePdfByIndex — where
|
||||
// File.getCanonicalPath() itself throws (an OS-level failure mid-resolution, not the
|
||||
// symlink-escape DomainException) — is intentionally NOT covered by a test. Unlike
|
||||
// isPdfMagicBytes, which has the package-private openFileStream(File) seam a Mockito spy can
|
||||
// make throw, getCanonicalPath() is called on a File built internally with no injection seam,
|
||||
// and there is no portable, deterministic way to make it throw on a temp file (it does not
|
||||
// throw for missing/symlinked paths — those are handled by isFile()/the containment check).
|
||||
// Adding a seam purely to test this would be production code in service of a non-defect; the
|
||||
// substantive fix is the log.warn() now emitted in that branch so the quiet skip surfaces in
|
||||
// ops. Left uncovered by deliberate decision, documented here so the branch is not assumed
|
||||
// tested.
|
||||
|
||||
// ─── PDF magic-byte guard — ported — do not remove ──────────────────────────────
|
||||
|
||||
@Test
|
||||
void load_skipsFile_whenNotPdfMagicBytes(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Files.writeString(tempDir.resolve("W-0001.pdf"), "not a pdf");
|
||||
lenient().when(documentService.findByOriginalFilename(any())).thenReturn(Optional.empty());
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0001", "", "", "", "", "", "", "", ""));
|
||||
|
||||
DocumentImporter.LoadResult result = importer.load(xlsx.toFile());
|
||||
|
||||
assertThat(result.skippedFiles())
|
||||
.extracting(ImportStatus.SkippedFile::reason)
|
||||
.containsExactly(ImportStatus.SkipReason.INVALID_PDF_SIGNATURE);
|
||||
verify(s3Client, never()).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_skipsFile_whenMagicByteCheckThrowsIoException(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Files.writeString(tempDir.resolve("W-0001.pdf"), "content");
|
||||
lenient().when(documentService.findByOriginalFilename(any())).thenReturn(Optional.empty());
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0001", "", "", "", "", "", "", "", ""));
|
||||
|
||||
DocumentImporter spyImporter = org.mockito.Mockito.spy(importer);
|
||||
org.mockito.Mockito.doThrow(new java.io.IOException("read error"))
|
||||
.when(spyImporter).openFileStream(any(File.class));
|
||||
|
||||
DocumentImporter.LoadResult result = spyImporter.load(xlsx.toFile());
|
||||
|
||||
assertThat(result.skippedFiles())
|
||||
.extracting(ImportStatus.SkippedFile::reason)
|
||||
.containsExactly(ImportStatus.SkipReason.FILE_READ_ERROR);
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_skipsAlreadyExists_whenDocumentUploadedNotPlaceholder(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Document existing = Document.builder().id(UUID.randomUUID())
|
||||
.originalFilename("W-0001").status(DocumentStatus.UPLOADED).build();
|
||||
when(documentService.findByOriginalFilename("W-0001")).thenReturn(Optional.of(existing));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0001", "", "", "", "", "", "", "", ""));
|
||||
|
||||
DocumentImporter.LoadResult result = importer.load(xlsx.toFile());
|
||||
|
||||
assertThat(result.skippedFiles())
|
||||
.extracting(ImportStatus.SkippedFile::reason)
|
||||
.containsExactly(ImportStatus.SkipReason.ALREADY_EXISTS);
|
||||
verify(documentService, never()).save(any());
|
||||
}
|
||||
|
||||
// ─── presence of importDir/<index>.pdf drives status: present → UPLOADED, absent → PLACEHOLDER ─
|
||||
|
||||
@Test
|
||||
void load_setsStatusPlaceholder_whenNoIndexedPdf(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("W-0099")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0099", "", "", "", "", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d -> d.getStatus() == DocumentStatus.PLACEHOLDER));
|
||||
verify(s3Client, never()).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
}
|
||||
|
||||
// ─── attribution routing — register-first + always retain raw ────────────────────
|
||||
|
||||
@Test
|
||||
void load_linksRegisterSender_andRetainsRawSenderText(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Person walter = Person.builder().id(UUID.randomUUID()).sourceRef("de-gruyter-walter")
|
||||
.firstName("Walter").lastName("de Gruyter").build();
|
||||
when(documentService.findByOriginalFilename("W-0001")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
when(personService.findBySourceRef("de-gruyter-walter")).thenReturn(Optional.of(walter));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0001", "de-gruyter-walter", "Walter de Gruyter",
|
||||
"", "", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
d.getSender() == walter && "Walter de Gruyter".equals(d.getSenderText())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_createsProvisionalSender_whenSlugUnmatchedInRegister(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Person provisional = Person.builder().id(UUID.randomUUID()).sourceRef("schwester-hanni")
|
||||
.lastName("Schwester Hanni").provisional(true).build();
|
||||
when(documentService.findByOriginalFilename("W-0002")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
when(personService.findBySourceRef("schwester-hanni")).thenReturn(Optional.empty());
|
||||
when(personService.upsertBySourceRef(any())).thenReturn(provisional);
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0002", "schwester-hanni", "Schwester Hanni",
|
||||
"", "", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
org.mockito.ArgumentCaptor<PersonUpsertCommand> captor =
|
||||
org.mockito.ArgumentCaptor.forClass(PersonUpsertCommand.class);
|
||||
verify(personService).upsertBySourceRef(captor.capture());
|
||||
assertThat(captor.getValue().provisional()).isTrue();
|
||||
assertThat(captor.getValue().lastName()).isEqualTo("Schwester Hanni");
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_createsNoSenderPerson_whenSlugEmptyButRawPresent(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("W-0003")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0003", "", "?",
|
||||
"", "", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(personService, never()).findBySourceRef(any());
|
||||
verify(personService, never()).upsertBySourceRef(any());
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
d.getSender() == null && "?".equals(d.getSenderText())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_splitsMultipleReceivers_andRetainsRawReceiverText(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Person herbert = Person.builder().id(UUID.randomUUID()).sourceRef("cram-herbert").lastName("Cram").build();
|
||||
Person clara = Person.builder().id(UUID.randomUUID()).sourceRef("clara").lastName("Clara").build();
|
||||
when(documentService.findByOriginalFilename("W-0004")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
when(personService.findBySourceRef("cram-herbert")).thenReturn(Optional.of(herbert));
|
||||
when(personService.findBySourceRef("clara")).thenReturn(Optional.of(clara));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0004", "", "",
|
||||
"cram-herbert|clara", "Herbert Cram|Clara", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
d.getReceivers().size() == 2
|
||||
&& d.getReceivers().contains(herbert)
|
||||
&& d.getReceivers().contains(clara)
|
||||
&& "Herbert Cram|Clara".equals(d.getReceiverText())));
|
||||
}
|
||||
|
||||
// ─── clean date values parse without semantic logic ──────────────────────────────
|
||||
|
||||
@Test
|
||||
void load_parsesCleanDateAndPrecision(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("W-0005")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0005", "", "",
|
||||
"", "", "1916-06-01", "1.6.1916", "MONTH", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
LocalDate.of(1916, 6, 1).equals(d.getDocumentDate())
|
||||
&& d.getMetaDatePrecision() == org.raddatz.familienarchiv.document.DatePrecision.MONTH
|
||||
&& "1.6.1916".equals(d.getMetaDateRaw())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_attachesTagBySourceRef(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Tag tag = Tag.builder().id(UUID.randomUUID()).name("Brautbriefe").sourceRef("Themen/Brautbriefe").build();
|
||||
when(documentService.findByOriginalFilename("W-0006")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
when(tagService.findBySourceRef("Themen/Brautbriefe")).thenReturn(Optional.of(tag));
|
||||
Path xlsx = writeDocs(tempDir, docRowWithTag("W-0006", "Themen/Brautbriefe"));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d -> d.getTags().contains(tag)));
|
||||
}
|
||||
|
||||
// ─── idempotency — update existing document in place by index ─────────────────────
|
||||
|
||||
@Test
|
||||
void load_updatesExistingDocumentInPlace_whenIndexExists(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Document existing = Document.builder().id(UUID.randomUUID())
|
||||
.originalFilename("W-0007").status(DocumentStatus.PLACEHOLDER).build();
|
||||
when(documentService.findByOriginalFilename("W-0007")).thenReturn(Optional.of(existing));
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0007", "", "", "", "", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d -> d.getId().equals(existing.getId())));
|
||||
}
|
||||
|
||||
// ─── canonical collections are authoritative — re-import prunes removed links ──────
|
||||
|
||||
@Test
|
||||
void load_prunesReceiversAndTags_whenCanonicalRowShrinks(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
Person staleReceiver = Person.builder().id(UUID.randomUUID()).sourceRef("stale-receiver").lastName("Stale").build();
|
||||
Tag staleTag = Tag.builder().id(UUID.randomUUID()).name("Stale").sourceRef("Themen/Stale").build();
|
||||
Document existing = Document.builder().id(UUID.randomUUID())
|
||||
.originalFilename("W-0008").status(DocumentStatus.PLACEHOLDER).build();
|
||||
existing.getReceivers().add(staleReceiver);
|
||||
existing.getTags().add(staleTag);
|
||||
when(documentService.findByOriginalFilename("W-0008")).thenReturn(Optional.of(existing));
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
// The canonical row now carries no receiver and no tag: both stale links must go.
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0008", "", "", "", "", "", "", "", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
d.getReceivers().isEmpty() && d.getTags().isEmpty()));
|
||||
}
|
||||
|
||||
// ─── title carries the honest date label — never a precision the data lacks ───────
|
||||
|
||||
@Test
|
||||
void load_buildsTitleWithMonthLabel_whenPrecisionIsMonth(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("W-0100")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0100", "", "", "", "",
|
||||
"1916-06-01", "Juni 1916", "MONTH", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
d.getTitle().contains("Juni 1916") && !d.getTitle().contains("1. Juni")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_buildsTitleWithFullDate_whenPrecisionIsDay(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("W-0101")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0101", "", "", "", "",
|
||||
"1943-12-24", "24.12.1943", "DAY", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
d.getTitle().contains("24. Dezember 1943")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_buildsTitleFromIndexOnly_whenDateUnknown(@TempDir Path tempDir) throws Exception {
|
||||
ReflectionTestUtils.setField(importer, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("W-0102")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
Path xlsx = writeDocs(tempDir, docRow("W-0102", "", "", "", "",
|
||||
"", "?", "UNKNOWN", ""));
|
||||
|
||||
importer.load(xlsx.toFile());
|
||||
|
||||
verify(documentService).save(org.mockito.ArgumentMatchers.argThat(d ->
|
||||
d.getTitle().equals("W-0102")));
|
||||
}
|
||||
|
||||
// ─── helpers ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
private Boolean validIndex(String index) {
|
||||
return ReflectionTestUtils.invokeMethod(importer, "isValidImportIndex", index);
|
||||
}
|
||||
|
||||
private Map<String, String> docRow(String index, String senderId, String senderName,
|
||||
String receiverIds, String receiverNames, String dateIso,
|
||||
String dateRaw, String datePrecision, String dateEnd) {
|
||||
Map<String, String> r = new LinkedHashMap<>();
|
||||
r.put("index", index);
|
||||
r.put("sender_person_id", senderId);
|
||||
r.put("sender_name", senderName);
|
||||
r.put("receiver_person_ids", receiverIds);
|
||||
r.put("receiver_names", receiverNames);
|
||||
r.put("date_iso", dateIso);
|
||||
r.put("date_raw", dateRaw);
|
||||
r.put("date_precision", datePrecision);
|
||||
r.put("date_end", dateEnd);
|
||||
r.put("location", "");
|
||||
r.put("tags", "");
|
||||
r.put("summary", "");
|
||||
return r;
|
||||
}
|
||||
|
||||
private Map<String, String> docRowWithTag(String index, String tagPath) {
|
||||
Map<String, String> r = docRow(index, "", "", "", "", "", "", "", "");
|
||||
r.put("tags", tagPath);
|
||||
return r;
|
||||
}
|
||||
|
||||
@SafeVarargs
|
||||
private Path writeDocs(Path dir, Map<String, String>... rows) throws Exception {
|
||||
Path xlsx = dir.resolve("canonical-documents.xlsx");
|
||||
List<String> headers = List.of("index", "sender_person_id", "sender_name",
|
||||
"receiver_person_ids", "receiver_names", "date_iso", "date_raw", "date_precision",
|
||||
"date_end", "location", "tags", "summary");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
header.createCell(i).setCellValue(headers.get(i));
|
||||
}
|
||||
for (int r = 0; r < rows.length; r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
for (int c = 0; c < headers.size(); c++) {
|
||||
row.createCell(c).setCellValue(rows[r].getOrDefault(headers.get(c), ""));
|
||||
}
|
||||
}
|
||||
try (OutputStream out = Files.newOutputStream(xlsx)) {
|
||||
wb.write(out);
|
||||
}
|
||||
}
|
||||
return xlsx;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.DynamicTest;
|
||||
import org.junit.jupiter.api.TestFactory;
|
||||
import org.raddatz.familienarchiv.document.DatePrecision;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Asserts the Java title label against the SAME shared fixture table the TS
|
||||
* formatter spec uses ({@code docs/date-label-fixtures.json}). This is the
|
||||
* drift guard requested in #666 review: the two label implementations cannot
|
||||
* silently diverge (en-dash vs hyphen, "ca." vs "circa", season words, range
|
||||
* collapse) because both are pinned to one committed rule set.
|
||||
*/
|
||||
class DocumentTitleFormatterTest {
|
||||
|
||||
@TestFactory
|
||||
List<DynamicTest> matchesSharedFixtureTable() throws Exception {
|
||||
// Maven runs tests from the backend/ module dir; the fixture lives at repo-root docs/.
|
||||
Path fixture = Path.of("..", "docs", "date-label-fixtures.json");
|
||||
JsonNode root = new ObjectMapper().readTree(Files.readString(fixture));
|
||||
List<DynamicTest> tests = new ArrayList<>();
|
||||
for (JsonNode c : root.get("cases")) {
|
||||
String name = c.get("name").asText();
|
||||
LocalDate anchor = parseDate(c.get("anchor"));
|
||||
DatePrecision precision = DatePrecision.valueOf(c.get("precision").asText());
|
||||
LocalDate end = parseDate(c.get("end"));
|
||||
String raw = c.get("raw").isNull() ? null : c.get("raw").asText();
|
||||
String expected = c.get("expected").asText();
|
||||
tests.add(DynamicTest.dynamicTest(name, () ->
|
||||
assertThat(DocumentTitleFormatter.formatTitleDate(anchor, precision, end, raw))
|
||||
.isEqualTo(expected)));
|
||||
}
|
||||
return tests;
|
||||
}
|
||||
|
||||
private static LocalDate parseDate(JsonNode node) {
|
||||
return node == null || node.isNull() ? null : LocalDate.parse(node.asText());
|
||||
}
|
||||
}
|
||||
@@ -1,896 +0,0 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
import org.raddatz.familienarchiv.document.DocumentService;
|
||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.raddatz.familienarchiv.document.ThumbnailAsyncRunner;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
import org.raddatz.familienarchiv.tag.TagService;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.xml.sax.SAXParseException;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.OutputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class MassImportServiceTest {
|
||||
|
||||
@Mock DocumentService documentService;
|
||||
@Mock PersonService personService;
|
||||
@Mock TagService tagService;
|
||||
@Mock S3Client s3Client;
|
||||
@Mock ThumbnailAsyncRunner thumbnailAsyncRunner;
|
||||
|
||||
MassImportService service;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new MassImportService(documentService, personService, tagService, s3Client, thumbnailAsyncRunner);
|
||||
ReflectionTestUtils.setField(service, "bucketName", "test-bucket");
|
||||
ReflectionTestUtils.setField(service, "importDir", "/import");
|
||||
ReflectionTestUtils.setField(service, "colIndex", 0);
|
||||
ReflectionTestUtils.setField(service, "colBox", 1);
|
||||
ReflectionTestUtils.setField(service, "colFolder", 2);
|
||||
ReflectionTestUtils.setField(service, "colSender", 3);
|
||||
ReflectionTestUtils.setField(service, "colReceivers", 5);
|
||||
ReflectionTestUtils.setField(service, "colDate", 7);
|
||||
ReflectionTestUtils.setField(service, "colLocation", 9);
|
||||
ReflectionTestUtils.setField(service, "colTags", 10);
|
||||
ReflectionTestUtils.setField(service, "colSummary", 11);
|
||||
ReflectionTestUtils.setField(service, "colTranscription", 13);
|
||||
}
|
||||
|
||||
// ─── getStatus ────────────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void getStatus_returnsIdleByDefault() {
|
||||
assertThat(service.getStatus().state()).isEqualTo(MassImportService.State.IDLE);
|
||||
}
|
||||
|
||||
@Test
|
||||
void getStatus_hasStatusCode_IMPORT_IDLE_byDefault() {
|
||||
assertThat(service.getStatus().statusCode()).isEqualTo("IMPORT_IDLE");
|
||||
}
|
||||
|
||||
// ─── runImportAsync ───────────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void runImportAsync_setsFailedStatus_whenImportDirectoryDoesNotExist() {
|
||||
// /import directory doesn't exist in test environment → IOException → IMPORT_FAILED_INTERNAL
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().state()).isEqualTo(MassImportService.State.FAILED);
|
||||
assertThat(service.getStatus().statusCode()).isEqualTo("IMPORT_FAILED_INTERNAL");
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_readsFromConfiguredImportDir(@TempDir Path tempDir) {
|
||||
// Empty temp dir → findSpreadsheetFile throws "no spreadsheet" with the
|
||||
// configured path in the message. Proves the field, not a constant,
|
||||
// drives the lookup.
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().state()).isEqualTo(MassImportService.State.FAILED);
|
||||
assertThat(service.getStatus().message()).contains(tempDir.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_setsStatusCode_IMPORT_FAILED_NO_SPREADSHEET_whenDirIsEmpty(@TempDir Path tempDir) {
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().statusCode()).isEqualTo("IMPORT_FAILED_NO_SPREADSHEET");
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_setsStatusCode_IMPORT_DONE_whenSpreadsheetHasNoDataRows(@TempDir Path tempDir) throws Exception {
|
||||
Path xlsx = tempDir.resolve("import.xlsx");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
wb.createSheet("Sheet1");
|
||||
try (OutputStream out = Files.newOutputStream(xlsx)) {
|
||||
wb.write(out);
|
||||
}
|
||||
}
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().statusCode()).isEqualTo("IMPORT_DONE");
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_throwsConflict_whenAlreadyRunning() {
|
||||
MassImportService.ImportStatus running = new MassImportService.ImportStatus(
|
||||
MassImportService.State.RUNNING, "IMPORT_RUNNING", "Running...", 0, List.of(), LocalDateTime.now());
|
||||
ReflectionTestUtils.setField(service, "currentStatus", running);
|
||||
|
||||
assertThatThrownBy(() -> service.runImportAsync())
|
||||
.isInstanceOf(DomainException.class)
|
||||
.hasMessageContaining("already in progress");
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — skip already uploaded ─────────────────────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_skips_whenDocumentAlreadyUploadedNotPlaceholder() {
|
||||
Document existing = Document.builder()
|
||||
.id(UUID.randomUUID())
|
||||
.originalFilename("doc001.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.build();
|
||||
when(documentService.findByOriginalFilename("doc001.pdf")).thenReturn(Optional.of(existing));
|
||||
|
||||
Optional<MassImportService.SkipReason> result = service.importSingleDocument(minimalCells("doc001.pdf"), Optional.empty(), "doc001.pdf", "doc001");
|
||||
|
||||
verify(documentService, never()).save(any());
|
||||
assertThat(result).isPresent().contains(MassImportService.SkipReason.ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — already-exists guard fires before file I/O ─────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_skipsWithAlreadyExists_whenDocumentUploadedAndFileIsPresent(@TempDir Path tempDir) throws Exception {
|
||||
// Document already exists with status UPLOADED (not PLACEHOLDER).
|
||||
// A physical PDF file is also present on disk (valid magic bytes).
|
||||
// Expected: ALREADY_EXISTS is returned and no S3 upload is attempted —
|
||||
// the guard fires before any file I/O, so no partial processing occurs.
|
||||
Document existing = Document.builder()
|
||||
.id(UUID.randomUUID())
|
||||
.originalFilename("present.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.build();
|
||||
when(documentService.findByOriginalFilename("present.pdf")).thenReturn(Optional.of(existing));
|
||||
|
||||
Path physicalFile = tempDir.resolve("present.pdf");
|
||||
byte[] pdfHeader = {0x25, 0x50, 0x44, 0x46, 0x2D}; // %PDF-
|
||||
Files.write(physicalFile, pdfHeader);
|
||||
|
||||
Optional<MassImportService.SkipReason> result = service.importSingleDocument(
|
||||
minimalCells("present.pdf"), Optional.of(physicalFile.toFile()), "present.pdf", "present");
|
||||
|
||||
assertThat(result).isPresent().contains(MassImportService.SkipReason.ALREADY_EXISTS);
|
||||
verify(s3Client, never()).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
verify(documentService, never()).save(any());
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — S3 failure surfaced in skippedFiles ──────────
|
||||
|
||||
@Test
|
||||
void runImportAsync_addsS3UploadFailed_toSkippedFiles_whenS3Throws(@TempDir Path tempDir) throws Exception {
|
||||
byte[] pdfHeader = {0x25, 0x50, 0x44, 0x46, 0x2D}; // %PDF-
|
||||
Files.write(tempDir.resolve("upload_fail.pdf"), pdfHeader);
|
||||
buildMinimalImportXlsx(tempDir, "upload_fail.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename("upload_fail.pdf")).thenReturn(Optional.empty());
|
||||
doThrow(new RuntimeException("S3 unavailable"))
|
||||
.when(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skipped()).isEqualTo(1);
|
||||
assertThat(service.getStatus().skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::filename, MassImportService.SkippedFile::reason)
|
||||
.containsExactly(org.assertj.core.groups.Tuple.tuple("upload_fail.pdf", MassImportService.SkipReason.S3_UPLOAD_FAILED));
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_addsAlreadyExists_toSkippedFiles_whenDocumentAlreadyUploaded(@TempDir Path tempDir) throws Exception {
|
||||
buildMinimalImportXlsx(tempDir, "existing.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
Document existing = Document.builder()
|
||||
.id(UUID.randomUUID())
|
||||
.originalFilename("existing.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.build();
|
||||
when(documentService.findByOriginalFilename("existing.pdf")).thenReturn(Optional.of(existing));
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skipped()).isEqualTo(1);
|
||||
assertThat(service.getStatus().skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::reason)
|
||||
.containsExactly(MassImportService.SkipReason.ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — create new document (metadata only) ───────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_createsNewDocument_whenNotExists() {
|
||||
when(documentService.findByOriginalFilename("doc002.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
service.importSingleDocument(minimalCells("doc002.pdf"), Optional.empty(), "doc002.pdf", "doc002");
|
||||
|
||||
verify(documentService).save(argThat(d ->
|
||||
d.getOriginalFilename().equals("doc002.pdf")
|
||||
&& d.getStatus() == DocumentStatus.PLACEHOLDER));
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — update existing placeholder ──────────────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_updatesExistingPlaceholder() {
|
||||
Document placeholder = Document.builder()
|
||||
.id(UUID.randomUUID())
|
||||
.originalFilename("existing.pdf")
|
||||
.status(DocumentStatus.PLACEHOLDER)
|
||||
.build();
|
||||
when(documentService.findByOriginalFilename("existing.pdf")).thenReturn(Optional.of(placeholder));
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
service.importSingleDocument(minimalCells("existing.pdf"), Optional.empty(), "existing.pdf", "existing");
|
||||
|
||||
verify(documentService).save(same(placeholder));
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — with file (S3 upload) ─────────────────────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_uploadsFileToS3_andSetsStatusUploaded(@TempDir Path tempDir) throws Exception {
|
||||
Path tempFile = tempDir.resolve("doc003.pdf");
|
||||
Files.write(tempFile, "PDF content".getBytes());
|
||||
|
||||
when(documentService.findByOriginalFilename("doc003.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
service.importSingleDocument(
|
||||
minimalCells("doc003.pdf"), Optional.of(tempFile.toFile()), "doc003.pdf", "doc003");
|
||||
|
||||
verify(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
verify(documentService).save(argThat(d -> d.getStatus() == DocumentStatus.UPLOADED));
|
||||
}
|
||||
|
||||
@Test
|
||||
void importSingleDocument_returnsS3UploadFailed_whenS3UploadFails(@TempDir Path tempDir) throws Exception {
|
||||
Path tempFile = tempDir.resolve("fail.pdf");
|
||||
Files.write(tempFile, "data".getBytes());
|
||||
|
||||
when(documentService.findByOriginalFilename("fail.pdf")).thenReturn(Optional.empty());
|
||||
doThrow(new RuntimeException("S3 error"))
|
||||
.when(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
|
||||
Optional<MassImportService.SkipReason> result = service.importSingleDocument(
|
||||
minimalCells("fail.pdf"), Optional.of(tempFile.toFile()), "fail.pdf", "fail");
|
||||
|
||||
verify(documentService, never()).save(any());
|
||||
assertThat(result).isPresent().contains(MassImportService.SkipReason.S3_UPLOAD_FAILED);
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — sender handling ───────────────────────────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_setsNullSender_whenSenderCellIsBlank() {
|
||||
when(documentService.findByOriginalFilename("nosender.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<String> cells = buildCells("nosender.pdf", "", "", "");
|
||||
service.importSingleDocument(cells, Optional.empty(), "nosender.pdf", "nosender");
|
||||
|
||||
verify(documentService).save(argThat(d -> d.getSender() == null));
|
||||
verify(personService, never()).findOrCreateByAlias(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void importSingleDocument_createsSender_whenSenderCellIsNonBlank() {
|
||||
Person sender = Person.builder().id(UUID.randomUUID()).firstName("Walter").lastName("Müller").build();
|
||||
when(documentService.findByOriginalFilename("withsender.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
when(personService.findOrCreateByAlias("Walter Müller")).thenReturn(sender);
|
||||
|
||||
List<String> cells = buildCells("withsender.pdf", "Walter Müller", "", "");
|
||||
service.importSingleDocument(cells, Optional.empty(), "withsender.pdf", "withsender");
|
||||
|
||||
verify(personService).findOrCreateByAlias("Walter Müller");
|
||||
verify(documentService).save(argThat(d -> d.getSender() == sender));
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — tag handling ─────────────────────────────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_createsTag_whenTagCellIsNonBlank() {
|
||||
Tag tag = Tag.builder().id(UUID.randomUUID()).name("Familie").build();
|
||||
when(documentService.findByOriginalFilename("tagged.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
when(tagService.findOrCreate("Familie")).thenReturn(tag);
|
||||
|
||||
List<String> cells = buildCells("tagged.pdf", "", "", "Familie");
|
||||
service.importSingleDocument(cells, Optional.empty(), "tagged.pdf", "tagged");
|
||||
|
||||
verify(tagService).findOrCreate("Familie");
|
||||
}
|
||||
|
||||
@Test
|
||||
void importSingleDocument_doesNotCreateTag_whenTagCellIsBlank() {
|
||||
when(documentService.findByOriginalFilename("notag.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<String> cells = buildCells("notag.pdf", "", "", "");
|
||||
service.importSingleDocument(cells, Optional.empty(), "notag.pdf", "notag");
|
||||
|
||||
verify(tagService, never()).findOrCreate(any());
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — metadataComplete heuristic ───────────────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_metadataComplete_whenSenderPresent() {
|
||||
Person sender = Person.builder().id(UUID.randomUUID()).firstName("A").lastName("B").build();
|
||||
when(documentService.findByOriginalFilename("meta.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
when(personService.findOrCreateByAlias("A B")).thenReturn(sender);
|
||||
|
||||
List<String> cells = buildCells("meta.pdf", "A B", "", "");
|
||||
service.importSingleDocument(cells, Optional.empty(), "meta.pdf", "meta");
|
||||
|
||||
verify(documentService).save(argThat(Document::isMetadataComplete));
|
||||
}
|
||||
|
||||
@Test
|
||||
void importSingleDocument_metadataIncomplete_whenNoKeyFieldsPresent() {
|
||||
when(documentService.findByOriginalFilename("nometa.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<String> cells = buildCells("nometa.pdf", "", "", "");
|
||||
service.importSingleDocument(cells, Optional.empty(), "nometa.pdf", "nometa");
|
||||
|
||||
verify(documentService).save(argThat(d -> !d.isMetadataComplete()));
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — blank fields set to null ─────────────────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_setsBlankFieldsToNull() {
|
||||
when(documentService.findByOriginalFilename("blank.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<String> cells = buildCells("blank.pdf", "", "", "");
|
||||
service.importSingleDocument(cells, Optional.empty(), "blank.pdf", "blank");
|
||||
|
||||
verify(documentService).save(argThat(d ->
|
||||
d.getLocation() == null &&
|
||||
d.getSummary() == null &&
|
||||
d.getTranscription() == null &&
|
||||
d.getArchiveBox() == null &&
|
||||
d.getArchiveFolder() == null));
|
||||
}
|
||||
|
||||
// ─── processRows — via ReflectionTestUtils ────────────────────────────────
|
||||
|
||||
@Test
|
||||
void processRows_returnsZero_whenOnlyHeaderRow() {
|
||||
List<List<String>> rows = List.of(List.of("header", "col1"));
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
assertThat(result.processed()).isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void processRows_skipsRowWithBlankIndex() {
|
||||
List<List<String>> rows = List.of(
|
||||
List.of("header"),
|
||||
minimalCells("") // blank index
|
||||
);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
assertThat(result.processed()).isEqualTo(0);
|
||||
verify(documentService, never()).findByOriginalFilename(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void processRows_addsExtension_whenIndexHasNoDot() {
|
||||
when(documentService.findByOriginalFilename("doc001.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<List<String>> rows = List.of(
|
||||
List.of("header"),
|
||||
minimalCells("doc001") // no dot → appends ".pdf"
|
||||
);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
|
||||
assertThat(result.processed()).isEqualTo(1);
|
||||
verify(documentService).findByOriginalFilename("doc001.pdf");
|
||||
}
|
||||
|
||||
@Test
|
||||
void processRows_usesFilenameAsIs_whenIndexHasDot() {
|
||||
when(documentService.findByOriginalFilename("doc002.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<List<String>> rows = List.of(
|
||||
List.of("header"),
|
||||
minimalCells("doc002.pdf") // has dot → used as-is
|
||||
);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
|
||||
assertThat(result.processed()).isEqualTo(1);
|
||||
verify(documentService).findByOriginalFilename("doc002.pdf");
|
||||
}
|
||||
|
||||
// ─── isValidImportFilename — security regression — do not remove ─────────
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameIsNull() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", (String) null);
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameIsBlank() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", " ");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsForwardSlash() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "etc/passwd");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsBackslash() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "..\\etc\\passwd");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsDotDot() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "doc..evil.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameIsDotDot() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "..");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameIsAbsolutePath() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "/etc/passwd");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsNullByte() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "file\0.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsTrue_whenFilenameIsPlainBasename() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "document.pdf");
|
||||
assertThat(result).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsUnicodeDivisionSlash() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "foo∕bar.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsFullwidthSlash() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "foo/bar.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsFalse_whenFilenameContainsUnicodeReverseSolidus() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "foo⧵bar.pdf");
|
||||
assertThat(result).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsTrue_whenFilenameHasLeadingDot() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", ".hidden.pdf");
|
||||
assertThat(result).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void isValidImportFilename_returnsTrue_whenFilenameHasSpaces() {
|
||||
boolean result = ReflectionTestUtils.invokeMethod(service, "isValidImportFilename", "Brief an Oma.pdf");
|
||||
assertThat(result).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void processRows_skipsRowAndContinues_whenFilenameIsPathTraversal() {
|
||||
when(documentService.findByOriginalFilename("legitimate.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<List<String>> rows = List.of(
|
||||
List.of("header"),
|
||||
minimalCells("../evil"), // row 1: path traversal — should be skipped
|
||||
minimalCells("legitimate.pdf") // row 2: valid — should be processed
|
||||
);
|
||||
MassImportService.ProcessResult result = ReflectionTestUtils.invokeMethod(service, "processRows", rows);
|
||||
|
||||
assertThat(result.processed()).isEqualTo(1);
|
||||
assertThat(result.skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::reason)
|
||||
.containsExactly(MassImportService.SkipReason.INVALID_FILENAME_PATH_TRAVERSAL);
|
||||
}
|
||||
|
||||
// ─── importSingleDocument — non-blank optional fields ────────────────────
|
||||
|
||||
@Test
|
||||
void importSingleDocument_setsNonNullOptionalFields_whenPresent() {
|
||||
when(documentService.findByOriginalFilename("rich.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
// box=1, folder=2, location=9, summary=11, transcription=13
|
||||
List<String> cells = List.of(
|
||||
"rich.pdf", // 0: index
|
||||
"Box A", // 1: box
|
||||
"Folder B", // 2: folder
|
||||
"", // 3: sender
|
||||
"", // 4: unused
|
||||
"", // 5: receivers
|
||||
"", // 6: unused
|
||||
"", // 7: date
|
||||
"", // 8: unused
|
||||
"Hamburg", // 9: location
|
||||
"", // 10: tags
|
||||
"A summary", // 11: summary
|
||||
"", // 12: unused
|
||||
"A transcript" // 13: transcription
|
||||
);
|
||||
|
||||
service.importSingleDocument(cells, Optional.empty(), "rich.pdf", "rich");
|
||||
|
||||
verify(documentService).save(argThat(d ->
|
||||
"Box A".equals(d.getArchiveBox()) &&
|
||||
"Folder B".equals(d.getArchiveFolder()) &&
|
||||
"Hamburg".equals(d.getLocation()) &&
|
||||
"A summary".equals(d.getSummary()) &&
|
||||
"A transcript".equals(d.getTranscription())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void importSingleDocument_setsMetadataComplete_whenReceiversArePresent() {
|
||||
Person receiver = Person.builder().id(UUID.randomUUID()).firstName("Walter").lastName("Müller").build();
|
||||
when(documentService.findByOriginalFilename("rcv.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
when(personService.findOrCreateByAlias("Walter Müller")).thenReturn(receiver);
|
||||
|
||||
List<String> cells = List.of(
|
||||
"rcv.pdf", "", "", "", "", "Walter Müller", "", "", "", "", "", "", "", "");
|
||||
service.importSingleDocument(cells, Optional.empty(), "rcv.pdf", "rcv");
|
||||
|
||||
verify(documentService).save(argThat(Document::isMetadataComplete));
|
||||
}
|
||||
|
||||
@Test
|
||||
void importSingleDocument_setsMetadataComplete_whenDateIsPresent() {
|
||||
when(documentService.findByOriginalFilename("dated.pdf")).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
List<String> cells = List.of(
|
||||
"dated.pdf", "", "", "", "", "", "", "2024-03-15", "", "", "", "", "", "");
|
||||
service.importSingleDocument(cells, Optional.empty(), "dated.pdf", "dated");
|
||||
|
||||
verify(documentService).save(argThat(Document::isMetadataComplete));
|
||||
}
|
||||
|
||||
// ─── buildTitle — null location ───────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void buildTitle_withNullLocation_skipsLocationPart() {
|
||||
String result = ReflectionTestUtils.invokeMethod(service, "buildTitle",
|
||||
"doc005", LocalDate.of(1940, 5, 1), (String) null);
|
||||
assertThat(result).contains("doc005").contains("1940");
|
||||
assertThat(result).doesNotContain("Berlin");
|
||||
}
|
||||
|
||||
// ─── parseDate — via ReflectionTestUtils ─────────────────────────────────
|
||||
|
||||
@Test
|
||||
void parseDate_returnsNull_whenValueIsNull() {
|
||||
LocalDate result = ReflectionTestUtils.invokeMethod(service, "parseDate", (String) null);
|
||||
assertThat(result).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseDate_returnsNull_whenValueIsBlank() {
|
||||
LocalDate result = ReflectionTestUtils.invokeMethod(service, "parseDate", " ");
|
||||
assertThat(result).isNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseDate_returnsDate_whenValidIsoFormat() {
|
||||
LocalDate result = ReflectionTestUtils.invokeMethod(service, "parseDate", "2024-03-15");
|
||||
assertThat(result).isEqualTo(LocalDate.of(2024, 3, 15));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseDate_returnsNull_whenInvalidDateString() {
|
||||
LocalDate result = ReflectionTestUtils.invokeMethod(service, "parseDate", "15.03.2024");
|
||||
assertThat(result).isNull();
|
||||
}
|
||||
|
||||
// ─── buildTitle — via ReflectionTestUtils ────────────────────────────────
|
||||
|
||||
@Test
|
||||
void buildTitle_withDateAndLocation() {
|
||||
String result = ReflectionTestUtils.invokeMethod(service, "buildTitle",
|
||||
"doc001", LocalDate.of(1940, 5, 1), "Berlin");
|
||||
assertThat(result).contains("doc001").contains("Berlin").contains("1940");
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildTitle_withDateOnly() {
|
||||
String result = ReflectionTestUtils.invokeMethod(service, "buildTitle",
|
||||
"doc002", LocalDate.of(1960, 8, 15), "");
|
||||
assertThat(result).contains("doc002").contains("1960");
|
||||
assertThat(result).doesNotContain("Berlin");
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildTitle_withIndexOnly_whenDateAndLocationAreNull() {
|
||||
String result = ReflectionTestUtils.invokeMethod(service, "buildTitle",
|
||||
"doc003", null, "");
|
||||
assertThat(result).isEqualTo("doc003");
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildTitle_withLocationOnly_whenDateIsNull() {
|
||||
// date=null, location present → date part skipped, location appended
|
||||
String result = ReflectionTestUtils.invokeMethod(service, "buildTitle",
|
||||
"doc004", null, "Berlin");
|
||||
assertThat(result).contains("doc004").contains("Berlin");
|
||||
assertThat(result).doesNotContain("("); // no date part
|
||||
}
|
||||
|
||||
// ─── getCell — via ReflectionTestUtils ───────────────────────────────────
|
||||
|
||||
@Test
|
||||
void getCell_returnsEmptyString_whenColBeyondListSize() {
|
||||
List<String> cells = List.of("a", "b");
|
||||
String result = ReflectionTestUtils.invokeMethod(service, "getCell", cells, 5);
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getCell_returnsEmptyString_whenValueIsNull() {
|
||||
List<String> cells = new ArrayList<>();
|
||||
cells.add(null);
|
||||
cells.add("b");
|
||||
String result = ReflectionTestUtils.invokeMethod(service, "getCell", cells, 0);
|
||||
assertThat(result).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getCell_returnsTrimmedValue() {
|
||||
List<String> cells = List.of(" hello ", "world");
|
||||
String result = ReflectionTestUtils.invokeMethod(service, "getCell", cells, 0);
|
||||
assertThat(result).isEqualTo("hello");
|
||||
}
|
||||
|
||||
// ─── PDF magic byte validation regression ─────────────────────────────────
|
||||
|
||||
@Test
|
||||
void runImportAsync_uploadsValidPdf_andSkipsFakeOne(@TempDir Path tempDir) throws Exception {
|
||||
setupOneValidOneFakeImport(tempDir);
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
verify(s3Client, times(1)).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_setsSkippedCount_toOne_whenOneFakeFile(@TempDir Path tempDir) throws Exception {
|
||||
setupOneValidOneFakeImport(tempDir);
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skipped()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_includesRejectedFilename_inSkippedFiles(@TempDir Path tempDir) throws Exception {
|
||||
setupOneValidOneFakeImport(tempDir);
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::filename)
|
||||
.contains("fake.pdf");
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_skipsFile_whenShorterThanFourBytes(@TempDir Path tempDir) throws Exception {
|
||||
Files.write(tempDir.resolve("tiny.pdf"), new byte[]{0x25, 0x50, 0x44}); // only 3 bytes
|
||||
buildMinimalImportXlsx(tempDir, "tiny.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
lenient().when(documentService.findByOriginalFilename(any())).thenReturn(Optional.empty());
|
||||
|
||||
service.runImportAsync();
|
||||
|
||||
assertThat(service.getStatus().skipped()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void runImportAsync_skipsFile_whenMagicBytesCheckThrowsIOException(@TempDir Path tempDir) throws Exception {
|
||||
Files.writeString(tempDir.resolve("unreadable.pdf"), "some content");
|
||||
buildMinimalImportXlsx(tempDir, "unreadable.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
lenient().when(documentService.findByOriginalFilename(any())).thenReturn(Optional.empty());
|
||||
|
||||
MassImportService spyService = spy(service);
|
||||
doThrow(new java.io.IOException("simulated read error")).when(spyService).openFileStream(any(File.class));
|
||||
|
||||
spyService.runImportAsync();
|
||||
|
||||
assertThat(spyService.getStatus().skipped()).isEqualTo(1);
|
||||
assertThat(spyService.getStatus().skippedFiles())
|
||||
.extracting(MassImportService.SkippedFile::reason)
|
||||
.containsExactly(MassImportService.SkipReason.FILE_READ_ERROR);
|
||||
}
|
||||
|
||||
// ─── findFileRecursive — symlink escape security regression — do not remove ─
|
||||
|
||||
@Test
|
||||
void findFileRecursive_throwsDomainException_whenSymlinkEscapesImportDir(
|
||||
@TempDir Path importDirPath, @TempDir Path outsideDir) throws Exception {
|
||||
Path outsideFile = outsideDir.resolve("secret.pdf");
|
||||
Files.writeString(outsideFile, "sensitive content");
|
||||
Files.createSymbolicLink(importDirPath.resolve("secret.pdf"), outsideFile);
|
||||
|
||||
ReflectionTestUtils.setField(service, "importDir", importDirPath.toString());
|
||||
|
||||
assertThatThrownBy(() -> ReflectionTestUtils.invokeMethod(service, "findFileRecursive", "secret.pdf"))
|
||||
.isInstanceOf(DomainException.class);
|
||||
}
|
||||
|
||||
// ─── readOds — XXE security regression ───────────────────────────────────
|
||||
|
||||
// Security regression — do not remove.
|
||||
@Test
|
||||
void readOds_rejects_xxe_doctype_payload(@TempDir Path tempDir) throws Exception {
|
||||
File malicious = buildXxeOds(tempDir, "file:///etc/hostname");
|
||||
assertThatThrownBy(() -> service.readOds(malicious))
|
||||
.isInstanceOf(SAXParseException.class)
|
||||
.hasMessageContaining("DOCTYPE is disallowed");
|
||||
}
|
||||
|
||||
@Test
|
||||
void readOds_parses_valid_ods_correctly(@TempDir Path tempDir) throws Exception {
|
||||
File valid = buildValidOds(tempDir, "Mustermann");
|
||||
List<List<String>> rows = service.readOds(valid);
|
||||
assertThat(rows).isNotEmpty();
|
||||
assertThat(rows.get(0)).contains("Mustermann");
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Builds a minimal 14-element cell row with the given filename at index 0
|
||||
* and blanks for all optional fields.
|
||||
*/
|
||||
private List<String> minimalCells(String filename) {
|
||||
return buildCells(filename, "", "", "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a cell row with sender, receiver, and tag controls.
|
||||
* Layout matches the default column indices set in setUp().
|
||||
*/
|
||||
private List<String> buildCells(String filename, String sender, String receivers, String tag) {
|
||||
// 14 elements: index=0,box=1,folder=2,sender=3,[4],receivers=5,[6],date=7,[8],location=9,tag=10,summary=11,[12],transcription=13
|
||||
return List.of(
|
||||
filename, // 0: index
|
||||
"", // 1: box
|
||||
"", // 2: folder
|
||||
sender, // 3: sender
|
||||
"", // 4: (unused)
|
||||
receivers, // 5: receivers
|
||||
"", // 6: (unused)
|
||||
"", // 7: date
|
||||
"", // 8: (unused)
|
||||
"", // 9: location
|
||||
tag, // 10: tags
|
||||
"", // 11: summary
|
||||
"", // 12: (unused)
|
||||
"" // 13: transcription
|
||||
);
|
||||
}
|
||||
|
||||
/** Creates a minimal ODS ZIP containing a content.xml with an XXE payload. */
|
||||
private File buildXxeOds(Path dir, String entityTarget) throws Exception {
|
||||
String xml = "<?xml version=\"1.0\"?>"
|
||||
+ "<!DOCTYPE foo [<!ENTITY xxe SYSTEM \"" + entityTarget + "\">]>"
|
||||
+ "<office:document-content"
|
||||
+ " xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
|
||||
+ " xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
|
||||
+ " xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\">"
|
||||
+ "<office:body><office:spreadsheet>"
|
||||
+ "<table:table><table:table-row><table:table-cell>"
|
||||
+ "<text:p>&xxe;</text:p>"
|
||||
+ "</table:table-cell></table:table-row></table:table>"
|
||||
+ "</office:spreadsheet></office:body>"
|
||||
+ "</office:document-content>";
|
||||
return writeOdsZip(dir.resolve("malicious.ods"), xml);
|
||||
}
|
||||
|
||||
/** Creates a minimal valid ODS ZIP containing a content.xml with the given cell value.
|
||||
* cellValue must not contain XML metacharacters ({@code < > &}). */
|
||||
private File buildValidOds(Path dir, String cellValue) throws Exception {
|
||||
String xml = "<?xml version=\"1.0\"?>"
|
||||
+ "<office:document-content"
|
||||
+ " xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\""
|
||||
+ " xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\""
|
||||
+ " xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\">"
|
||||
+ "<office:body><office:spreadsheet>"
|
||||
+ "<table:table><table:table-row><table:table-cell>"
|
||||
+ "<text:p>" + cellValue + "</text:p>"
|
||||
+ "</table:table-cell></table:table-row></table:table>"
|
||||
+ "</office:spreadsheet></office:body>"
|
||||
+ "</office:document-content>";
|
||||
return writeOdsZip(dir.resolve("valid.ods"), xml);
|
||||
}
|
||||
|
||||
private File writeOdsZip(Path destination, String contentXml) throws Exception {
|
||||
try (OutputStream fos = Files.newOutputStream(destination);
|
||||
ZipOutputStream zip = new ZipOutputStream(fos)) {
|
||||
zip.putNextEntry(new ZipEntry("content.xml"));
|
||||
zip.write(contentXml.getBytes(StandardCharsets.UTF_8));
|
||||
zip.closeEntry();
|
||||
}
|
||||
return destination.toFile();
|
||||
}
|
||||
|
||||
private void setupOneValidOneFakeImport(Path tempDir) throws Exception {
|
||||
byte[] pdfHeader = {0x25, 0x50, 0x44, 0x46, 0x2D}; // %PDF-
|
||||
Files.write(tempDir.resolve("real.pdf"), pdfHeader);
|
||||
Files.writeString(tempDir.resolve("fake.pdf"), "not a pdf");
|
||||
buildMinimalImportXlsx(tempDir, "real.pdf", "fake.pdf");
|
||||
ReflectionTestUtils.setField(service, "importDir", tempDir.toString());
|
||||
when(documentService.findByOriginalFilename(any())).thenReturn(Optional.empty());
|
||||
when(documentService.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
}
|
||||
|
||||
private void buildMinimalImportXlsx(Path dir, String... filenames) throws Exception {
|
||||
Path xlsx = dir.resolve("import.xlsx");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
org.apache.poi.ss.usermodel.Sheet sheet = wb.createSheet("Sheet1");
|
||||
sheet.createRow(0).createCell(0).setCellValue("Index");
|
||||
for (int i = 0; i < filenames.length; i++) {
|
||||
sheet.createRow(i + 1).createCell(0).setCellValue(filenames[i]);
|
||||
}
|
||||
try (OutputStream out = Files.newOutputStream(xlsx)) {
|
||||
wb.write(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.raddatz.familienarchiv.person.PersonUpsertCommand;
|
||||
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class PersonRegisterImporterTest {
|
||||
|
||||
@Test
|
||||
void load_upsertsPersonBySourceRef_withProvisionalFalse(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
when(personService.upsertBySourceRef(any())).thenAnswer(inv -> personOf(inv.getArgument(0)));
|
||||
Path xlsx = writePersons(tempDir, row(
|
||||
"allemeyer-elsgard", "Allemeyer", "Elsgard", "Wöhler", "Nichte von Herbert", "False"));
|
||||
|
||||
new PersonRegisterImporter(personService).load(xlsx.toFile());
|
||||
|
||||
ArgumentCaptor<PersonUpsertCommand> captor = ArgumentCaptor.forClass(PersonUpsertCommand.class);
|
||||
verify(personService).upsertBySourceRef(captor.capture());
|
||||
PersonUpsertCommand cmd = captor.getValue();
|
||||
assertThat(cmd.sourceRef()).isEqualTo("allemeyer-elsgard");
|
||||
assertThat(cmd.lastName()).isEqualTo("Allemeyer");
|
||||
assertThat(cmd.firstName()).isEqualTo("Elsgard");
|
||||
assertThat(cmd.maidenName()).isEqualTo("Wöhler");
|
||||
assertThat(cmd.notes()).isEqualTo("Nichte von Herbert");
|
||||
assertThat(cmd.provisional()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_parsesCapitalisedPythonBool_True(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
when(personService.upsertBySourceRef(any())).thenAnswer(inv -> personOf(inv.getArgument(0)));
|
||||
Path xlsx = writePersons(tempDir, row(
|
||||
"noise-geschirr", "Geschirr", "", "", "", "True"));
|
||||
|
||||
new PersonRegisterImporter(personService).load(xlsx.toFile());
|
||||
|
||||
ArgumentCaptor<PersonUpsertCommand> captor = ArgumentCaptor.forClass(PersonUpsertCommand.class);
|
||||
verify(personService).upsertBySourceRef(captor.capture());
|
||||
assertThat(captor.getValue().provisional()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_skipsRowWithBlankPersonId(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
Path xlsx = writePersons(tempDir, row("", "NoId", "", "", "", "False"));
|
||||
|
||||
new PersonRegisterImporter(personService).load(xlsx.toFile());
|
||||
|
||||
verify(personService, times(0)).upsertBySourceRef(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_returnsCountOfProcessedRows(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
when(personService.upsertBySourceRef(any())).thenAnswer(inv -> personOf(inv.getArgument(0)));
|
||||
Path xlsx = writePersons(tempDir,
|
||||
row("a-one", "One", "A", "", "", "False"),
|
||||
row("a-two", "Two", "B", "", "", "False"));
|
||||
|
||||
int processed = new PersonRegisterImporter(personService).load(xlsx.toFile());
|
||||
|
||||
assertThat(processed).isEqualTo(2);
|
||||
}
|
||||
|
||||
private static Person personOf(PersonUpsertCommand cmd) {
|
||||
return Person.builder().id(UUID.randomUUID()).sourceRef(cmd.sourceRef())
|
||||
.firstName(cmd.firstName()).lastName(cmd.lastName())
|
||||
.provisional(cmd.provisional()).build();
|
||||
}
|
||||
|
||||
private Map<String, String> row(String personId, String lastName, String firstName,
|
||||
String maidenName, String notes, String provisional) {
|
||||
Map<String, String> r = new LinkedHashMap<>();
|
||||
r.put("person_id", personId);
|
||||
r.put("last_name", lastName);
|
||||
r.put("first_name", firstName);
|
||||
r.put("maiden_name", maidenName);
|
||||
r.put("notes", notes);
|
||||
r.put("provisional", provisional);
|
||||
return r;
|
||||
}
|
||||
|
||||
@SafeVarargs
|
||||
private Path writePersons(Path dir, Map<String, String>... rows) throws Exception {
|
||||
Path xlsx = dir.resolve("canonical-persons.xlsx");
|
||||
List<String> headers = List.of("person_id", "last_name", "first_name", "maiden_name", "notes", "provisional");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
for (int i = 0; i < headers.size(); i++) {
|
||||
header.createCell(i).setCellValue(headers.get(i));
|
||||
}
|
||||
for (int r = 0; r < rows.length; r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
for (int c = 0; c < headers.size(); c++) {
|
||||
row.createCell(c).setCellValue(rows[r].getOrDefault(headers.get(c), ""));
|
||||
}
|
||||
}
|
||||
try (OutputStream out = Files.newOutputStream(xlsx)) {
|
||||
wb.write(out);
|
||||
}
|
||||
}
|
||||
return xlsx;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.raddatz.familienarchiv.person.PersonUpsertCommand;
|
||||
import org.raddatz.familienarchiv.person.relationship.RelationType;
|
||||
import org.raddatz.familienarchiv.person.relationship.RelationshipService;
|
||||
import org.raddatz.familienarchiv.person.relationship.dto.CreateRelationshipRequest;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class PersonTreeImporterTest {
|
||||
|
||||
@Test
|
||||
void load_upsertsTreePersonBySourceRef_withFamilyMemberFlag(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
RelationshipService relationshipService = mock(RelationshipService.class);
|
||||
when(personService.upsertBySourceRef(any())).thenAnswer(inv -> personOf(inv.getArgument(0)));
|
||||
Path json = write(tempDir, """
|
||||
{"persons":[
|
||||
{"rowId":"row_002","firstName":"Elsgard","lastName":"Allemeyer","maidenName":"Wöhler",
|
||||
"notes":"Nichte","birthYear":1920,"deathYear":1999,"familyMember":true,"personId":"allemeyer-elsgard"}
|
||||
],"relationships":[]}
|
||||
""");
|
||||
|
||||
new PersonTreeImporter(personService, relationshipService)
|
||||
.load(json.toFile());
|
||||
|
||||
ArgumentCaptor<PersonUpsertCommand> captor = ArgumentCaptor.forClass(PersonUpsertCommand.class);
|
||||
verify(personService).upsertBySourceRef(captor.capture());
|
||||
PersonUpsertCommand cmd = captor.getValue();
|
||||
assertThat(cmd.sourceRef()).isEqualTo("allemeyer-elsgard");
|
||||
assertThat(cmd.familyMember()).isTrue();
|
||||
assertThat(cmd.provisional()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_createsRelationship_resolvingRowIdsToUpsertedPersons(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
RelationshipService relationshipService = mock(RelationshipService.class);
|
||||
UUID idA = UUID.randomUUID();
|
||||
UUID idB = UUID.randomUUID();
|
||||
when(personService.upsertBySourceRef(any())).thenAnswer(inv -> {
|
||||
PersonUpsertCommand c = inv.getArgument(0);
|
||||
return Person.builder().id(c.sourceRef().equals("a") ? idA : idB)
|
||||
.sourceRef(c.sourceRef()).lastName(c.lastName()).build();
|
||||
});
|
||||
Path json = write(tempDir, """
|
||||
{"persons":[
|
||||
{"rowId":"row_a","lastName":"A","familyMember":true,"personId":"a"},
|
||||
{"rowId":"row_b","lastName":"B","familyMember":true,"personId":"b"}
|
||||
],"relationships":[
|
||||
{"personId":"row_a","relatedPersonId":"row_b","type":"SPOUSE_OF","source":"verheiratet_mit"}
|
||||
]}
|
||||
""");
|
||||
|
||||
new PersonTreeImporter(personService, relationshipService)
|
||||
.load(json.toFile());
|
||||
|
||||
ArgumentCaptor<CreateRelationshipRequest> captor = ArgumentCaptor.forClass(CreateRelationshipRequest.class);
|
||||
verify(relationshipService).addRelationship(eq(idA), captor.capture());
|
||||
assertThat(captor.getValue().relatedPersonId()).isEqualTo(idB);
|
||||
assertThat(captor.getValue().relationType()).isEqualTo(RelationType.SPOUSE_OF);
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_swallowsDuplicateRelationship_forIdempotentReimport(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
RelationshipService relationshipService = mock(RelationshipService.class);
|
||||
when(personService.upsertBySourceRef(any()))
|
||||
.thenAnswer(inv -> personOf(inv.getArgument(0)));
|
||||
doThrow(DomainException.conflict(ErrorCode.DUPLICATE_RELATIONSHIP, "exists"))
|
||||
.when(relationshipService).addRelationship(any(), any());
|
||||
Path json = write(tempDir, """
|
||||
{"persons":[
|
||||
{"rowId":"row_a","lastName":"A","familyMember":true,"personId":"a"},
|
||||
{"rowId":"row_b","lastName":"B","familyMember":true,"personId":"b"}
|
||||
],"relationships":[
|
||||
{"personId":"row_a","relatedPersonId":"row_b","type":"SPOUSE_OF","source":"verheiratet_mit"}
|
||||
]}
|
||||
""");
|
||||
|
||||
PersonTreeImporter importer = new PersonTreeImporter(personService, relationshipService);
|
||||
|
||||
// Must not propagate the conflict — re-import is idempotent.
|
||||
importer.load(json.toFile());
|
||||
|
||||
verify(relationshipService).addRelationship(any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_propagatesUnexpectedDomainException_fromAddRelationship(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
RelationshipService relationshipService = mock(RelationshipService.class);
|
||||
when(personService.upsertBySourceRef(any()))
|
||||
.thenAnswer(inv -> personOf(inv.getArgument(0)));
|
||||
// An unexpected ErrorCode (not DUPLICATE/CIRCULAR) must NOT be swallowed.
|
||||
doThrow(DomainException.internal(ErrorCode.INTERNAL_ERROR, "boom"))
|
||||
.when(relationshipService).addRelationship(any(), any());
|
||||
Path json = write(tempDir, """
|
||||
{"persons":[
|
||||
{"rowId":"row_a","lastName":"A","familyMember":true,"personId":"a"},
|
||||
{"rowId":"row_b","lastName":"B","familyMember":true,"personId":"b"}
|
||||
],"relationships":[
|
||||
{"personId":"row_a","relatedPersonId":"row_b","type":"SPOUSE_OF","source":"verheiratet_mit"}
|
||||
]}
|
||||
""");
|
||||
|
||||
PersonTreeImporter importer = new PersonTreeImporter(personService, relationshipService);
|
||||
|
||||
assertThatThrownBy(() -> importer.load(json.toFile()))
|
||||
.isInstanceOf(DomainException.class)
|
||||
.extracting("code").isEqualTo(ErrorCode.INTERNAL_ERROR);
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_skipsRelationship_whenRowIdUnresolved(@TempDir Path tempDir) throws Exception {
|
||||
PersonService personService = mock(PersonService.class);
|
||||
RelationshipService relationshipService = mock(RelationshipService.class);
|
||||
when(personService.upsertBySourceRef(any())).thenAnswer(inv -> personOf(inv.getArgument(0)));
|
||||
Path json = write(tempDir, """
|
||||
{"persons":[
|
||||
{"rowId":"row_a","lastName":"A","familyMember":true,"personId":"a"}
|
||||
],"relationships":[
|
||||
{"personId":"row_a","relatedPersonId":"row_ghost","type":"SPOUSE_OF","source":"x"}
|
||||
]}
|
||||
""");
|
||||
|
||||
new PersonTreeImporter(personService, relationshipService)
|
||||
.load(json.toFile());
|
||||
|
||||
verify(relationshipService, org.mockito.Mockito.never()).addRelationship(any(), any());
|
||||
}
|
||||
|
||||
private static Person personOf(PersonUpsertCommand cmd) {
|
||||
return Person.builder().id(UUID.randomUUID()).sourceRef(cmd.sourceRef()).lastName(cmd.lastName()).build();
|
||||
}
|
||||
|
||||
private Path write(Path dir, String json) throws Exception {
|
||||
Path file = dir.resolve("canonical-persons-tree.json");
|
||||
Files.writeString(file, json);
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
import org.raddatz.familienarchiv.tag.TagService;
|
||||
|
||||
import java.io.OutputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TagTreeImporterTest {
|
||||
|
||||
@Test
|
||||
void load_upsertsRootTagWithNullParent(@TempDir Path tempDir) throws Exception {
|
||||
TagService tagService = mock(TagService.class);
|
||||
when(tagService.upsertBySourceRef(any(), any(), any()))
|
||||
.thenAnswer(inv -> tagOf(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2)));
|
||||
Path xlsx = writeTagTree(tempDir, List.<String[]>of(
|
||||
new String[]{"Themen", "", "Themen"}));
|
||||
|
||||
new TagTreeImporter(tagService).load(xlsx.toFile());
|
||||
|
||||
verify(tagService).upsertBySourceRef("Themen", "Themen", null);
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_resolvesParentByPath_forChildTag(@TempDir Path tempDir) throws Exception {
|
||||
TagService tagService = mock(TagService.class);
|
||||
UUID rootId = UUID.randomUUID();
|
||||
when(tagService.upsertBySourceRef(eq("Themen"), eq("Themen"), isNull()))
|
||||
.thenReturn(tagOf("Themen", "Themen", null, rootId));
|
||||
when(tagService.upsertBySourceRef(eq("Themen/Brautbriefe"), eq("Brautbriefe"), eq(rootId)))
|
||||
.thenReturn(tagOf("Themen/Brautbriefe", "Brautbriefe", rootId));
|
||||
Path xlsx = writeTagTree(tempDir, List.<String[]>of(
|
||||
new String[]{"Themen", "", "Themen"},
|
||||
new String[]{"Themen/Brautbriefe", "Themen", "Brautbriefe"}));
|
||||
|
||||
new TagTreeImporter(tagService).load(xlsx.toFile());
|
||||
|
||||
verify(tagService).upsertBySourceRef("Themen/Brautbriefe", "Brautbriefe", rootId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void load_returnsCountOfProcessedRows(@TempDir Path tempDir) throws Exception {
|
||||
TagService tagService = mock(TagService.class);
|
||||
when(tagService.upsertBySourceRef(any(), any(), any()))
|
||||
.thenAnswer(inv -> tagOf(inv.getArgument(0), inv.getArgument(1), inv.getArgument(2)));
|
||||
Path xlsx = writeTagTree(tempDir, List.<String[]>of(
|
||||
new String[]{"Themen", "", "Themen"},
|
||||
new String[]{"Themen/Brautbriefe", "Themen", "Brautbriefe"}));
|
||||
|
||||
int processed = new TagTreeImporter(tagService).load(xlsx.toFile());
|
||||
|
||||
assertThat(processed).isEqualTo(2);
|
||||
}
|
||||
|
||||
private static Tag tagOf(String sourceRef, String name, UUID parentId) {
|
||||
return tagOf(sourceRef, name, parentId, UUID.randomUUID());
|
||||
}
|
||||
|
||||
private static Tag tagOf(String sourceRef, String name, UUID parentId, UUID id) {
|
||||
return Tag.builder().id(id).sourceRef(sourceRef).name(name).parentId(parentId).build();
|
||||
}
|
||||
|
||||
private Path writeTagTree(Path dir, List<String[]> rows) throws Exception {
|
||||
Path xlsx = dir.resolve("canonical-tag-tree.xlsx");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
Sheet sheet = wb.createSheet("Sheet1");
|
||||
Row header = sheet.createRow(0);
|
||||
header.createCell(0).setCellValue("tag_path");
|
||||
header.createCell(1).setCellValue("parent_name");
|
||||
header.createCell(2).setCellValue("tag_name");
|
||||
for (int r = 0; r < rows.size(); r++) {
|
||||
Row row = sheet.createRow(r + 1);
|
||||
String[] values = rows.get(r);
|
||||
for (int c = 0; c < values.length; c++) {
|
||||
row.createCell(c).setCellValue(values[c]);
|
||||
}
|
||||
}
|
||||
try (OutputStream out = Files.newOutputStream(xlsx)) {
|
||||
wb.write(out);
|
||||
}
|
||||
}
|
||||
return xlsx;
|
||||
}
|
||||
}
|
||||
@@ -65,44 +65,144 @@ class PersonControllerTest {
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_returns200_withEmptyList() throws Exception {
|
||||
when(personService.findAll(null)).thenReturn(Collections.emptyList());
|
||||
void getPersons_returns200_withEmptyPagedResult() throws Exception {
|
||||
when(personService.search(any(), eq(0), eq(50), eq(null)))
|
||||
.thenReturn(PersonSearchResult.paged(Collections.emptyList(), 0, 50, 0));
|
||||
mockMvc.perform(get("/api/persons"))
|
||||
.andExpect(status().isOk());
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.items").isArray())
|
||||
.andExpect(jsonPath("$.totalElements").value(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_delegatesQueryParam_toService() throws Exception {
|
||||
PersonSummaryDTO dto = mockPersonSummary("Hans", "Müller");
|
||||
when(personService.findAll("Hans")).thenReturn(List.of(dto));
|
||||
when(personService.search(any(), eq(0), eq(50), eq("Hans")))
|
||||
.thenReturn(PersonSearchResult.paged(List.of(dto), 0, 50, 1));
|
||||
|
||||
mockMvc.perform(get("/api/persons").param("q", "Hans"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$[0].firstName").value("Hans"));
|
||||
.andExpect(jsonPath("$.items[0].firstName").value("Hans"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_delegatesTopByDocumentCount_whenSortAndSizeGiven() throws Exception {
|
||||
void getPersons_passesFilterParams_toService() throws Exception {
|
||||
ArgumentCaptor<PersonFilter> filterCaptor = ArgumentCaptor.forClass(PersonFilter.class);
|
||||
when(personService.search(filterCaptor.capture(), eq(0), eq(50), eq(null)))
|
||||
.thenReturn(PersonSearchResult.paged(Collections.emptyList(), 0, 50, 0));
|
||||
|
||||
mockMvc.perform(get("/api/persons")
|
||||
.param("type", "INSTITUTION")
|
||||
.param("familyOnly", "true")
|
||||
.param("hasDocuments", "true")
|
||||
.param("provisional", "false"))
|
||||
.andExpect(status().isOk());
|
||||
|
||||
PersonFilter captured = filterCaptor.getValue();
|
||||
assertThat(captured.type()).isEqualTo(PersonType.INSTITUTION);
|
||||
assertThat(captured.familyOnly()).isTrue();
|
||||
assertThat(captured.hasDocuments()).isTrue();
|
||||
assertThat(captured.provisional()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_defaultsToReaderDefault_whenNoReviewFlag() throws Exception {
|
||||
ArgumentCaptor<PersonFilter> filterCaptor = ArgumentCaptor.forClass(PersonFilter.class);
|
||||
when(personService.search(filterCaptor.capture(), eq(0), eq(50), eq(null)))
|
||||
.thenReturn(PersonSearchResult.paged(Collections.emptyList(), 0, 50, 0));
|
||||
|
||||
mockMvc.perform(get("/api/persons")).andExpect(status().isOk());
|
||||
|
||||
assertThat(filterCaptor.getValue().readerDefault()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_dropsReaderDefault_whenReviewFlagSet() throws Exception {
|
||||
ArgumentCaptor<PersonFilter> filterCaptor = ArgumentCaptor.forClass(PersonFilter.class);
|
||||
when(personService.search(filterCaptor.capture(), eq(0), eq(50), eq(null)))
|
||||
.thenReturn(PersonSearchResult.paged(Collections.emptyList(), 0, 50, 0));
|
||||
|
||||
mockMvc.perform(get("/api/persons").param("review", "true")).andExpect(status().isOk());
|
||||
|
||||
assertThat(filterCaptor.getValue().readerDefault()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_passesPageAndSize_toService() throws Exception {
|
||||
when(personService.search(any(), eq(2), eq(25), eq(null)))
|
||||
.thenReturn(PersonSearchResult.paged(Collections.emptyList(), 2, 25, 0));
|
||||
|
||||
mockMvc.perform(get("/api/persons").param("page", "2").param("size", "25"))
|
||||
.andExpect(status().isOk());
|
||||
|
||||
verify(personService).search(any(), eq(2), eq(25), eq(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_returns400_whenSizeIsZero() throws Exception {
|
||||
mockMvc.perform(get("/api/persons").param("size", "0"))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_returns400_whenSizeExceeds100() throws Exception {
|
||||
mockMvc.perform(get("/api/persons").param("size", "101"))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_returns400_whenPageIsNegative() throws Exception {
|
||||
mockMvc.perform(get("/api/persons").param("page", "-1"))
|
||||
.andExpect(status().isBadRequest());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_delegatesTopByDocumentCount_whenSortGiven() throws Exception {
|
||||
PersonSummaryDTO top = mockPersonSummary("Käthe", "Raddatz");
|
||||
when(personService.findTopByDocumentCount(4)).thenReturn(List.of(top));
|
||||
|
||||
mockMvc.perform(get("/api/persons").param("sort", "documentCount").param("size", "4"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$[0].firstName").value("Käthe"));
|
||||
.andExpect(jsonPath("$.items[0].firstName").value("Käthe"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_capsTopByDocumentCount_atFifty() throws Exception {
|
||||
ArgumentCaptor<Integer> sizeCaptor = ArgumentCaptor.forClass(Integer.class);
|
||||
when(personService.findTopByDocumentCount(sizeCaptor.capture())).thenReturn(Collections.emptyList());
|
||||
void getPersons_topByDocumentCount_isNonPaged_totalElementsEqualsReturnedCount() throws Exception {
|
||||
// The top-N dashboard path is deliberately NON-paged: it returns the complete result
|
||||
// (no further page exists), so totalElements equals the number of rows returned and
|
||||
// totalPages is 1. Pinned so nobody "fixes" it into a misleading paged total.
|
||||
when(personService.findTopByDocumentCount(50))
|
||||
.thenReturn(List.of(mockPersonSummary("Käthe", "Raddatz"),
|
||||
mockPersonSummary("Hans", "Müller")));
|
||||
|
||||
mockMvc.perform(get("/api/persons").param("sort", "documentCount").param("size", "999"))
|
||||
.andExpect(status().isOk());
|
||||
mockMvc.perform(get("/api/persons").param("sort", "documentCount"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.items.length()").value(2))
|
||||
.andExpect(jsonPath("$.totalElements").value(2))
|
||||
.andExpect(jsonPath("$.pageNumber").value(0))
|
||||
.andExpect(jsonPath("$.pageSize").value(2))
|
||||
.andExpect(jsonPath("$.totalPages").value(1));
|
||||
}
|
||||
|
||||
assertThat(sizeCaptor.getValue()).isEqualTo(50);
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void getPersons_topByDocumentCount_emptyResult_reportsZeroPages() throws Exception {
|
||||
when(personService.findTopByDocumentCount(50)).thenReturn(Collections.emptyList());
|
||||
|
||||
mockMvc.perform(get("/api/persons").param("sort", "documentCount"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.totalElements").value(0))
|
||||
.andExpect(jsonPath("$.totalPages").value(0));
|
||||
}
|
||||
|
||||
private PersonSummaryDTO mockPersonSummary(String firstName, String lastName) {
|
||||
@@ -117,6 +217,7 @@ class PersonControllerTest {
|
||||
public Integer getDeathYear() { return null; }
|
||||
public String getNotes() { return null; }
|
||||
public boolean isFamilyMember() { return false; }
|
||||
public boolean isProvisional() { return false; }
|
||||
public long getDocumentCount() { return 0; }
|
||||
};
|
||||
}
|
||||
@@ -397,6 +498,61 @@ class PersonControllerTest {
|
||||
.andExpect(status().isNoContent());
|
||||
}
|
||||
|
||||
// ─── PATCH /api/persons/{id}/confirm ──────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void confirmPerson_returns401_whenUnauthenticated() throws Exception {
|
||||
mockMvc.perform(patch("/api/persons/{id}/confirm", UUID.randomUUID()).with(csrf()))
|
||||
.andExpect(status().isUnauthorized());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void confirmPerson_returns403_whenUserHasOnlyReadPermission() throws Exception {
|
||||
mockMvc.perform(patch("/api/persons/{id}/confirm", UUID.randomUUID()).with(csrf()))
|
||||
.andExpect(status().isForbidden());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "WRITE_ALL")
|
||||
void confirmPerson_returns200_andClearsProvisional() throws Exception {
|
||||
UUID id = UUID.randomUUID();
|
||||
Person confirmed = Person.builder().id(id).firstName("Bald").lastName("Bestaetigt").provisional(false).build();
|
||||
when(personService.confirmPerson(id)).thenReturn(confirmed);
|
||||
|
||||
mockMvc.perform(patch("/api/persons/{id}/confirm", id).with(csrf()))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.provisional").value(false));
|
||||
|
||||
verify(personService).confirmPerson(id);
|
||||
}
|
||||
|
||||
// ─── DELETE /api/persons/{id} ──────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void deletePerson_returns401_whenUnauthenticated() throws Exception {
|
||||
mockMvc.perform(delete("/api/persons/{id}", UUID.randomUUID()).with(csrf()))
|
||||
.andExpect(status().isUnauthorized());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "READ_ALL")
|
||||
void deletePerson_returns403_whenUserHasOnlyReadPermission() throws Exception {
|
||||
mockMvc.perform(delete("/api/persons/{id}", UUID.randomUUID()).with(csrf()))
|
||||
.andExpect(status().isForbidden());
|
||||
}
|
||||
|
||||
@Test
|
||||
@WithMockUser(authorities = "WRITE_ALL")
|
||||
void deletePerson_returns204_whenValid() throws Exception {
|
||||
UUID id = UUID.randomUUID();
|
||||
|
||||
mockMvc.perform(delete("/api/persons/{id}", id).with(csrf()))
|
||||
.andExpect(status().isNoContent());
|
||||
|
||||
verify(personService).deletePerson(id);
|
||||
}
|
||||
|
||||
// ─── PUT /api/persons/{id} — lastName blank branch ────────────────────────
|
||||
|
||||
@Test
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
package org.raddatz.familienarchiv.person;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class PersonImportUpsertTest {
|
||||
|
||||
@Mock PersonRepository personRepository;
|
||||
@Mock PersonNameAliasRepository aliasRepository;
|
||||
@InjectMocks PersonService personService;
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_insertsNewPerson_whenSourceRefUnknown() {
|
||||
when(personRepository.findBySourceRef("clara-cram")).thenReturn(Optional.empty());
|
||||
when(personRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
PersonUpsertCommand cmd = PersonUpsertCommand.builder()
|
||||
.sourceRef("clara-cram").firstName("Clara").lastName("Cram")
|
||||
.personType(PersonType.PERSON).provisional(false).build();
|
||||
|
||||
Person result = personService.upsertBySourceRef(cmd);
|
||||
|
||||
assertThat(result.getSourceRef()).isEqualTo("clara-cram");
|
||||
assertThat(result.getFirstName()).isEqualTo("Clara");
|
||||
assertThat(result.getLastName()).isEqualTo("Cram");
|
||||
assertThat(result.isProvisional()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_updatesInPlace_whenSourceRefExists() {
|
||||
Person existing = Person.builder()
|
||||
.id(UUID.randomUUID()).sourceRef("clara-cram")
|
||||
.firstName("Clara").lastName("Cram").build();
|
||||
when(personRepository.findBySourceRef("clara-cram")).thenReturn(Optional.of(existing));
|
||||
when(personRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
PersonUpsertCommand cmd = PersonUpsertCommand.builder()
|
||||
.sourceRef("clara-cram").firstName("Clara").lastName("Cram")
|
||||
.notes("Updated note").personType(PersonType.PERSON).provisional(false).build();
|
||||
|
||||
personService.upsertBySourceRef(cmd);
|
||||
|
||||
verify(personRepository).save(argThat(p -> p.getId().equals(existing.getId())));
|
||||
verify(personRepository, never()).save(argThat(p -> p.getId() == null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_preservesHumanEditedNonBlankFields() {
|
||||
// A human renamed the maiden-name register person and added notes in-app.
|
||||
Person humanEdited = Person.builder()
|
||||
.id(UUID.randomUUID()).sourceRef("clara-cram")
|
||||
.firstName("Klara").lastName("Cram-Müller").notes("Verified by Marcel").build();
|
||||
when(personRepository.findBySourceRef("clara-cram")).thenReturn(Optional.of(humanEdited));
|
||||
when(personRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
PersonUpsertCommand cmd = PersonUpsertCommand.builder()
|
||||
.sourceRef("clara-cram").firstName("Clara").lastName("Cram")
|
||||
.notes("Auto note").personType(PersonType.PERSON).provisional(false).build();
|
||||
|
||||
Person result = personService.upsertBySourceRef(cmd);
|
||||
|
||||
// Human edits survive the re-import.
|
||||
assertThat(result.getFirstName()).isEqualTo("Klara");
|
||||
assertThat(result.getLastName()).isEqualTo("Cram-Müller");
|
||||
assertThat(result.getNotes()).isEqualTo("Verified by Marcel");
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_fillsOnlyBlankFields_onReimport() {
|
||||
Person existing = Person.builder()
|
||||
.id(UUID.randomUUID()).sourceRef("clara-cram")
|
||||
.firstName("Clara").lastName("Cram").notes(null).build();
|
||||
when(personRepository.findBySourceRef("clara-cram")).thenReturn(Optional.of(existing));
|
||||
when(personRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
PersonUpsertCommand cmd = PersonUpsertCommand.builder()
|
||||
.sourceRef("clara-cram").firstName("Clara").lastName("Cram")
|
||||
.notes("Nichte von Herbert").personType(PersonType.PERSON).provisional(false).build();
|
||||
|
||||
Person result = personService.upsertBySourceRef(cmd);
|
||||
|
||||
// Blank field gets filled by canonical value.
|
||||
assertThat(result.getNotes()).isEqualTo("Nichte von Herbert");
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_fillsBlankYears_butPreservesHumanEditedYears_onReimport() {
|
||||
// Existing has a human-set birthYear and a blank deathYear.
|
||||
Person existing = Person.builder()
|
||||
.id(UUID.randomUUID()).sourceRef("clara-cram")
|
||||
.lastName("Cram").birthYear(1890).deathYear(null).build();
|
||||
when(personRepository.findBySourceRef("clara-cram")).thenReturn(Optional.of(existing));
|
||||
when(personRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
PersonUpsertCommand cmd = PersonUpsertCommand.builder()
|
||||
.sourceRef("clara-cram").lastName("Cram")
|
||||
.birthYear(1888).deathYear(1965)
|
||||
.personType(PersonType.PERSON).provisional(false).build();
|
||||
|
||||
Person result = personService.upsertBySourceRef(cmd);
|
||||
|
||||
assertThat(result.getBirthYear()).isEqualTo(1890); // human value kept
|
||||
assertThat(result.getDeathYear()).isEqualTo(1965); // blank filled from canonical
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_neverFlipsProvisionalBackToTrue_onceHumanConfirmed() {
|
||||
// A human confirmed this provisional importer-created person (provisional -> false).
|
||||
Person confirmed = Person.builder()
|
||||
.id(UUID.randomUUID()).sourceRef("schwester-hanni")
|
||||
.firstName(null).lastName("Schwester Hanni").provisional(false).build();
|
||||
when(personRepository.findBySourceRef("schwester-hanni")).thenReturn(Optional.of(confirmed));
|
||||
when(personRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
PersonUpsertCommand cmd = PersonUpsertCommand.builder()
|
||||
.sourceRef("schwester-hanni").lastName("Schwester Hanni")
|
||||
.personType(PersonType.PERSON).provisional(true).build();
|
||||
|
||||
Person result = personService.upsertBySourceRef(cmd);
|
||||
|
||||
assertThat(result.isProvisional()).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_setsProvisionalTrue_forNewProvisionalPerson() {
|
||||
when(personRepository.findBySourceRef("noise-geschirr")).thenReturn(Optional.empty());
|
||||
when(personRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
PersonUpsertCommand cmd = PersonUpsertCommand.builder()
|
||||
.sourceRef("noise-geschirr").lastName("Tante Tüten")
|
||||
.personType(PersonType.PERSON).provisional(true).build();
|
||||
|
||||
Person result = personService.upsertBySourceRef(cmd);
|
||||
|
||||
assertThat(result.isProvisional()).isTrue();
|
||||
}
|
||||
}
|
||||
@@ -463,4 +463,213 @@ class PersonRepositoryTest {
|
||||
assertThat(result).hasSize(1);
|
||||
assertThat(result.get(0).getLastName()).isEqualTo("Gesellschafter des Verlages");
|
||||
}
|
||||
|
||||
// ─── #671: provisional must be SELECTed in all three native projections ───
|
||||
// Adding isProvisional() to the interface compiles even if a native query forgets
|
||||
// to SELECT p.provisional — it then silently returns false. These tests are the only
|
||||
// guard against that trap, so they must run against real Postgres.
|
||||
|
||||
@Test
|
||||
void findAllWithDocumentCount_projectsProvisionalTrue() {
|
||||
personRepository.save(Person.builder()
|
||||
.firstName("Inferred").lastName("Person").provisional(true).build());
|
||||
|
||||
List<PersonSummaryDTO> result = personRepository.findAllWithDocumentCount();
|
||||
|
||||
assertThat(result).anyMatch(PersonSummaryDTO::isProvisional);
|
||||
}
|
||||
|
||||
@Test
|
||||
void searchWithDocumentCount_projectsProvisionalTrue() {
|
||||
personRepository.save(Person.builder()
|
||||
.firstName("Provisorisch").lastName("Müller").provisional(true).build());
|
||||
|
||||
List<PersonSummaryDTO> result = personRepository.searchWithDocumentCount("Provisorisch");
|
||||
|
||||
assertThat(result).hasSize(1);
|
||||
assertThat(result.get(0).isProvisional()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void findTopByDocumentCount_projectsProvisionalTrue() {
|
||||
Person provisional = personRepository.save(Person.builder()
|
||||
.firstName("Top").lastName("Provisional").provisional(true).build());
|
||||
documentRepository.save(Document.builder()
|
||||
.title("Brief").originalFilename("b.pdf")
|
||||
.status(DocumentStatus.UPLOADED)
|
||||
.sender(provisional).build());
|
||||
|
||||
List<PersonSummaryDTO> result = personRepository.findTopByDocumentCount(10);
|
||||
|
||||
PersonSummaryDTO summary = result.stream()
|
||||
.filter(p -> p.getId().equals(provisional.getId())).findFirst().orElseThrow();
|
||||
assertThat(summary.isProvisional()).isTrue();
|
||||
}
|
||||
|
||||
// ─── #667: filter-aware paged slice + paired COUNT (Postgres-only) ────────
|
||||
// The slice query (findByFilter) and the count query (countByFilter) MUST share one
|
||||
// WHERE clause so totalElements can never drift from the rendered page. These tests run
|
||||
// against real Postgres because the slice ORDER BY uses a computed alias that fails on H2.
|
||||
|
||||
private void seedDirectoryFixture() {
|
||||
// Register family member, no documents — visible by reader default (familyMember)
|
||||
personRepository.save(Person.builder().firstName("Karl").lastName("Register").familyMember(true).build());
|
||||
// Person with one document — visible by reader default (documentCount > 0)
|
||||
Person hasDoc = personRepository.save(Person.builder().firstName("Doku").lastName("Person").build());
|
||||
documentRepository.save(Document.builder().title("B").originalFilename("b.pdf")
|
||||
.status(DocumentStatus.UPLOADED).sender(hasDoc).build());
|
||||
// Provisional, zero-document, non-family — hidden by reader default
|
||||
personRepository.save(Person.builder().firstName("Unbe").lastName("Staetigt").provisional(true).build());
|
||||
// An institution with no documents, non-family, non-provisional
|
||||
personRepository.save(Person.builder().lastName("Verlag GmbH").personType(PersonType.INSTITUTION).build());
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_readerDefault_returnsOnlyFamilyOrWithDocuments() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, null, null, null, true, null, 50, 0);
|
||||
|
||||
assertThat(slice).extracting(PersonSummaryDTO::getLastName)
|
||||
.containsExactlyInAnyOrder("Register", "Person");
|
||||
}
|
||||
|
||||
@Test
|
||||
void countByFilter_readerDefault_matchesSliceSize() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
long count = personRepository.countByFilter(null, null, null, null, true, null);
|
||||
|
||||
assertThat(count).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_showAll_returnsEveryone() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, null, null, null, false, null, 50, 0);
|
||||
|
||||
assertThat(slice).hasSize(4);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_typeInstitution_returnsOnlyInstitutions() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
"INSTITUTION", null, null, null, false, null, 50, 0);
|
||||
|
||||
assertThat(slice).extracting(PersonSummaryDTO::getLastName).containsExactly("Verlag GmbH");
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_familyOnly_returnsOnlyFamilyMembers() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, true, null, null, false, null, 50, 0);
|
||||
|
||||
assertThat(slice).extracting(PersonSummaryDTO::getLastName).containsExactly("Register");
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_hasDocuments_returnsOnlyPersonsWithDocuments() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, null, true, null, false, null, 50, 0);
|
||||
|
||||
assertThat(slice).extracting(PersonSummaryDTO::getLastName).containsExactly("Person");
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_provisionalTrue_returnsOnlyProvisional() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, null, null, true, false, null, 50, 0);
|
||||
|
||||
assertThat(slice).extracting(PersonSummaryDTO::getLastName).containsExactly("Staetigt");
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_combinedFilters_andTogether() {
|
||||
seedDirectoryFixture();
|
||||
// family + has-documents → intersection is empty (Register has no docs, Doku is not family)
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, true, true, null, false, null, 50, 0);
|
||||
|
||||
assertThat(slice).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_query_combinesWithFilters() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, null, null, null, false, "Verlag", 50, 0);
|
||||
|
||||
assertThat(slice).extracting(PersonSummaryDTO::getLastName).containsExactly("Verlag GmbH");
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_pageBeyondRange_returnsEmptySlice() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, null, null, null, false, null, 50, 999 * 50);
|
||||
|
||||
assertThat(slice).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_respectsPageSize() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> firstPage = personRepository.findByFilter(
|
||||
null, null, null, null, false, null, 2, 0);
|
||||
List<PersonSummaryDTO> secondPage = personRepository.findByFilter(
|
||||
null, null, null, null, false, null, 2, 2);
|
||||
|
||||
assertThat(firstPage).hasSize(2);
|
||||
assertThat(secondPage).hasSize(2);
|
||||
assertThat(firstPage).extracting(PersonSummaryDTO::getId)
|
||||
.doesNotContainAnyElementsOf(secondPage.stream().map(PersonSummaryDTO::getId).toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void countByFilter_typeInstitution_matchesSlice() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
long count = personRepository.countByFilter("INSTITUTION", null, null, null, false, null);
|
||||
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countByFilter_query_matchesSliceSize() {
|
||||
// The whole point of the shared FILTER_WHERE is that the slice and the count can never
|
||||
// drift. Pin the query (LIKE) path explicitly: countByFilter must equal the slice size
|
||||
// so a future edit to one query's LIKE clause is caught.
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, null, null, null, false, "Verlag", 50, 0);
|
||||
long count = personRepository.countByFilter(null, null, null, null, false, "Verlag");
|
||||
|
||||
assertThat(count).isEqualTo(slice.size());
|
||||
assertThat(count).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findByFilter_projectsDocumentCount() {
|
||||
seedDirectoryFixture();
|
||||
|
||||
List<PersonSummaryDTO> slice = personRepository.findByFilter(
|
||||
null, null, true, null, false, null, 50, 0);
|
||||
|
||||
assertThat(slice.get(0).getDocumentCount()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@ package org.raddatz.familienarchiv.person;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
import org.raddatz.familienarchiv.document.DocumentRepository;
|
||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonType;
|
||||
import org.raddatz.familienarchiv.person.PersonRepository;
|
||||
@@ -13,6 +16,11 @@ import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
|
||||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.persistence.PersistenceContext;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE)
|
||||
@@ -24,6 +32,9 @@ class PersonServiceIntegrationTest {
|
||||
@MockitoBean S3Client s3Client;
|
||||
@Autowired PersonService personService;
|
||||
@Autowired PersonRepository personRepository;
|
||||
@Autowired DocumentRepository documentRepository;
|
||||
|
||||
@PersistenceContext EntityManager entityManager;
|
||||
|
||||
@Test
|
||||
void findOrCreateByAlias_skipReturnsNull_noRecordCreated() {
|
||||
@@ -63,4 +74,97 @@ class PersonServiceIntegrationTest {
|
||||
assertThat(result.getFirstName()).isEqualTo("Clara");
|
||||
assertThat(result.getLastName()).isEqualTo("Cram");
|
||||
}
|
||||
|
||||
// ─── #667: confirm round-trip + reader-default semantics ──────────────────
|
||||
|
||||
@Test
|
||||
void search_readerDefault_hidesProvisionalZeroDocumentPerson() {
|
||||
personRepository.save(Person.builder()
|
||||
.firstName("Unbe").lastName("Staetigt").provisional(true).build());
|
||||
|
||||
PersonSearchResult result = personService.search(PersonFilter.cleanDefault(), 0, 50, null);
|
||||
|
||||
assertThat(result.items()).noneMatch(p -> p.getLastName().equals("Staetigt"));
|
||||
assertThat(result.totalElements()).isEqualTo(result.items().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_showAll_includesProvisionalZeroDocumentPerson() {
|
||||
personRepository.save(Person.builder()
|
||||
.firstName("Unbe").lastName("Staetigt").provisional(true).build());
|
||||
|
||||
PersonSearchResult result = personService.search(PersonFilter.showAll(), 0, 50, null);
|
||||
|
||||
assertThat(result.items()).anyMatch(p -> p.getLastName().equals("Staetigt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void confirmPerson_clearsProvisional_andShowAllTreatsItAsConfirmed() {
|
||||
Person provisional = personRepository.save(Person.builder()
|
||||
.firstName("Bald").lastName("Bestaetigt").provisional(true).build());
|
||||
|
||||
personService.confirmPerson(provisional.getId());
|
||||
|
||||
Person reloaded = personRepository.findById(provisional.getId()).orElseThrow();
|
||||
assertThat(reloaded.isProvisional()).isFalse();
|
||||
|
||||
PersonSearchResult showAll = personService.search(PersonFilter.showAll(), 0, 50, null);
|
||||
assertThat(showAll.items())
|
||||
.filteredOn(p -> p.getId().equals(provisional.getId()))
|
||||
.allMatch(p -> !p.isProvisional());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletePerson_removesPerson() {
|
||||
Person target = personRepository.save(Person.builder()
|
||||
.firstName("Weg").lastName("Person").provisional(true).build());
|
||||
|
||||
personService.deletePerson(target.getId());
|
||||
|
||||
assertThat(personRepository.findById(target.getId())).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletePerson_detachesSentAndReceivedReferences_beforeDelete_noOrphan() {
|
||||
// A person referenced as BOTH a document sender and a document receiver must delete
|
||||
// cleanly: deletePerson nulls the sender_id FK and removes the receiver join row first
|
||||
// (reassignSenderToNull → deleteReceiverReferences → deleteById), so no FK orphan and
|
||||
// the documents themselves survive.
|
||||
Person target = personRepository.save(Person.builder()
|
||||
.firstName("Weg").lastName("Person").provisional(true).build());
|
||||
Person bystander = personRepository.save(Person.builder()
|
||||
.firstName("Bleibt").lastName("Hier").build());
|
||||
|
||||
Document sent = documentRepository.save(Document.builder()
|
||||
.title("Sent letter").originalFilename("sent.pdf")
|
||||
.status(DocumentStatus.UPLOADED).sender(target).build());
|
||||
Document received = documentRepository.save(Document.builder()
|
||||
.title("Received letter").originalFilename("received.pdf")
|
||||
.status(DocumentStatus.UPLOADED).sender(bystander)
|
||||
.receivers(new java.util.HashSet<>(Set.of(target))).build());
|
||||
|
||||
// Persist the fixture and detach everything so the native @Modifying deletes operate on
|
||||
// the database directly without the persistence context holding stale references that
|
||||
// would re-flush a now-deleted person as a transient association.
|
||||
entityManager.flush();
|
||||
entityManager.clear();
|
||||
|
||||
personService.deletePerson(target.getId());
|
||||
|
||||
// Native @Modifying queries bypass the persistence context — clear it so the asserting
|
||||
// reads observe the post-delete database state, not stale managed entities.
|
||||
entityManager.flush();
|
||||
entityManager.clear();
|
||||
|
||||
assertThat(personRepository.findById(target.getId())).isEmpty();
|
||||
|
||||
Document reloadedSent = documentRepository.findById(sent.getId()).orElseThrow();
|
||||
assertThat(reloadedSent.getSender()).isNull();
|
||||
|
||||
Document reloadedReceived = documentRepository.findById(received.getId()).orElseThrow();
|
||||
assertThat(reloadedReceived.getReceivers())
|
||||
.noneMatch(p -> p.getId().equals(target.getId()));
|
||||
// The other person and the documents themselves survive the delete.
|
||||
assertThat(personRepository.findById(bystander.getId())).isPresent();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,33 +58,109 @@ class PersonServiceTest {
|
||||
assertThat(personService.getById(id)).isEqualTo(person);
|
||||
}
|
||||
|
||||
// ─── findAll ─────────────────────────────────────────────────────────────
|
||||
// ─── #667: search (filter + pagination) ──────────────────────────────────
|
||||
|
||||
@Test
|
||||
void findAll_returnsAll_whenQueryIsNull() {
|
||||
List<PersonSummaryDTO> expected = List.of();
|
||||
when(personRepository.findAllWithDocumentCount()).thenReturn(expected);
|
||||
void search_returnsPagedResult_withTotalsFromCountQuery() {
|
||||
PersonFilter filter = PersonFilter.cleanDefault();
|
||||
when(personRepository.countByFilter(null, null, null, null, true, null)).thenReturn(120L);
|
||||
when(personRepository.findByFilter(null, null, null, null, true, null, 50, 0))
|
||||
.thenReturn(List.of());
|
||||
|
||||
assertThat(personService.findAll(null)).isEqualTo(expected);
|
||||
verify(personRepository).findAllWithDocumentCount();
|
||||
verify(personRepository, never()).searchWithDocumentCount(any());
|
||||
PersonSearchResult result = personService.search(filter, 0, 50, null);
|
||||
|
||||
assertThat(result.totalElements()).isEqualTo(120L);
|
||||
assertThat(result.pageNumber()).isEqualTo(0);
|
||||
assertThat(result.pageSize()).isEqualTo(50);
|
||||
assertThat(result.totalPages()).isEqualTo(3); // ceil(120 / 50)
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll_returnsEmpty_whenQueryIsWhitespaceOnly() {
|
||||
assertThat(personService.findAll(" ")).isEmpty();
|
||||
verify(personRepository, never()).findAllWithDocumentCount();
|
||||
verify(personRepository, never()).searchWithDocumentCount(any());
|
||||
void search_passesTypeAsEnumName_toRepository() {
|
||||
PersonFilter filter = PersonFilter.builder().type(PersonType.INSTITUTION).build();
|
||||
when(personRepository.countByFilter("INSTITUTION", null, null, null, false, null)).thenReturn(0L);
|
||||
when(personRepository.findByFilter("INSTITUTION", null, null, null, false, null, 50, 0))
|
||||
.thenReturn(List.of());
|
||||
|
||||
personService.search(filter, 0, 50, null);
|
||||
|
||||
verify(personRepository).findByFilter("INSTITUTION", null, null, null, false, null, 50, 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void findAll_searchesByName_whenQueryIsNonBlank() {
|
||||
List<PersonSummaryDTO> expected = List.of();
|
||||
when(personRepository.searchWithDocumentCount("Anna")).thenReturn(expected);
|
||||
void search_computesOffset_fromPageAndSize() {
|
||||
PersonFilter filter = PersonFilter.showAll();
|
||||
when(personRepository.countByFilter(null, null, null, null, false, null)).thenReturn(0L);
|
||||
when(personRepository.findByFilter(null, null, null, null, false, null, 20, 40))
|
||||
.thenReturn(List.of());
|
||||
|
||||
assertThat(personService.findAll("Anna")).isEqualTo(expected);
|
||||
verify(personRepository).searchWithDocumentCount("Anna");
|
||||
verify(personRepository, never()).findAllWithDocumentCount();
|
||||
personService.search(filter, 2, 20, null); // offset = page * size = 40
|
||||
|
||||
verify(personRepository).findByFilter(null, null, null, null, false, null, 20, 40);
|
||||
}
|
||||
|
||||
@Test
|
||||
void search_trimsBlankQueryToNull() {
|
||||
PersonFilter filter = PersonFilter.showAll();
|
||||
when(personRepository.countByFilter(null, null, null, null, false, null)).thenReturn(0L);
|
||||
when(personRepository.findByFilter(null, null, null, null, false, null, 50, 0))
|
||||
.thenReturn(List.of());
|
||||
|
||||
personService.search(filter, 0, 50, " ");
|
||||
|
||||
verify(personRepository).findByFilter(null, null, null, null, false, null, 50, 0);
|
||||
}
|
||||
|
||||
// ─── #667: confirmPerson ──────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void confirmPerson_clearsProvisionalFlag() {
|
||||
UUID id = UUID.randomUUID();
|
||||
Person provisional = Person.builder().id(id).firstName("Inferred").lastName("Person").provisional(true).build();
|
||||
when(personRepository.findById(id)).thenReturn(Optional.of(provisional));
|
||||
when(personRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
Person result = personService.confirmPerson(id);
|
||||
|
||||
assertThat(result.isProvisional()).isFalse();
|
||||
verify(personRepository).save(argThat(p -> !p.isProvisional()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void confirmPerson_throwsNotFound_whenMissing() {
|
||||
UUID id = UUID.randomUUID();
|
||||
when(personRepository.findById(id)).thenReturn(Optional.empty());
|
||||
|
||||
assertThatThrownBy(() -> personService.confirmPerson(id))
|
||||
.isInstanceOf(DomainException.class)
|
||||
.extracting(e -> ((DomainException) e).getStatus().value())
|
||||
.isEqualTo(404);
|
||||
}
|
||||
|
||||
// ─── #667: deletePerson ───────────────────────────────────────────────────
|
||||
|
||||
@Test
|
||||
void deletePerson_deletes_whenPersonExists() {
|
||||
UUID id = UUID.randomUUID();
|
||||
Person person = Person.builder().id(id).firstName("Weg").lastName("Person").build();
|
||||
when(personRepository.findById(id)).thenReturn(Optional.of(person));
|
||||
|
||||
personService.deletePerson(id);
|
||||
|
||||
verify(personRepository).reassignSenderToNull(id);
|
||||
verify(personRepository).deleteReceiverReferences(id);
|
||||
verify(personRepository).deleteById(id);
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletePerson_throwsNotFound_whenMissing() {
|
||||
UUID id = UUID.randomUUID();
|
||||
when(personRepository.findById(id)).thenReturn(Optional.empty());
|
||||
|
||||
assertThatThrownBy(() -> personService.deletePerson(id))
|
||||
.isInstanceOf(DomainException.class)
|
||||
.extracting(e -> ((DomainException) e).getStatus().value())
|
||||
.isEqualTo(404);
|
||||
}
|
||||
|
||||
// ─── createPerson ─────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package org.raddatz.familienarchiv.tag;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class TagImportUpsertTest {
|
||||
|
||||
@Mock TagRepository tagRepository;
|
||||
@InjectMocks TagService tagService;
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_insertsNewTag_whenSourceRefUnknown() {
|
||||
when(tagRepository.findBySourceRef("Themen/Brautbriefe")).thenReturn(Optional.empty());
|
||||
when(tagRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
UUID parentId = UUID.randomUUID();
|
||||
Tag result = tagService.upsertBySourceRef("Themen/Brautbriefe", "Brautbriefe", parentId);
|
||||
|
||||
assertThat(result.getSourceRef()).isEqualTo("Themen/Brautbriefe");
|
||||
assertThat(result.getName()).isEqualTo("Brautbriefe");
|
||||
assertThat(result.getParentId()).isEqualTo(parentId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_updatesInPlace_whenSourceRefExists() {
|
||||
Tag existing = Tag.builder().id(UUID.randomUUID()).name("Brautbriefe")
|
||||
.sourceRef("Themen/Brautbriefe").build();
|
||||
when(tagRepository.findBySourceRef("Themen/Brautbriefe")).thenReturn(Optional.of(existing));
|
||||
when(tagRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
tagService.upsertBySourceRef("Themen/Brautbriefe", "Brautbriefe", null);
|
||||
|
||||
verify(tagRepository).save(argThat(t -> t.getId().equals(existing.getId())));
|
||||
verify(tagRepository, never()).save(argThat(t -> t.getId() == null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void upsertBySourceRef_preservesHumanRenamedTag_onReimport() {
|
||||
Tag humanRenamed = Tag.builder().id(UUID.randomUUID()).name("Verlobungsbriefe")
|
||||
.sourceRef("Themen/Brautbriefe").build();
|
||||
when(tagRepository.findBySourceRef("Themen/Brautbriefe")).thenReturn(Optional.of(humanRenamed));
|
||||
when(tagRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
|
||||
|
||||
Tag result = tagService.upsertBySourceRef("Themen/Brautbriefe", "Brautbriefe", null);
|
||||
|
||||
assertThat(result.getName()).isEqualTo("Verlobungsbriefe");
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,8 @@ import org.raddatz.familienarchiv.security.PermissionAspect;
|
||||
import org.raddatz.familienarchiv.user.CustomUserDetailsService;
|
||||
import org.raddatz.familienarchiv.document.DocumentService;
|
||||
import org.raddatz.familienarchiv.document.DocumentVersionService;
|
||||
import org.raddatz.familienarchiv.importing.MassImportService;
|
||||
import org.raddatz.familienarchiv.importing.CanonicalImportOrchestrator;
|
||||
import org.raddatz.familienarchiv.importing.ImportStatus;
|
||||
import org.raddatz.familienarchiv.document.ThumbnailBackfillService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.aop.AopAutoConfiguration;
|
||||
@@ -35,7 +36,7 @@ class AdminControllerTest {
|
||||
|
||||
@Autowired MockMvc mockMvc;
|
||||
|
||||
@MockitoBean MassImportService massImportService;
|
||||
@MockitoBean CanonicalImportOrchestrator importOrchestrator;
|
||||
@MockitoBean DocumentService documentService;
|
||||
@MockitoBean DocumentVersionService documentVersionService;
|
||||
@MockitoBean ThumbnailBackfillService thumbnailBackfillService;
|
||||
@@ -46,9 +47,9 @@ class AdminControllerTest {
|
||||
@Test
|
||||
@WithMockUser(authorities = "ADMIN")
|
||||
void importStatus_returns200_withStatusCode_whenAdmin() throws Exception {
|
||||
MassImportService.ImportStatus status = new MassImportService.ImportStatus(
|
||||
MassImportService.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
when(massImportService.getStatus()).thenReturn(status);
|
||||
ImportStatus status = new ImportStatus(
|
||||
ImportStatus.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
when(importOrchestrator.getStatus()).thenReturn(status);
|
||||
|
||||
mockMvc.perform(get("/api/admin/import-status"))
|
||||
.andExpect(status().isOk())
|
||||
@@ -60,9 +61,9 @@ class AdminControllerTest {
|
||||
@Test
|
||||
@WithMockUser(authorities = "ADMIN")
|
||||
void importStatus_messageField_notPresentInApiResponse() throws Exception {
|
||||
MassImportService.ImportStatus status = new MassImportService.ImportStatus(
|
||||
MassImportService.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
when(massImportService.getStatus()).thenReturn(status);
|
||||
ImportStatus status = new ImportStatus(
|
||||
ImportStatus.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
when(importOrchestrator.getStatus()).thenReturn(status);
|
||||
|
||||
mockMvc.perform(get("/api/admin/import-status"))
|
||||
.andExpect(status().isOk())
|
||||
|
||||
@@ -26,8 +26,10 @@
|
||||
# MAIL_HOST, MAIL_PORT, SMTP relay (production only; staging uses mailpit)
|
||||
# MAIL_USERNAME, MAIL_PASSWORD
|
||||
# APP_MAIL_FROM sender address (e.g. noreply@raddatz.cloud)
|
||||
# IMPORT_HOST_DIR absolute host path holding ONLY the ODS
|
||||
# spreadsheet and PDFs for /admin/system mass
|
||||
# IMPORT_HOST_DIR absolute host path holding the canonical
|
||||
# import artifacts (canonical-*.xlsx +
|
||||
# canonical-persons-tree.json) and the
|
||||
# <index>.pdf files for /admin/system
|
||||
# import — mounted read-only at /import inside
|
||||
# the backend. Compose refuses to start when
|
||||
# this var is unset, so staging and prod cannot
|
||||
@@ -217,12 +219,13 @@ services:
|
||||
# Bound to localhost only — Caddy fronts external traffic.
|
||||
ports:
|
||||
- "127.0.0.1:${PORT_BACKEND}:8080"
|
||||
# Host path holding the ODS spreadsheet + PDFs for the mass-import endpoint.
|
||||
# Read-only; MassImportService only reads (Files.list / Files.walk on /import).
|
||||
# Host path holding the canonical import artifacts (canonical-*.xlsx +
|
||||
# canonical-persons-tree.json) + <index>.pdf files for the import endpoint.
|
||||
# Read-only; the canonical importer only reads them from /import.
|
||||
# Required — no default — so staging and prod cannot accidentally share an
|
||||
# import source. CI workflows pin this per-env (see .gitea/workflows/).
|
||||
volumes:
|
||||
- ${IMPORT_HOST_DIR:?Set IMPORT_HOST_DIR to a host path holding the mass-import payload (ODS + PDFs). See docs/DEPLOYMENT.md.}:/import:ro
|
||||
- ${IMPORT_HOST_DIR:?Set IMPORT_HOST_DIR to a host path holding the import payload (canonical artifacts + <index>.pdf files). See docs/DEPLOYMENT.md.}:/import:ro
|
||||
environment:
|
||||
SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/archiv
|
||||
SPRING_DATASOURCE_USERNAME: archiv
|
||||
|
||||
@@ -65,7 +65,7 @@ Members of the cross-cutting layer have no entity of their own, no user-facing C
|
||||
| `dashboard` | Stats aggregation for the admin dashboard and Family Pulse widget | Aggregates from 3+ domains; no owned entities |
|
||||
| `exception` | `DomainException`, `ErrorCode` enum, `GlobalExceptionHandler` | Framework infra; consumed by every controller and service. Adding a new `ErrorCode` requires matching updates in `frontend/src/lib/shared/errors.ts` and all three `messages/*.json` locale files. Current security-related codes: `CSRF_TOKEN_MISSING` (403 on mutating request without valid `X-XSRF-TOKEN` header), `TOO_MANY_LOGIN_ATTEMPTS` (429 when login rate limit exceeded). |
|
||||
| `filestorage` | `FileService` — MinIO/S3 upload, download, presigned-URL generation | Generic service; consumed by `document` and `ocr` |
|
||||
| `importing` | `MassImportService` — async ODS/Excel batch import | Orchestrates across `person`, `tag`, `document` |
|
||||
| `importing` | `CanonicalImportOrchestrator` — async canonical import running four idempotent loaders (`TagTreeImporter` → `PersonRegisterImporter` → `PersonTreeImporter` → `DocumentImporter`) over the normalizer's committed canonical artifacts (`canonical-*.xlsx` + `canonical-persons-tree.json`) | Orchestrates across `person`, `tag`, `document` |
|
||||
| `security` | `SecurityConfig`, `Permission` enum, `@RequirePermission` annotation, `PermissionAspect` (AOP) | Framework infra; enforced globally across all controllers |
|
||||
|
||||
**Frontend `shared/`** follows the same admission criteria. Key members: `api.server.ts` (typed openapi-fetch client factory), `errors.ts` (backend `ErrorCode` → i18n mapping), `shared/primitives/` (generic UI components used across ≥2 domains), `shared/discussion/` (comment/mention editor used by `document` and `geschichte`), `shared/utils/` (pure date/sort/debounce utilities).
|
||||
|
||||
@@ -99,7 +99,7 @@ All vars are set in `.env` at the repo root (copy from `.env.example`). The back
|
||||
| `APP_BASE_URL` | Public-facing URL for email links | `http://localhost:3000` | YES (prod) | — |
|
||||
| `APP_OCR_BASE_URL` | Internal URL of the OCR service | — | YES | — |
|
||||
| `APP_OCR_TRAINING_TOKEN` | Secret token for OCR training endpoints | — | YES (prod) | YES |
|
||||
| `IMPORT_HOST_DIR` | Absolute host path holding the ODS spreadsheet + PDFs for the `/admin/system` mass-import card. Mounted read-only at `/import` inside the backend (compose-only — backend reads via `app.import.dir`). Compose refuses to start when unset, so staging and prod cannot accidentally share the source. Convention: `/srv/familienarchiv-staging/import` and `/srv/familienarchiv-production/import` | — | YES (prod compose) | — |
|
||||
| `IMPORT_HOST_DIR` | Absolute host path holding the normalizer's canonical artifacts (`canonical-{documents,persons,tag-tree}.xlsx` + `canonical-persons-tree.json`) **plus the `<index>.pdf` files** for the `/admin/system` import. Mounted read-only at `/import` inside the backend (the canonical importer reads via `app.import.dir`). Compose refuses to start when unset, so staging and prod cannot accidentally share the source. Convention: `/srv/familienarchiv-staging/import` and `/srv/familienarchiv-production/import` | — | YES (prod compose) | — |
|
||||
| `MAIL_HOST` | SMTP host | `mailpit` (dev) | YES (prod) | — |
|
||||
| `MAIL_PORT` | SMTP port | `1025` (dev) | YES (prod) | — |
|
||||
| `MAIL_USERNAME` | SMTP username | — | YES (prod) | YES |
|
||||
@@ -559,20 +559,45 @@ bash scripts/download-kraken-models.sh
|
||||
|
||||
> Downloads the Kurrent/Sütterlin HTR models. Run once after a fresh clone or when models are updated.
|
||||
|
||||
### Trigger a mass import (Excel/ODS)
|
||||
### Trigger a canonical import
|
||||
|
||||
**Dev:** drop the ODS spreadsheet + PDFs into `./import/` at the repo root — the dev compose bind-mounts it to `/import` automatically.
|
||||
The importer no longer parses the raw spreadsheet. It consumes the **canonical artifacts**
|
||||
produced by the normalizer (`tools/import-normalizer/`) — `canonical-tag-tree.xlsx`,
|
||||
`canonical-persons.xlsx`, `canonical-persons-tree.json`, `canonical-documents.xlsx` — which
|
||||
are committed under `tools/import-normalizer/out/`. The semantic transformation
|
||||
(German-date parsing, name classification) lives entirely in the normalizer; the backend
|
||||
maps the clean columns by header name. See [ADR-025](adr/025-canonical-import-and-single-migration-schema-foundation.md).
|
||||
|
||||
**Prerequisite — regenerate the artifacts when the source data changes:**
|
||||
|
||||
```bash
|
||||
cd tools/import-normalizer
|
||||
python3 -m venv .venv && .venv/bin/pip install -r requirements.txt # once, on a fresh clone
|
||||
.venv/bin/python normalize.py
|
||||
# writes the four canonical artifacts into ./out/
|
||||
```
|
||||
|
||||
**Dev:** place all four canonical artifacts **plus** the PDFs into `./import/`
|
||||
at the repo root (the dev compose bind-mounts it to `/import`, which is `app.import.dir`).
|
||||
Each PDF must be named `<index>.pdf` (e.g. `W-0124.pdf`, `Mü-0001.pdf`) and live flat in the
|
||||
import dir: since #686 the importer resolves a document's PDF directly by its index
|
||||
(`importDir/<index>.pdf`), not via a `datei`/`file` column — the recursive directory walk and
|
||||
its basename/homoglyph guards are gone, replaced by strict index validation plus a
|
||||
canonical-path containment assertion (a document whose `<index>.pdf` is absent simply becomes a
|
||||
`PLACEHOLDER`). The orchestrator smoke-checks that all four artifacts are present before
|
||||
starting and fails closed (`IMPORT_ARTIFACT_INVALID`) if any is missing.
|
||||
|
||||
**Staging/production:**
|
||||
|
||||
1. Pre-stage the payload on the host. Convention: `/srv/familienarchiv-staging/import/` or `/srv/familienarchiv-production/import/`.
|
||||
1. Pre-stage the four canonical artifacts + PDFs on the host. Convention:
|
||||
`/srv/familienarchiv-staging/import/` or `/srv/familienarchiv-production/import/`.
|
||||
```bash
|
||||
rsync -avh --progress ./import/ user@host:/srv/familienarchiv-staging/import/
|
||||
```
|
||||
2. Make sure `IMPORT_HOST_DIR=<host-path>` is set in `.env.staging` / `.env.production` (the nightly/release workflows already write this — see §3). Compose refuses to start without it.
|
||||
3. Redeploy the stack so the bind mount picks up — or, if the mount is already in place, skip to step 4.
|
||||
4. Call `POST /api/admin/trigger-import` (requires `ADMIN` permission), or click the "Import starten" button on `/admin/system`.
|
||||
5. The import runs asynchronously — poll `GET /api/admin/import-status`, watch `/admin/system`, or tail the backend logs.
|
||||
5. The import runs asynchronously — poll `GET /api/admin/import-status`, watch `/admin/system`, or tail the backend logs. Re-running is safe and idempotent (upsert by `source_ref` / document `index`). Person and tag scalar fields you edited in the app are preserved on re-import; a document's sender/receivers/tags are **canonical-authoritative** — a re-import re-applies them to exactly match the export, so a link removed from the export is removed from the document (the raw sender/receiver cell text is always kept).
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -25,6 +25,11 @@ _Not to be confused with [AppUser](#appuser-appuser)_ — `Person` is a historic
|
||||
|
||||
**UserGroup** (`UserGroup`) — a named permission bundle assigned to one or more `AppUser`s. A user's effective permissions are the union of all permissions across all groups they belong to.
|
||||
|
||||
**source_ref** (`Person.sourceRef`, `Tag.sourceRef`) — the import normalizer's stable identity for a `Person` (its `person_id`) or `Tag` (its canonical `tag_path`). It is the join key linking normalized records to documents and the idempotency key for re-import; null for manually created records and unique among non-null values.
|
||||
|
||||
**provisional person** (`Person.provisional`) — a `Person` the importer inferred from raw attribution text but could not confidently match to a known individual. The flag lets the persons directory surface uncertainty honestly rather than fabricate a confident identity; it defaults to `false` and is set `true` only by the importer.
|
||||
_Not to be confused with `family_member`_ — `provisional` expresses import confidence, while `family_member` is a genealogical fact about whether the person belongs to the family tree.
|
||||
|
||||
---
|
||||
|
||||
## Document-Related Terms
|
||||
@@ -36,6 +41,10 @@ _See also [TranscriptionBlock](#transcriptionblock-transcriptionblock)._
|
||||
|
||||
**Document** (`Document`) — a single archival item (letter, postcard, photograph) with a file stored in MinIO/S3 and associated metadata (sender, receivers, date, tags, transcription blocks).
|
||||
|
||||
**date precision** (`Document.metaDatePrecision`, enum `DatePrecision`) — how exactly a document's date is known, one of `DAY, MONTH, SEASON, YEAR, RANGE, APPROX, UNKNOWN`. A verbatim mirror of the import normalizer's `Precision` enum so honest dates can be rendered (`APPROX` → "ca.", `RANGE` uses `meta_date_end`) instead of fabricating a false `DAY`-level date. `UNKNOWN` is the explicit value for undated documents.
|
||||
|
||||
**raw attribution** (`Document.senderText`, `Document.receiverText`, `Document.metaDateRaw`) — the original spreadsheet cell text for a document's sender, receiver, and date, preserved verbatim even after a `Person` or normalized date is linked. It keeps provenance intact and enables an "as written in the original" view.
|
||||
|
||||
**DocumentVersion** (`DocumentVersion`) — an append-only snapshot of a `Document`'s metadata at a point in time. Append-only by convention; no consumer-facing create or update endpoint exists. The entity uses Lombok `@Data` (which generates setters), so immutability is enforced by application convention, not at the Java level.
|
||||
|
||||
**Tag** (`Tag`) — a hierarchical category that can be applied to `Document`s. Tags are self-referencing via a `parent_id` foreign key, forming a tree structure.
|
||||
@@ -55,9 +64,13 @@ _See also [Annotation](#annotation-documentannotation)._
|
||||
- `REVIEWED`: a reviewer has approved the transcription.
|
||||
- `ARCHIVED`: the document is finalized and read-only.
|
||||
|
||||
**Mass import** — an asynchronous batch process (`MassImportService`) that reads an Excel or ODS file and creates `Person`s, `Tag`s, and `PLACEHOLDER` `Document`s in one shot. Only one import can run at a time (`IMPORT_ALREADY_RUNNING` error if attempted concurrently).
|
||||
**Canonical import** — an asynchronous batch process (`CanonicalImportOrchestrator`) that consumes the normalizer's committed canonical artifacts and creates `Tag`s, `Person`s (register + tree), family relationships, and `Document`s. Four idempotent loaders run in a fixed dependency order — `TagTreeImporter` → `PersonRegisterImporter` → `PersonTreeImporter` → `DocumentImporter` — each calling the owning domain's service. Re-running it never duplicates rows (upsert by `source_ref` / document `index`) and never overwrites a human-edited field. Only one import can run at a time (`IMPORT_ALREADY_RUNNING` error if attempted concurrently); a missing or malformed artifact fails closed (`IMPORT_ARTIFACT_INVALID`). Replaced the legacy raw-spreadsheet `MassImportService` (see ADR-025).
|
||||
|
||||
**SkippedFile** (`MassImportService.SkippedFile`) — a file that was presented for import but not processed, recorded with a `filename` and a `reason` code. Possible reasons: `INVALID_PDF_SIGNATURE` (magic-byte validation failed), `S3_UPLOAD_FAILED` (file upload to MinIO/S3 threw an exception), `FILE_READ_ERROR` (the file could not be opened for reading), or `ALREADY_EXISTS` (a document with the same filename already exists in the archive with a status other than `PLACEHOLDER`).
|
||||
**canonical artifact** — one of the four files the normalizer (`tools/import-normalizer/`) emits and commits to `tools/import-normalizer/out/`: `canonical-tag-tree.xlsx`, `canonical-persons.xlsx`, `canonical-persons-tree.json`, `canonical-documents.xlsx`. They are the contract the backend importer reads (mapped by header name); the semantic transformation (German-date parsing, name classification) lives only in the normalizer, never in Java.
|
||||
|
||||
**CanonicalSheetReader** — the value-level POI helper that opens a canonical `.xlsx`, maps the header row to column indices by name (replacing the brittle positional column config), splits pipe-delimited list columns, and throws `IMPORT_ARTIFACT_INVALID` on a missing required header rather than NPE-ing on a null index.
|
||||
|
||||
**SkippedFile** (`ImportStatus.SkippedFile`) — a file that was presented for import but not processed, recorded with a `filename` and a `reason` code. Possible reasons: `INVALID_FILENAME_PATH_TRAVERSAL` (the file-column basename failed the path-traversal guard), `INVALID_PDF_SIGNATURE` (magic-byte validation failed), `S3_UPLOAD_FAILED` (file upload to MinIO/S3 threw an exception), `FILE_READ_ERROR` (the file could not be opened for reading), or `ALREADY_EXISTS` (a document with the same `index` already exists in the archive with a status other than `PLACEHOLDER`).
|
||||
|
||||
**skipped count** — the total number of `SkippedFile` entries accumulated during a single import run (`ImportStatus.skipped()`). Shown in the amber warning section of the Import Status Card in the admin UI; a value of zero suppresses the section entirely.
|
||||
|
||||
|
||||
@@ -94,17 +94,6 @@ The schema includes `spring_session` and `spring_session_attributes` tables, but
|
||||
|
||||
---
|
||||
|
||||
### `MassImportService` provides no status or error feedback
|
||||
**File:** `service/MassImportService.java`, `controller/AdminController.java`
|
||||
|
||||
`/api/admin/trigger-import` returns immediately (async), but there is no way for the admin to know whether the import succeeded, failed, or is still running. Errors during async execution are silently swallowed.
|
||||
|
||||
**Fix options:**
|
||||
- Store import job status in a DB table (`import_jobs`) with state (`RUNNING`, `DONE`, `FAILED`) and expose a `GET /api/admin/import-status` endpoint
|
||||
- Alternatively, make the endpoint synchronous since it already blocks on file I/O — only use async if you need true non-blocking behaviour
|
||||
|
||||
---
|
||||
|
||||
## Missing Capabilities
|
||||
|
||||
### No test coverage
|
||||
@@ -114,7 +103,7 @@ The only test is a Spring context load test. No unit or integration tests exist
|
||||
|
||||
**Suggested starting points (highest value for effort):**
|
||||
1. `DocumentSpecifications` — pure logic, easy to unit test with an in-memory H2 or Testcontainers PostgreSQL
|
||||
2. `ExcelService` — parsing logic, test with fixture `.xlsx` files (one exists in `api_tests/`)
|
||||
2. Canonical import loaders (`CanonicalSheetReader`, `DocumentImporter`, etc.) — parsing/upsert logic, test with fixture canonical `.xlsx` files
|
||||
3. `PermissionAspect` — security logic should be tested; use `@WithMockUser` from Spring Security Test
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
# ADR-025 — Canonical Import Output as Contract & Single-Migration Schema Foundation
|
||||
|
||||
**Date:** 2026-05-27
|
||||
**Status:** Accepted
|
||||
**Issue:** #671 (schema, decisions 1–2); #669 (importer architecture, decision 3)
|
||||
**Milestone:** Handling the Unknowns — honest uncertainty in dates & people
|
||||
|
||||
---
|
||||
|
||||
## Context
|
||||
|
||||
The "Handling the Unknowns" milestone introduces honest uncertainty into the archive:
|
||||
documents whose dates are known only approximately or as a range, and people the importer
|
||||
infers from raw attribution text but cannot confidently identify. Three sibling issues —
|
||||
date precision (#666), name triage (#665), and the importer (#669) — each independently
|
||||
planned a Flyway `V69` migration that altered `persons`. Three `V69`s is a boot failure
|
||||
(Flyway versions must be unique), and `persons.provisional` was at risk of being defined
|
||||
twice.
|
||||
|
||||
Two durable decisions had to be made before any application code in Phases 3–6 could
|
||||
compile against the new schema.
|
||||
|
||||
---
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. All import/precision/attribution/identity schema lives in ONE migration with a single owner
|
||||
|
||||
`V69__import_precision_attribution_identity_schema.sql` adds every new column for this
|
||||
milestone in a single, atomic, forward-only migration:
|
||||
|
||||
- `documents`: `meta_date_precision` (backfilled `DAY` where dated / `UNKNOWN` where not,
|
||||
then `NOT NULL`), `meta_date_end`, `meta_date_raw`, `sender_text`, `receiver_text`.
|
||||
- `persons`: `source_ref` (unique index, nullable), `provisional` (`NOT NULL DEFAULT false`).
|
||||
- `tag`: `source_ref` (unique index, nullable).
|
||||
|
||||
Integrity is pushed to the database as fail-closed `CHECK` constraints (the precedent is
|
||||
`V22`'s `person_type` allowlist):
|
||||
|
||||
- `meta_date_precision` must be one of the seven enum values.
|
||||
- `meta_date_end` may be non-null **only** when precision = `RANGE` (one-directional, not
|
||||
biconditional — see Consequences).
|
||||
- `meta_date_end >= meta_date` for ranges with both endpoints (a `CHECK`, not a trigger).
|
||||
- `meta_date_raw`, `sender_text`, `receiver_text` are length-capped at 10 000 (mirrors the
|
||||
`transcription_blocks` cap in `V18`).
|
||||
|
||||
No sibling issue adds another migration that alters `persons` or `documents` in this
|
||||
milestone.
|
||||
|
||||
### 2. The backend `DatePrecision` enum is a verbatim mirror of the normalizer's `Precision`; the canonical output is the contract
|
||||
|
||||
The importer reads the Python normalizer's canonical output
|
||||
(`tools/import-normalizer/`). The backend `DatePrecision` enum
|
||||
(`DAY, MONTH, SEASON, YEAR, RANGE, APPROX, UNKNOWN`) is a verbatim copy of the normalizer's
|
||||
`Precision(StrEnum)` (`dates.py`). There is **no translation layer**: the normalizer's
|
||||
output strings are persisted as-is. The same applies to `source_ref`, which carries the
|
||||
normalizer's `person_id` / canonical `tag_path` unchanged as the re-import idempotency key.
|
||||
|
||||
### 3. The importer is four idempotent loaders over the canonical artifacts; Java no longer parses the raw spreadsheet (Phase 3, #669)
|
||||
|
||||
The legacy `MassImportService` read the *raw* original spreadsheet by positional column
|
||||
index (`@Value app.import.col.*`) and re-derived everything in Java (ISO-only date parsing,
|
||||
name classification via `findOrCreateByAlias`, an ODS/XXE XML path). It is **deleted**.
|
||||
|
||||
The rebuild is a `CanonicalImportOrchestrator` driving four single-responsibility loaders in
|
||||
an explicit dependency DAG — `TagTreeImporter` → `PersonRegisterImporter` →
|
||||
`PersonTreeImporter` → `DocumentImporter` — that **consume the committed canonical artifacts**
|
||||
(`tools/import-normalizer/out/`). A shared `CanonicalSheetReader` maps columns **by header
|
||||
name** (not by index) and fails closed (`IMPORT_ARTIFACT_INVALID`) on a missing header. Each
|
||||
loader calls the **owning domain's service**, never a repository (layering rule); the tree
|
||||
loader uses `RelationshipService`, never the relationship repository.
|
||||
|
||||
Settled sub-decisions:
|
||||
|
||||
- **Idempotency precedence is domain-specific.** Persons/tags upsert by `source_ref`,
|
||||
documents by `index`. Two distinct rules apply:
|
||||
- **Person/Tag scalar fields = preserve human edits.** On re-import a non-blank field a human
|
||||
changed in-app is never overwritten (blank fields are filled from canonical via the single
|
||||
`preferHuman` idiom), and `provisional` is monotonic-downward — once a human confirms a
|
||||
person (`false`) it never reverts to `true`. Because the orchestrator loads the register and
|
||||
tree *before* documents, a person already `false` can never be flipped provisional by a
|
||||
later document row that references the same `source_ref`, regardless of document-row order.
|
||||
- **Document sender/receivers/tags = canonical-authoritative.** A document's sender, receiver
|
||||
set, and tag set are owned by the canonical row, not the archivist. On re-import of a
|
||||
PLACEHOLDER document `DocumentImporter` clears and re-populates `receivers`/`tags` so a row
|
||||
whose set *shrinks* prunes the removed links rather than accumulating stale ones. The
|
||||
"preserve human edits" rule above does **not** extend to these collections. The raw
|
||||
`sender_text`/`receiver_text` cells are always retained verbatim (a separate invariant).
|
||||
Note non-PLACEHOLDER documents are skipped entirely (`ALREADY_EXISTS`), so once a document
|
||||
has a file the importer never touches it again — this bounds the authoritative-overwrite
|
||||
blast radius to placeholder rows.
|
||||
Verified against real Postgres in `CanonicalImportIntegrationTest`
|
||||
(`reimport_preservesHumanEditedPersonField`, `reimport_prunesRemovedReceiverAndTag…`,
|
||||
`import_neverFlipsRegisterPersonToProvisional…`).
|
||||
- **Name policy = Option A.** The normalizer resolved attribution upstream: the document sheet
|
||||
carries the resolved slug in `sender_person_id` / `receiver_person_ids` and the raw cell in
|
||||
`sender_name` / `receiver_names`. The importer routes register-first by `source_ref`
|
||||
(provisional `Person` when a slug is unmatched), and **always retains the raw cell** in
|
||||
`sender_text` / `receiver_text` even when a person is linked — the load-bearing invariant
|
||||
behind the merge story. A row with no slug but raw text (prose / `?` / object-noise) links
|
||||
no person and keeps only the raw text.
|
||||
- **`provisional` is now populated.** Importer-minted persons are `provisional = true`;
|
||||
register and tree persons stay `false`. This is the Phase-3 contract the schema (decision 1)
|
||||
left at default-`false`.
|
||||
- **PDFs resolve directly by index (`<index>.pdf`), not by a `file` column.** The corpus is
|
||||
uniform — all PDFs are named `<index>.pdf` flat in the import dir (e.g. `W-0124.pdf`,
|
||||
`Mü-0001.pdf`) — so `DocumentImporter` resolves a document's PDF with an O(1)
|
||||
`importDir.resolve(index + ".pdf")` lookup. The redundant `file` column (carrying the
|
||||
spreadsheet's messy `datei` value) and the recursive directory walk that resolved it were
|
||||
removed (#686, which also closed #676 — the O(rows×tree) walk is gone). The normalizer no
|
||||
longer emits `file` or the `index_file_mismatch` review flag.
|
||||
- **Security guards are defense-in-depth, not upstream-trust.** The `index` is the only thing
|
||||
that drives the on-disk lookup, so it is treated as hostile (CWE-22 does not care it came from
|
||||
our tool): `isValidImportIndex` rejects slash/backslash, three Unicode slash homoglyphs, any
|
||||
`.` (so `<index>.pdf` is the only extension and `..` can never appear), null byte, and
|
||||
absolute paths, and requires a strict catalog shape (1–4 Latin letters incl. umlauts, one or
|
||||
more hyphens, digits, optional trailing `x`). A bad index skips the row with a clear
|
||||
`SkipReason` (`INVALID_FILENAME_PATH_TRAVERSAL`). The resolved canonical path is still asserted
|
||||
to stay inside the import dir as a second line of defense (a symlinked `<index>.pdf` cannot
|
||||
escape), and the `%PDF` magic-byte check still gates upload. These guards and their tests were
|
||||
ported from the file-column resolution (originally from `MassImportService`).
|
||||
|
||||
---
|
||||
|
||||
## Consequences
|
||||
|
||||
- **RANGE is one-directional, not biconditional.** A `RANGE` row may have a null
|
||||
`meta_date_end` (an open-ended range with only a start), because the normalizer can emit
|
||||
start-only ranges. A biconditional `RANGE ⟺ end IS NOT NULL` rule would reject valid
|
||||
normalizer output, so it was rejected. Phase 4 rendering must handle a `RANGE` with no end
|
||||
gracefully.
|
||||
- **`provisional` stays `false` throughout this phase.** The column and flag exist, but no
|
||||
code path sets it `true`; the importer (Phase 3) is the only writer. This is intentional,
|
||||
not a half-built feature.
|
||||
- **A future dev must not "improve" the enum.** Renaming or dropping a `DatePrecision` value
|
||||
without changing the normalizer silently breaks import idempotency and date rendering. The
|
||||
enum's Javadoc states this; the DB `CHECK` enforces validity independent of the Java enum.
|
||||
- **`source_ref` is unique + nullable.** Manually created persons/tags have `source_ref =
|
||||
NULL`; Postgres allows multiple NULLs under a plain unique index, so no backfill is needed.
|
||||
- **Forward-only.** The migration is immutable once shipped (Flyway checksum model); any fix
|
||||
goes in a later version. There is no down-migration — rollback means restoring from the
|
||||
nightly `pg_dump`, the standard procedure.
|
||||
- **`runImport()` is non-transactional — per-loader transactions only.** The orchestrator
|
||||
does not wrap the four loaders in a single transaction; each loader (or the per-call
|
||||
`upsertBySourceRef` / `DocumentImporter.load`) carries its own `@Transactional` boundary. A
|
||||
partial failure mid-run (e.g. the document loader throws after tags + persons committed)
|
||||
leaves the earlier loaders' data committed and the `ImportStatus` set to `FAILED`. This is
|
||||
acceptable precisely because the import is idempotent: re-running is safe and converges to
|
||||
the same state, so the operational recovery for a partial failure is simply to fix the
|
||||
offending artifact and re-trigger the import — no manual cleanup of half-written data is
|
||||
required. A future maintainer must not assume all-or-nothing semantics.
|
||||
- **The index pattern is corpus-specific and must be revisited if the catalog scheme grows.**
|
||||
`INDEX_PATTERN` accepts only the *current* corpus shape — at most four Latin-1 letters (incl.
|
||||
umlauts) followed by one or more hyphens, ASCII digits, and an optional trailing `x`. This is a
|
||||
conscious constraint, not a general filename validator: a future sub-collection catalogued with
|
||||
a 5-letter prefix, a digit-led id, or a non-Latin-1 letter (e.g. `Č` or a Cyrillic id) would
|
||||
fail `isValidImportIndex` and its rows would be **skipped** (`INVALID_FILENAME_PATH_TRAVERSAL`),
|
||||
not imported. Likewise a real PDF that does not follow `<index>.pdf` produces a `PLACEHOLDER`
|
||||
(the importer logs both cases distinctly — see #686). If the catalog scheme ever changes, the
|
||||
pattern and its tests must be widened deliberately; do not loosen it casually, as it is the
|
||||
allowlist that keeps the on-disk lookup safe. Note `\d` is intentionally ASCII-only — adding
|
||||
`Pattern.UNICODE_CHARACTER_CLASS` would silently widen the accepted digit set.
|
||||
- **A malicious/garbage index skips its row with a loud `SkipReason`, by design.** Since #686
|
||||
the index is the only on-disk lookup key. An index that fails `isValidImportIndex`
|
||||
(path separator, traversal token, slash homoglyph, null byte, absolute path, or a non-catalog
|
||||
shape) is recorded as a `SkippedFile` with reason `INVALID_FILENAME_PATH_TRAVERSAL` and the
|
||||
import continues with the remaining rows — nothing outside the import dir is ever read. A
|
||||
symlinked `<index>.pdf` whose canonical path escapes the import dir is the one case that still
|
||||
aborts the import (a `DomainException` from the containment assertion), because a syntactically
|
||||
valid index resolving outside the dir is an environment-level attack signal, not a row typo.
|
||||
- **`PersonSummaryDTO` coupling.** `provisional` was added to the `PersonSummaryDTO` native
|
||||
interface projection; because the projection is backed by native SQL, the column had to be
|
||||
added to all three native `SELECT`s (`findAllWithDocumentCount`, `searchWithDocumentCount`,
|
||||
`findTopByDocumentCount`) or it would silently return `false`. Guarded by integration tests
|
||||
against real Postgres.
|
||||
@@ -93,7 +93,7 @@ C4Component
|
||||
|
||||
### 3b — Document Management & Import
|
||||
|
||||
Document management, file storage, and bulk Excel/ODS import.
|
||||
Document management, file storage, and the canonical import.
|
||||
|
||||
```mermaid
|
||||
C4Component
|
||||
@@ -105,12 +105,11 @@ C4Component
|
||||
|
||||
System_Boundary(backend, "API Backend (Spring Boot)") {
|
||||
Component(docCtrl, "DocumentController", "Spring MVC — /api/documents", "CRUD for documents: search, get by ID, update metadata, upload/download file, conversation thread, and batch metadata updates.")
|
||||
Component(adminCtrl, "AdminController", "Spring MVC — /api/admin", "Triggers asynchronous Excel/ODS mass import (requires ADMIN permission). Reports import state (IDLE/RUNNING/DONE/FAILED).")
|
||||
Component(adminCtrl, "AdminController", "Spring MVC — /api/admin", "Triggers the asynchronous canonical import (requires ADMIN permission). Reports import state via GET /api/admin/import-status (IDLE/RUNNING/DONE/FAILED).")
|
||||
|
||||
Component(docSvc, "DocumentService", "Spring Service", "Core document business logic: store, update, search. Resolves persons and tags, delegates file I/O to FileService, builds dynamic JPA Specifications, and integrates with audit logging.")
|
||||
Component(fileSvc, "FileService", "Spring Service", "Wraps AWS SDK v2 S3Client. Uploads files with UUID-keyed paths, computes SHA-256 hash, downloads with content-type detection, and generates presigned URLs for OCR access.")
|
||||
Component(massImport, "MassImportService", "Spring Service — @Async", "Reads Excel/ODS files from /import mount. Tracks import state (IDLE/RUNNING/DONE/FAILED) and delegates to ExcelService. Returns immediately; processing runs asynchronously.")
|
||||
Component(excelSvc, "ExcelService", "Spring Service", "Parses Excel/ODS workbooks (Apache POI). Column indices configurable via application.properties. Creates/updates document records per row.")
|
||||
Component(importOrch, "CanonicalImportOrchestrator", "Spring Service — @Async", "Runs four idempotent loaders (TagTree → PersonRegister → PersonTree → Document) in a fixed DAG over the normalizer's committed canonical artifacts (canonical-*.xlsx + canonical-persons-tree.json) from /import — see diagram 3b. Owns the IDLE/RUNNING/DONE/FAILED state machine.")
|
||||
Component(minioConf, "MinioConfig", "Spring @Configuration", "Creates the S3Client and S3Presigner beans with path-style access for MinIO. Validates MinIO connectivity on startup.")
|
||||
|
||||
Component(docRepo, "DocumentRepository", "Spring Data JPA", "Queries documents with Specification-based dynamic search, bidirectional conversation thread queries, full-text search with ranking and match highlighting, and transcription pipeline queue projections.")
|
||||
@@ -123,14 +122,15 @@ C4Component
|
||||
Rel(frontend, docCtrl, "Document requests", "HTTP / JSON")
|
||||
Rel(frontend, adminCtrl, "Trigger import", "HTTP / JSON")
|
||||
Rel(docCtrl, docSvc, "Delegates to", "")
|
||||
Rel(adminCtrl, massImport, "Triggers", "")
|
||||
Rel(adminCtrl, importOrch, "Triggers", "")
|
||||
Rel(docSvc, fileSvc, "Upload / download files", "")
|
||||
Rel(docSvc, docRepo, "Reads / writes documents", "")
|
||||
Rel(docSvc, docSpec, "Builds search predicates", "")
|
||||
Rel(docSvc, personSvc, "Resolves sender / receivers", "")
|
||||
Rel(docSvc, tagSvc, "Finds or creates tags", "")
|
||||
Rel(massImport, excelSvc, "Parses Excel/ODS file", "")
|
||||
Rel(excelSvc, docSvc, "Creates / updates documents", "")
|
||||
Rel(importOrch, docSvc, "Upserts documents (PDF by index) — see 3b", "")
|
||||
Rel(importOrch, personSvc, "Upserts persons + relationships", "")
|
||||
Rel(importOrch, tagSvc, "Upserts tag hierarchy", "")
|
||||
Rel(minioConf, fileSvc, "Provides S3Client and S3Presigner beans", "")
|
||||
Rel(fileSvc, minio, "PUT / GET / presigned URL objects", "S3 API / HTTP")
|
||||
Rel(docRepo, db, "SQL queries", "JDBC")
|
||||
@@ -492,7 +492,7 @@ C4Component
|
||||
Component(adminGroups, "/admin/groups, /admin/groups/[id], /admin/groups/new", "SvelteKit Routes", "Permission group management: create/edit groups and their permission sets.")
|
||||
Component(adminTags, "/admin/tags and /admin/tags/[id]", "SvelteKit Routes", "Tag administration: edit tag hierarchy, merge tags, delete subtrees.")
|
||||
Component(adminOcr, "/admin/ocr and /admin/ocr/[personId]", "SvelteKit Routes", "Global and per-person OCR configuration. Manages script types and triggers sender model training.")
|
||||
Component(adminSystem, "/admin/system", "SvelteKit Route", "System status panel. Triggers Excel/ODS mass import (POST /api/admin/trigger-import). Displays import state.")
|
||||
Component(adminSystem, "/admin/system", "SvelteKit Route", "System status panel. Triggers the canonical import (POST /api/admin/trigger-import). Displays import state.")
|
||||
Component(hilfe, "/hilfe/transkription", "SvelteKit Route", "Static transcription style guide for Kurrent and Sütterlin character recognition. No backend calls.")
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
@startuml
|
||||
!include <C4/C4_Component>
|
||||
|
||||
title Component Diagram: API Backend — Document Management & Import
|
||||
title Component Diagram: API Backend — Document Management & Canonical Import
|
||||
|
||||
Container(frontend, "Web Frontend", "SvelteKit")
|
||||
ContainerDb(db, "PostgreSQL", "PostgreSQL 16")
|
||||
@@ -9,30 +9,50 @@ ContainerDb(minio, "Object Storage", "MinIO (S3-compatible)")
|
||||
|
||||
System_Boundary(backend, "API Backend (Spring Boot)") {
|
||||
Component(docCtrl, "DocumentController", "Spring MVC — /api/documents", "CRUD for documents: search, get by ID, update metadata, upload/download file, conversation thread, batch metadata updates, and per-month density aggregation for the timeline filter widget.")
|
||||
Component(adminCtrl, "AdminController", "Spring MVC — /api/admin", "Triggers asynchronous Excel/ODS mass import (requires ADMIN permission). Reports import state (IDLE/RUNNING/DONE/FAILED).")
|
||||
Component(adminCtrl, "AdminController", "Spring MVC — /api/admin", "Triggers the asynchronous canonical import (requires ADMIN permission). Reports import state (IDLE/RUNNING/DONE/FAILED).")
|
||||
Component(docSvc, "DocumentService", "Spring Service", "Core document business logic: store, update, search. Resolves persons and tags, delegates file I/O to FileService, builds dynamic JPA Specifications, and integrates with audit logging.")
|
||||
Component(fileSvc, "FileService", "Spring Service", "Wraps AWS SDK v2 S3Client. Uploads files with UUID-keyed paths, computes SHA-256 hash, downloads with content-type detection, and generates presigned URLs for OCR access.")
|
||||
Component(massImport, "MassImportService", "Spring Service — @Async", "Reads Excel/ODS files from /import mount. Tracks import state (IDLE/RUNNING/DONE/FAILED) and delegates to ExcelService. Returns immediately; processing runs asynchronously.")
|
||||
Component(excelSvc, "ExcelService", "Spring Service", "Parses Excel/ODS workbooks (Apache POI). Column indices configurable via application.properties. Creates/updates document records per row.")
|
||||
Component(importOrch, "CanonicalImportOrchestrator", "Spring Service — @Async", "Runs the four canonical loaders in an explicit dependency DAG (TagTree → PersonRegister → PersonTree → Document). Smoke-checks all four artifacts before starting, owns the IDLE/RUNNING/DONE/FAILED state machine, fails closed on a malformed artifact.")
|
||||
Component(tagTreeLoader, "TagTreeImporter", "Spring Component", "Upserts the tag hierarchy from canonical-tag-tree.xlsx via TagService (by canonical tag_path).")
|
||||
Component(personRegLoader, "PersonRegisterImporter", "Spring Component", "Upserts register persons from canonical-persons.xlsx via PersonService (by normalizer person_id).")
|
||||
Component(personTreeLoader, "PersonTreeImporter", "Spring Component", "Upserts tree persons + relationships from canonical-persons-tree.json via PersonService and RelationshipService.")
|
||||
Component(docLoader, "DocumentImporter", "Spring Component", "Loads canonical-documents.xlsx: routes attribution register-first (raw cell always retained in sender_text/receiver_text), parses clean dates, builds an honest precision-aware title via DocumentTitleFormatter, keeps the S3 upload + thumbnail plumbing, and resolves each PDF by index (importDir/<index>.pdf) guarded by strict index validation + canonical-path containment + %PDF magic-byte check (no recursive walk).")
|
||||
Component(titleFmt, "DocumentTitleFormatter", "Pure helper", "Formats the date label baked into an import title at exactly the data's precision (MONTH -> 'Juni 1916', never a fabricated day). Mirrors the frontend formatDocumentDate; both are pinned to docs/date-label-fixtures.json (#666).")
|
||||
Component(sheetReader, "CanonicalSheetReader", "POI helper", "Maps a canonical .xlsx by header name (no positional indices), splits pipe-delimited list columns, fails closed (IMPORT_ARTIFACT_INVALID) on a missing required header.")
|
||||
Component(minioConf, "MinioConfig", "Spring @Configuration", "Creates the S3Client and S3Presigner beans with path-style access for MinIO. Validates MinIO connectivity on startup.")
|
||||
Component(docRepo, "DocumentRepository", "Spring Data JPA", "Queries documents with Specification-based dynamic search, bidirectional conversation thread queries, full-text search with ranking and match highlighting, and transcription pipeline queue projections.")
|
||||
Component(docSpec, "DocumentSpecifications", "JPA Criteria API", "Factory for composable predicates: hasText (full-text), hasSender, hasReceiver, isBetween (date range), hasTags (subquery AND/OR logic).")
|
||||
}
|
||||
|
||||
Component(personSvc, "PersonService", "Spring Service", "See diagram 3e. Called by DocumentService to resolve sender / receiver persons by ID.")
|
||||
Component(tagSvc, "TagService", "Spring Service", "See diagram 3d. Called by DocumentService to find or create tags by name.")
|
||||
Component(personSvc, "PersonService", "Spring Service", "See diagram 3e. Resolves sender / receiver persons by ID; upserts persons by source_ref for the importer.")
|
||||
Component(tagSvc, "TagService", "Spring Service", "See diagram 3d. Finds or creates tags by name; upserts tags by source_ref for the importer.")
|
||||
Component(relSvc, "RelationshipService", "Spring Service", "See diagram 3e. Creates family relationships from the person tree during import.")
|
||||
|
||||
Rel(frontend, docCtrl, "Document requests", "HTTP / JSON")
|
||||
Rel(frontend, adminCtrl, "Trigger import", "HTTP / JSON")
|
||||
Rel(docCtrl, docSvc, "Delegates to")
|
||||
Rel(adminCtrl, massImport, "Triggers")
|
||||
Rel(adminCtrl, importOrch, "Triggers")
|
||||
Rel(docSvc, fileSvc, "Upload / download files")
|
||||
Rel(docSvc, docRepo, "Reads / writes documents")
|
||||
Rel(docSvc, docSpec, "Builds search predicates")
|
||||
Rel(docSvc, personSvc, "Resolves sender / receivers")
|
||||
Rel(docSvc, tagSvc, "Finds or creates tags")
|
||||
Rel(massImport, excelSvc, "Parses Excel/ODS file")
|
||||
Rel(excelSvc, docSvc, "Creates / updates documents")
|
||||
Rel(importOrch, tagTreeLoader, "1. Loads tags")
|
||||
Rel(importOrch, personRegLoader, "2. Loads register persons")
|
||||
Rel(importOrch, personTreeLoader, "3. Loads tree persons + relationships")
|
||||
Rel(importOrch, docLoader, "4. Loads documents")
|
||||
Rel(tagTreeLoader, sheetReader, "Reads canonical .xlsx")
|
||||
Rel(personRegLoader, sheetReader, "Reads canonical .xlsx")
|
||||
Rel(docLoader, sheetReader, "Reads canonical .xlsx")
|
||||
Rel(docLoader, titleFmt, "Builds honest title date")
|
||||
Rel(tagTreeLoader, tagSvc, "Upserts tags by source_ref")
|
||||
Rel(personRegLoader, personSvc, "Upserts persons by source_ref")
|
||||
Rel(personTreeLoader, personSvc, "Upserts persons by source_ref")
|
||||
Rel(personTreeLoader, relSvc, "Creates relationships")
|
||||
Rel(docLoader, docSvc, "Upserts documents by index")
|
||||
Rel(docLoader, personSvc, "Register-first match / provisional person")
|
||||
Rel(docLoader, tagSvc, "Attaches tag by source_ref")
|
||||
Rel(docLoader, fileSvc, "Uploads resolved file")
|
||||
Rel(minioConf, fileSvc, "Provides S3Client and S3Presigner beans")
|
||||
Rel(fileSvc, minio, "PUT / GET / presigned URL objects", "S3 API / HTTP")
|
||||
Rel(docRepo, db, "SQL queries", "JDBC")
|
||||
|
||||
@@ -7,12 +7,12 @@ Container(frontend, "Web Frontend", "SvelteKit")
|
||||
ContainerDb(db, "PostgreSQL", "PostgreSQL 16")
|
||||
|
||||
System_Boundary(backend, "API Backend (Spring Boot)") {
|
||||
Component(personCtrl, "PersonController", "Spring MVC — /api/persons", "Lists and searches family members. Returns documents sent by or received by a person, correspondent suggestions, and person summary with document counts.")
|
||||
Component(personCtrl, "PersonController", "Spring MVC — /api/persons", "Filtered, paginated directory (type/familyOnly/hasDocuments/provisional + page/size -> PersonSearchResult). Returns documents sent/received, correspondent suggestions, person summaries with counts. PATCH /{id}/confirm clears provisional; DELETE /{id} removes a person (both WRITE_ALL).")
|
||||
Component(relCtrl, "RelationshipController", "Spring MVC — /api/network, /api/persons/{id}/relationships", "CRUD for explicit person relationships and the full family network graph (nodes + edges) used by the Stammbaum view.")
|
||||
Component(personSvc, "PersonService", "Spring Service", "Person CRUD, alias management, and merge operations (reassigns all document sender/receiver references before deleting duplicate persons).")
|
||||
Component(personSvc, "PersonService", "Spring Service", "Person CRUD, alias management, filtered paged search (PersonFilter -> paired slice/count), confirm (clears provisional), delete (detaches document refs first), and merge operations (reassigns all document sender/receiver references before deleting duplicate persons).")
|
||||
Component(relSvc, "RelationshipService", "Spring Service", "Manages explicit directional family relationships (PARENT_OF, SPOUSE_OF, SIBLING_OF, etc.) with optional date ranges and notes.")
|
||||
Component(relInference, "RelationshipInferenceService", "Spring Service", "Computes transitive family relationships from explicit edges to infer grandparent/grandchild, aunt/uncle, and other extended-family links for the network graph.")
|
||||
Component(personRepo, "PersonRepository", "Spring Data JPA", "Queries persons with name search (including aliases), correspondent discovery, person summaries with document counts, and merge/reassignment helpers.")
|
||||
Component(personRepo, "PersonRepository", "Spring Data JPA", "Queries persons with name search (including aliases), correspondent discovery, person summaries with document counts, paired filter-aware slice + COUNT queries (one shared WHERE clause), and merge/reassignment helpers.")
|
||||
Component(relRepo, "PersonRelationshipRepository", "Spring Data JPA", "Reads and writes PersonRelationship records. Supports lookup by person ID, by relation type, and existence checks for deduplication.")
|
||||
}
|
||||
|
||||
|
||||
@@ -7,8 +7,9 @@ Person(user, "User")
|
||||
Container(backend, "API Backend", "Spring Boot")
|
||||
|
||||
System_Boundary(frontend, "Web Frontend (SvelteKit / SSR)") {
|
||||
Component(personsPage, "/persons and /persons/[id]", "SvelteKit Routes", "Person directory and detail. Detail: metadata, document list sent/received, correspondents, explicit and inferred family relationships.")
|
||||
Component(personsPage, "/persons and /persons/[id]", "SvelteKit Routes", "Person directory (server-side filtered + paginated) and detail. Directory: type/family/has-documents chips, reader default (familyMember OR documentCount > 0), writer-only show-all toggle. Detail: metadata, document list sent/received, correspondents, family relationships.")
|
||||
Component(personEdit, "/persons/[id]/edit and /persons/new", "SvelteKit Routes", "Create and edit person forms. Edit: metadata, aliases, explicit relationships. Actions: PUT/POST /api/persons.")
|
||||
Component(personReview, "/persons/review", "SvelteKit Route", "Transcriber triage view (WRITE-gated link). Lists provisional persons; per-row Merge / Umbenennen / Bestätigen / Löschen. Actions: POST /merge, PUT /{id}, PATCH /{id}/confirm, DELETE /{id}.")
|
||||
Component(briefwechsel, "/briefwechsel", "SvelteKit Route", "Bilateral conversation timeline. Selects two persons via PersonTypeahead, fetches GET /api/documents/conversation, displays chronological exchange.")
|
||||
Component(aktivitaeten, "/aktivitaeten", "SvelteKit Route", "Unified activity feed (Chronik). Loader: GET /api/dashboard/activity and GET /api/notifications?read=false.")
|
||||
Component(geschichten, "/geschichten and /geschichten/[id]", "SvelteKit Routes", "Story list and detail pages. Loader: GET /api/geschichten?status=PUBLISHED.")
|
||||
@@ -20,8 +21,9 @@ System_Boundary(frontend, "Web Frontend (SvelteKit / SSR)") {
|
||||
}
|
||||
|
||||
Rel(user, personsPage, "Browses family members", "HTTPS / Browser")
|
||||
Rel(personsPage, backend, "GET /api/persons, GET /api/persons/{id}", "HTTP / JSON")
|
||||
Rel(personsPage, backend, "GET /api/persons (filter + page params -> PersonSearchResult), GET /api/persons/{id}", "HTTP / JSON")
|
||||
Rel(personEdit, backend, "GET /api/persons/{id}, PUT /api/persons/{id}, POST /api/persons", "HTTP / JSON")
|
||||
Rel(personReview, backend, "GET /api/persons?provisional=true, PATCH /api/persons/{id}/confirm, DELETE /api/persons/{id}, POST /api/persons/{id}/merge", "HTTP / JSON")
|
||||
Rel(briefwechsel, backend, "GET /api/documents/conversation", "HTTP / JSON")
|
||||
Rel(aktivitaeten, backend, "GET /api/dashboard/activity, GET /api/notifications", "HTTP / JSON")
|
||||
Rel(geschichten, backend, "GET /api/geschichten", "HTTP / JSON")
|
||||
|
||||
@@ -12,7 +12,7 @@ System_Boundary(frontend, "Web Frontend (SvelteKit / SSR)") {
|
||||
Component(adminGroups, "/admin/groups, /admin/groups/[id], /admin/groups/new", "SvelteKit Routes", "Permission group management: create/edit groups and their permission sets.")
|
||||
Component(adminTags, "/admin/tags and /admin/tags/[id]", "SvelteKit Routes", "Tag administration: edit tag hierarchy, merge tags, delete subtrees.")
|
||||
Component(adminOcr, "/admin/ocr and /admin/ocr/[personId]", "SvelteKit Routes", "Global and per-person OCR configuration. Manages script types and triggers sender model training.")
|
||||
Component(adminSystem, "/admin/system", "SvelteKit Route", "System status panel. Triggers Excel/ODS mass import (POST /api/admin/trigger-import). Displays import state.")
|
||||
Component(adminSystem, "/admin/system", "SvelteKit Route", "System status panel. Triggers the canonical import (POST /api/admin/trigger-import). Displays import state.")
|
||||
Component(hilfe, "/hilfe/transkription", "SvelteKit Route", "Static transcription style guide for Kurrent and Sütterlin character recognition. No backend calls.")
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
@startuml db-orm
|
||||
' Schema source: Flyway V1–V60 (excl. V37, V43 — intentionally removed)
|
||||
' Schema as of: V60 (2026-05-06)
|
||||
' Schema source: Flyway V1–V69 (excl. V37, V43 — intentionally removed)
|
||||
' Schema as of: V69 (2026-05-27)
|
||||
' ⚠ This is a versioned snapshot. Update when the schema changes significantly.
|
||||
|
||||
hide circle
|
||||
@@ -88,6 +88,11 @@ package "Documents" {
|
||||
summary : TEXT
|
||||
transcription : TEXT
|
||||
meta_date : DATE
|
||||
meta_date_precision : VARCHAR(16) NOT NULL
|
||||
meta_date_end : DATE
|
||||
meta_date_raw : TEXT
|
||||
sender_text : TEXT
|
||||
receiver_text : TEXT
|
||||
meta_location : VARCHAR(255)
|
||||
meta_document_location : VARCHAR(255)
|
||||
archive_box : VARCHAR(255)
|
||||
@@ -182,6 +187,8 @@ package "Persons" {
|
||||
birth_year : INTEGER
|
||||
death_year : INTEGER
|
||||
family_member : BOOLEAN NOT NULL
|
||||
source_ref : VARCHAR(255) UNIQUE
|
||||
provisional : BOOLEAN NOT NULL
|
||||
}
|
||||
|
||||
entity person_name_aliases {
|
||||
@@ -217,6 +224,7 @@ package "Tags" {
|
||||
name : VARCHAR(255) NOT NULL UNIQUE
|
||||
parent_id : UUID <<FK>>
|
||||
color : VARCHAR(20)
|
||||
source_ref : VARCHAR(255) UNIQUE
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
@startuml db-relationships
|
||||
' Schema source: Flyway V1–V60 (excl. V37, V43 — intentionally removed)
|
||||
' Schema as of: V60 (2026-05-06)
|
||||
' Schema source: Flyway V1–V69 (excl. V37, V43 — intentionally removed)
|
||||
' Schema as of: V69 (2026-05-27)
|
||||
' ⚠ This is a versioned snapshot. Update when the schema changes significantly.
|
||||
' Note: V69 adds columns only (persons.source_ref, tag.source_ref, document
|
||||
' precision/attribution fields); no new FK relationships, so this diagram is unchanged.
|
||||
|
||||
hide circle
|
||||
skinparam linetype ortho
|
||||
|
||||
140
docs/date-label-fixtures.json
Normal file
140
docs/date-label-fixtures.json
Normal file
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"_comment": "Single source of truth for the honest date-label rule set shared by the TS formatDocumentDate (frontend/src/lib/shared/utils/documentDate.ts) and the Java formatTitleDate (backend importing/DocumentTitleFormatter.java). The 'cases' array holds the GERMAN (de) canonical form and is asserted by BOTH suites — that is the Java<->TS drift guard (en-dash vs hyphen, 'ca.' vs 'circa', season words, range collapse). The Java title formatter intentionally renders German server-side (import titles are always German); only the TS UI formatter is locale-aware, so 'localeCases' (en/es month-name output) is asserted by the TS spec ONLY and must NOT be fed to the Java test. Do not edit one side's expectation without editing this file and the relevant test(s). Season->month mapping note: the Python import normalizer (tools/import-normalizer) is the UPSTREAM authority for which representative month a season maps to (4/7/10/1); both formatters mirror it but it sits OUTSIDE this Java<->TS guard, so a normalizer change is not caught here. See issue #666 and the Markus/Sara drift-guard decision.",
|
||||
"cases": [
|
||||
{
|
||||
"name": "DAY renders a full long date",
|
||||
"precision": "DAY",
|
||||
"anchor": "1943-12-24",
|
||||
"end": null,
|
||||
"raw": null,
|
||||
"expected": "24. Dezember 1943"
|
||||
},
|
||||
{
|
||||
"name": "MONTH renders month and year only — never a fabricated day",
|
||||
"precision": "MONTH",
|
||||
"anchor": "1916-06-01",
|
||||
"end": null,
|
||||
"raw": "Juni 1916",
|
||||
"expected": "Juni 1916"
|
||||
},
|
||||
{
|
||||
"name": "SEASON renders the season word from raw",
|
||||
"precision": "SEASON",
|
||||
"anchor": "1916-06-01",
|
||||
"end": null,
|
||||
"raw": "Sommer 1916",
|
||||
"expected": "Sommer 1916"
|
||||
},
|
||||
{
|
||||
"name": "SEASON with null raw derives the season from the anchor month",
|
||||
"precision": "SEASON",
|
||||
"anchor": "1916-04-01",
|
||||
"end": null,
|
||||
"raw": null,
|
||||
"expected": "Frühling 1916"
|
||||
},
|
||||
{
|
||||
"name": "YEAR renders the year only — suppresses month and day",
|
||||
"precision": "YEAR",
|
||||
"anchor": "1916-06-15",
|
||||
"end": null,
|
||||
"raw": null,
|
||||
"expected": "1916"
|
||||
},
|
||||
{
|
||||
"name": "APPROX renders a ca. prefix before the year",
|
||||
"precision": "APPROX",
|
||||
"anchor": "1920-01-01",
|
||||
"end": null,
|
||||
"raw": null,
|
||||
"expected": "ca. 1920"
|
||||
},
|
||||
{
|
||||
"name": "RANGE in the same month collapses the shared month and year",
|
||||
"precision": "RANGE",
|
||||
"anchor": "1917-01-10",
|
||||
"end": "1917-01-11",
|
||||
"raw": null,
|
||||
"expected": "10.–11. Jan. 1917"
|
||||
},
|
||||
{
|
||||
"name": "RANGE across months expands both months, sharing the year",
|
||||
"precision": "RANGE",
|
||||
"anchor": "1917-01-30",
|
||||
"end": "1917-02-02",
|
||||
"raw": null,
|
||||
"expected": "30. Jan. – 2. Feb. 1917"
|
||||
},
|
||||
{
|
||||
"name": "RANGE across a year boundary expands both full dates",
|
||||
"precision": "RANGE",
|
||||
"anchor": "1916-12-30",
|
||||
"end": "1917-01-02",
|
||||
"raw": null,
|
||||
"expected": "30. Dez. 1916 – 2. Jan. 1917"
|
||||
},
|
||||
{
|
||||
"name": "RANGE where end equals start collapses to a single day",
|
||||
"precision": "RANGE",
|
||||
"anchor": "1917-01-10",
|
||||
"end": "1917-01-10",
|
||||
"raw": null,
|
||||
"expected": "10. Jan. 1917"
|
||||
},
|
||||
{
|
||||
"name": "RANGE with a null end renders an open-range indicator, never a fabricated end",
|
||||
"precision": "RANGE",
|
||||
"anchor": "1917-01-10",
|
||||
"end": null,
|
||||
"raw": null,
|
||||
"expected": "ab 10. Jan. 1917"
|
||||
},
|
||||
{
|
||||
"name": "UNKNOWN renders the unknown label regardless of anchor",
|
||||
"precision": "UNKNOWN",
|
||||
"anchor": null,
|
||||
"end": null,
|
||||
"raw": "?",
|
||||
"expected": "Datum unbekannt"
|
||||
}
|
||||
],
|
||||
"localeComment": "TS-only locale parity for the read path (the younger phone audience may use en/es). Asserted ONLY by documentDate.spec.ts — the Java title formatter is German-only by design, so these MUST NOT be fed to DocumentTitleFormatterTest. Each case pins the localized month-name output for DAY and MONTH so a locale regression (e.g. a future de-DE hard-coding) is caught by the drift table, not just by ad-hoc tests.",
|
||||
"localeCases": [
|
||||
{
|
||||
"name": "DAY in English renders the English month name",
|
||||
"precision": "DAY",
|
||||
"anchor": "1943-12-24",
|
||||
"end": null,
|
||||
"raw": null,
|
||||
"locale": "en",
|
||||
"expected": "December 24, 1943"
|
||||
},
|
||||
{
|
||||
"name": "DAY in Spanish renders the Spanish month name",
|
||||
"precision": "DAY",
|
||||
"anchor": "1943-12-24",
|
||||
"end": null,
|
||||
"raw": null,
|
||||
"locale": "es",
|
||||
"expected": "24 de diciembre de 1943"
|
||||
},
|
||||
{
|
||||
"name": "MONTH in English renders the English month name, never a day",
|
||||
"precision": "MONTH",
|
||||
"anchor": "1916-06-01",
|
||||
"end": null,
|
||||
"raw": "Juni 1916",
|
||||
"locale": "en",
|
||||
"expected": "June 1916"
|
||||
},
|
||||
{
|
||||
"name": "MONTH in Spanish renders the Spanish month name, never a day",
|
||||
"precision": "MONTH",
|
||||
"anchor": "1916-06-01",
|
||||
"end": null,
|
||||
"raw": "Juni 1916",
|
||||
"locale": "es",
|
||||
"expected": "junio de 1916"
|
||||
}
|
||||
]
|
||||
}
|
||||
313
docs/import-migration/01-findings-spreadsheet-analysis.md
Normal file
313
docs/import-migration/01-findings-spreadsheet-analysis.md
Normal file
@@ -0,0 +1,313 @@
|
||||
# Spreadsheet Analysis — Findings (2026-05-25)
|
||||
|
||||
Analysis of the **real raw archive** spreadsheets against the current `MassImportService`
|
||||
(`backend/.../importing/MassImportService.java`). Goal: import ~7,600 letter rows + a
|
||||
163-person register, with PDFs to follow.
|
||||
|
||||
Every issue has an ID (`IMP-NN`), severity, evidence, and a proposed approach.
|
||||
|
||||
---
|
||||
|
||||
## 0. Context: how the importer reads a row today
|
||||
|
||||
`MassImportService` reads **sheet index 0** and maps columns by configurable indices
|
||||
(`app.import.col.*`, defaults in the source):
|
||||
|
||||
| Property | Default col | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `colIndex` | 0 | Index (→ filename `<index>.pdf`) |
|
||||
| `colBox` | 1 | Box |
|
||||
| `colFolder` | 2 | Mappe |
|
||||
| `colSender` | 3 | Sender (raw) |
|
||||
| `colReceivers` | 5 | Receivers (raw) |
|
||||
| `colDate` | 7 | Date |
|
||||
| `colLocation` | 9 | Location |
|
||||
| `colTags` | 10 | Tag (single) |
|
||||
| `colSummary` | 11 | Summary |
|
||||
| `colTranscription` | 13 | Transcription |
|
||||
|
||||
These defaults match the **ODS** file exactly (`Index, Box, Mappe, Von, BriefeschreiberIn,
|
||||
An, EmpfängerIn, Datum, Datum Originalformat, Ort, Schlagwort, Inhalt, Zeitlicher Kontext,
|
||||
Transkript` = 14 cols). The ODS was the development target. The new xlsx is a different beast.
|
||||
|
||||
Per-row pipeline: skip if Index blank → derive filename from Index → validate filename →
|
||||
look for file on disk (recursive; metadata-only if absent) → check PDF magic bytes →
|
||||
`importSingleDocument` (upsert by `originalFilename`, dedupe non-placeholders as
|
||||
`ALREADY_EXISTS`). Date parsing is **ISO-only** (`LocalDate.parse`).
|
||||
|
||||
---
|
||||
|
||||
## IMP-01 — New xlsx column layout ≠ importer defaults 🔴 BLOCKER
|
||||
|
||||
The new `…aktuell…xlsx` (sheet `Familienarchiv`, 7,943 rows × 12 cols) has a **denser,
|
||||
different** layout. There is an extra `Datei` column at index 1, and the normalized
|
||||
`Von`/`An`/ISO-`Datum` columns from the ODS **do not exist**.
|
||||
|
||||
| col | New xlsx header | Importer default expects | Result with defaults |
|
||||
| --- | --- | --- | --- |
|
||||
| 0 | Index | Index | ✅ ok |
|
||||
| 1 | **Datei** (path) | Box | ❌ Box ← `..\__scan\W-0001.pdf` |
|
||||
| 2 | Box | Mappe | ❌ Mappe ← `V` |
|
||||
| 3 | Mappe | Sender | ❌ Sender ← `1` |
|
||||
| 4 | BriefeschreiberIn (sender) | — (unused) | ❌ sender ignored |
|
||||
| 5 | EmpfängerIn (receiver) | Receivers | ✅ coincidentally ok |
|
||||
| 6 | Datum des Briefes | — (unused) | ❌ date ignored |
|
||||
| 7 | Ort (location) | Date | ❌ Date ← `Rotterdam` → null |
|
||||
| 8 | Schlagwort (tag) | — (unused) | ❌ tag ignored |
|
||||
| 9 | Inhalt (summary) | Location | ❌ Location ← summary text |
|
||||
| 10 | — | Tag | ❌ empty |
|
||||
| 11 | — | Summary | ❌ empty |
|
||||
| 13 | — | Transcription | ❌ column doesn't exist |
|
||||
|
||||
**Impact:** importing as-is produces almost entirely garbage metadata.
|
||||
|
||||
**Proposed approach (decide with Marcel):**
|
||||
- (a) Re-map via the existing `app.import.col.*` properties — fast, no code. New mapping:
|
||||
`index=0, box=2, folder=3, sender=4, receivers=5, date=6, location=7, tags=8, summary=9`,
|
||||
and there is **no** transcription column (point it past the end or add a "missing column"
|
||||
convention). Caveat: tags land in `colTags` but the real per-letter keywords are in
|
||||
`Inhalt` (col 9) — see IMP-08 note on tags vs summary.
|
||||
- (b) Make the importer **header-driven** (map by header name, not index) so it survives
|
||||
layout drift across files. More robust, needs a code change (→ Gitea issue).
|
||||
|
||||
Recommendation: (b) is the durable fix given we have ≥3 different layouts already.
|
||||
|
||||
---
|
||||
|
||||
## IMP-02 — 90% of dates are free-text the parser can't read 🔴 BLOCKER
|
||||
|
||||
The dates are written **as in the letter**. `parseDate()` only does `LocalDate.parse()`
|
||||
(ISO `yyyy-MM-dd`), so anything non-ISO becomes `null`.
|
||||
|
||||
Of **7,319** rows with a date value (col 6):
|
||||
|
||||
| kind | count | parses today? |
|
||||
| --- | --- | --- |
|
||||
| Real Excel date cells (→ ISO via POI) | 748 | ✅ |
|
||||
| Free-text date strings | 6,571 | ❌ → null |
|
||||
|
||||
→ **90% of dated rows lose their date.** (623 rows have no date at all.)
|
||||
|
||||
Observed free-text formats (counts approximate, from col 6):
|
||||
|
||||
| Format | Count | Examples |
|
||||
| --- | --- | --- |
|
||||
| `D.M.YY` | 1,338 | `11.10.08`, `13.5.09` |
|
||||
| `D.RomanMonth.YY/YYYY` | ~1,527 | `22.III.18`, `19.XII.1954`, `1.III.27` |
|
||||
| `D.Month YYYY` | 950 | `6.März 1888`, `9.März 1888` (note: **no space** after the dot) |
|
||||
| `D.M.YYYY` | 358 | `15.2.1888`, `7.3.1888` |
|
||||
| Approximate / unknown | 146 | `?`, `13.7.18?`, `17.Nov (?) 1887`, `13.Januar ? 1907` |
|
||||
| `Month YYYY` / season / holiday | 41+27 | `Mai 1895`, `Herbst 1913`, `Pfingsten 1922`, `Ostern 1890` |
|
||||
| `YYYY` only | 17 | `1905`, `1949` |
|
||||
| `D.M.` no year | 10 | `8.9.`, `14.3.` |
|
||||
| Ranges | 5+ | `8.1.1916 - 15.3.1916`, `1881/82`, `1945/46?` |
|
||||
| Abbrev/English months, no space | many | `29.Sept.1891`, `10.Oct.95`, `9.December1889`, `18.Dez.1916` |
|
||||
| Slash separator | ~315 | `2/2. 18`, `17/6. 1916`, `10/4. 1917` |
|
||||
| English `Month D. YYYY` | several | `April 12. 1922`, `Oct.5. 1916`, `Mai 23. 1917` |
|
||||
| Trailing notes | 5+ | `26.4.1888, 2. Brief`, `31.8.1888,2.Brief` |
|
||||
| 3-digit year (typo) | 107 | `30.1.889` (→ 1889), `4.3.1023` (in person file → 1923) |
|
||||
| Day-range within month | several | `7./8. Sept.1923` |
|
||||
|
||||
**Proposed approach:** build a tolerant German/historical date parser (→ Gitea issue, it's
|
||||
a code change). Requirements:
|
||||
- Numeric `D.M.YY[YY]` and `D/M. YY[YY]` (slash = dot).
|
||||
- Roman-numeral months (`I`–`XII`).
|
||||
- German + English month names, full + abbreviated, with/without separating space
|
||||
(`März`, `Sept.`, `Dez`, `December`, `Oct.`).
|
||||
- 2-digit and 3-digit year normalization (`08`→1908? needs a century rule; `889`→1889).
|
||||
- Partial dates → store what's known. The schema only has a single `documentDate
|
||||
LocalDate`; **decide** whether to (i) store first-of-month/year, (ii) add a
|
||||
`datePrecision` enum + `dateOriginal` text column, or (iii) keep raw text in a new
|
||||
`documentDateRaw` field and leave `documentate` null when imprecise. Recommendation:
|
||||
preserve the **original string** always (new column) + best-effort parsed date +
|
||||
precision flag, so nothing is lost and the UI can show "ca. 1916".
|
||||
- Unparseable/approximate (`?`, `Herbst 1913`) → keep raw, leave parsed date null, **do
|
||||
not drop the row**.
|
||||
|
||||
**Cross-check:** even after IMP-01 is fixed so the date column is read, IMP-02 still bites.
|
||||
Both must be solved before a real import.
|
||||
|
||||
---
|
||||
|
||||
## IMP-03 — New xlsx has no normalized/ISO date or name columns 🔴 BLOCKER
|
||||
|
||||
The ODS had helper columns the importer relied on: `Von`/`An` (normalized names) and
|
||||
`Datum` (ISO) alongside `Datum Originalformat`. The new xlsx has **only the raw**
|
||||
`BriefeschreiberIn` / `EmpfängerIn` / `Datum des Briefes`. So:
|
||||
- Names must be parsed from raw strings (PersonNameParser already does receivers; **sender
|
||||
is taken raw, never split** — fine for senders, which are single, but no normalization).
|
||||
- Dates must be parsed from raw (IMP-02).
|
||||
|
||||
This is the root reason IMP-01/02 exist: the new file is the *uncurated* source, not the
|
||||
hand-normalized ODS. Tie any importer redesign to this reality — we will not get clean
|
||||
helper columns in the 7k-row file.
|
||||
|
||||
---
|
||||
|
||||
## IMP-04 — Person register not imported at all 🟠 MAJOR
|
||||
|
||||
`Personendatei 2.xlsx` → sheet `Tabelle1`, **163 people**, columns:
|
||||
`Generation, Familienname, Vorname, geb als (maiden), Geburtsdatum, Geburtsort,
|
||||
Todesdatum, Sterbeort, verheiratet mit, Bemerkung`.
|
||||
|
||||
Today `MassImportService` has **no person-register import**. Persons are only
|
||||
auto-created as bare aliases from the document sender/receiver strings
|
||||
(`personService.findOrCreateByAlias`). All this rich genealogical data is unused:
|
||||
- birth/death dates + places,
|
||||
- maiden names (the key to dedup — see IMP-05),
|
||||
- `verheiratet mit` (marriage links → `PersonRelationship` domain),
|
||||
- `Bemerkung` relationship hints (`"Schwester v Marie Cram"`, `"Nichte von Herbert"`),
|
||||
- `Generation` (G 1–G 4),
|
||||
- nicknames in quotes (`"Tante Lolly"`).
|
||||
|
||||
Data-quality notes in this file too: multi-value `Vorname` (`Charlotte,Meta,Jacobi`);
|
||||
mixed Excel-date vs text dates; typos (`4.3.1023`); missing-day dates (`.12.1955`);
|
||||
trailing spaces (`30.8.1862 `).
|
||||
|
||||
**Proposed approach:** a separate **Person import** (→ Gitea issue). Order matters: import
|
||||
persons *first* so documents can link to real people instead of creating alias stubs.
|
||||
Use `geb als` + `verheiratet mit` to pre-build the alias/relationship graph.
|
||||
|
||||
---
|
||||
|
||||
## IMP-05 — Name variations create duplicate Persons 🟠 MAJOR
|
||||
|
||||
The same person appears under several surface forms across the document sheet:
|
||||
- `Eugenie Müller` (151) vs `Eugenie de Gruyter` (452) — maiden vs married.
|
||||
- `Clara Cram` (sender 1,284) vs `Clara de Gruyter` (455) vs `Clara de Gruyter sen.` (66).
|
||||
- `Walter de Gruyter` (589) vs bare `Walter` (78).
|
||||
|
||||
`findOrCreateByAlias` keys on the raw string, so each variant becomes (or matches) a
|
||||
distinct alias and likely a **distinct Person**. Result: fragmented person records,
|
||||
broken Briefwechsel pairing, wrong stats.
|
||||
|
||||
**Proposed approach:** drive dedup from the register's `geb als` column (IMP-04) —
|
||||
`Eugenie de Gruyter geb Müller` tells us the two strings are one person. Build an alias
|
||||
map (married ↔ maiden ↔ nickname) before/while importing documents. This is partly data
|
||||
(an alias mapping table/sheet) and partly code (consume it). Likely a Gitea issue once the
|
||||
mapping format is decided.
|
||||
|
||||
945 distinct sender strings / 274 distinct receiver strings — expect a long-tail of
|
||||
variants to reconcile. Don't try to be perfect on the first pass; get the high-frequency
|
||||
names right.
|
||||
|
||||
---
|
||||
|
||||
## IMP-06 — 93 data rows with blank Index are silently dropped 🟠 MAJOR
|
||||
|
||||
`processRows` does `if (index.isBlank()) continue;`. **93 rows** have a blank Index but
|
||||
carry other data (sender/receiver/date/etc.). These are silently skipped — they don't even
|
||||
appear in the `skippedFiles` report (that list only covers rows that *had* an index but
|
||||
failed file checks).
|
||||
|
||||
**Proposed approach:** before import, triage these 93 rows — are they continuation rows,
|
||||
section markers, or genuine letters missing an ID? At minimum, surface a count/warning so
|
||||
nothing vanishes unnoticed. Possibly a small importer change to report blank-index skips.
|
||||
|
||||
---
|
||||
|
||||
## IMP-07 — 43 duplicate Index values 🟡 MINOR
|
||||
|
||||
43 Index values repeat (e.g. `W-0388`, `Eu-0332`, `C-0234`, `C-0235`, `C-0236`, `J-0175`).
|
||||
Since the filename is derived from Index, the importer's upsert keys both rows on the same
|
||||
`originalFilename`: the second occurrence is treated as `ALREADY_EXISTS` (if the first
|
||||
isn't a placeholder) and **its metadata is lost**, or it overwrites a placeholder.
|
||||
|
||||
**Proposed approach:** list the 43 duplicates, check whether they're true duplicates or
|
||||
two distinct letters that share an ID by mistake. Fix in the source data, or extend the ID
|
||||
scheme. Data task first; software only if the ID scheme must change.
|
||||
|
||||
---
|
||||
|
||||
## IMP-08 — Section/title rows interleaved with data 🟡 MINOR
|
||||
|
||||
Row 2 of the sheet is a section header sitting only in the sender column
|
||||
(`Brautbriefe von Walter der Gruyter an Eugenie Müller`) with a blank Index — caught by the
|
||||
blank-Index skip (overlaps IMP-06). There may be more such banners scattered through 7,943
|
||||
rows. Also relevant: the per-letter **keywords live in `Inhalt` (col 9)** as comma-joined
|
||||
values (`Tilburg,Verwandschaft`, `poetisch,Reise nach Breda`), while `Schlagwort` (col 8)
|
||||
holds a single broad tag (`Brautbriefe`). The importer only takes **one** tag column —
|
||||
decide which column feeds tags vs summary, and whether to split comma-lists into multiple
|
||||
tags.
|
||||
|
||||
**Proposed approach:** scan for rows where Index is blank but other cells are set (already
|
||||
have the count: relates to the 93 in IMP-06). Confirm tag vs summary column choice with
|
||||
Marcel.
|
||||
|
||||
---
|
||||
|
||||
## IMP-09 — Index ↔ Datei filename mismatches 🟡 MINOR
|
||||
|
||||
The `Datei` column (col 1) holds explicit relative paths (`..\__scan\W-0001.pdf`) but they
|
||||
don't always agree with the Index. Example: row 20 has Index `W-0010x` but Datei
|
||||
`..\__scan\W-0011x.pdf`. The importer derives the filename from **Index**, so it will look
|
||||
for `W-0010x.pdf` and may miss the actual scan. (Note: the `Datei` paths themselves are
|
||||
Windows-style with `\` and `..` and would be **rejected** by `isValidImportFilename` if anyone
|
||||
tried to use that column directly — 7,623 rows use backslashes, 7,455 contain `..`.)
|
||||
|
||||
**Proposed approach:** when the PDFs arrive, reconcile Index-derived names against actual
|
||||
filenames; produce a mismatch report. Keep deriving from Index (stable IDs) but flag
|
||||
disagreements. Mostly a data/QA task.
|
||||
|
||||
---
|
||||
|
||||
## IMP-10 — `x`-suffix rows (letter backsides / enclosures) 🟡 MINOR
|
||||
|
||||
**42 rows** have an `x`-suffixed Index (`W-0001x`, `W-0002x`, …). They're sparse — typically
|
||||
only Index + Datei + sender + receiver, no box/folder/date. They appear to be the reverse
|
||||
side or an enclosure of the preceding letter. The importer treats each as an independent
|
||||
Document, and the `metadataComplete` heuristic flags them complete as soon as a sender is
|
||||
present (date/box/folder all missing).
|
||||
|
||||
**Proposed approach:** decide whether `x` rows should be (a) separate documents, (b) extra
|
||||
pages/files attached to their parent, or (c) skipped. Affects both the data model and the
|
||||
`metadataComplete` heuristic. Discuss with Marcel.
|
||||
|
||||
---
|
||||
|
||||
## IMP-11 — Multi-receiver separators include bare `u` / `u.` 🟡 MINOR
|
||||
|
||||
`PersonNameParser.parseReceivers` already handles ` und `, ` u `, `//`, `geb.`,
|
||||
parenthesised shared surnames, and `Familie` filtering — good. But the real data also uses
|
||||
the abbreviation in forms the top-receivers list shows are common:
|
||||
`Eugenie u Walter de Gruyter` (230), `Herbert u Clara` (94), `Juan u Marie Cram` (75),
|
||||
and space-joined pairs like `Ella Anita` (79) that may be two people.
|
||||
Raw separator tally on receivers: ` und ` ×70, `,` ×11, `;` ×2, `/` ×1 — plus the many ` u `
|
||||
cases above. Senders are **not** parsed at all (taken raw), which is fine unless a sender
|
||||
cell ever holds two names.
|
||||
|
||||
**Proposed approach:** add `MassImportServiceTest` cases for the real-world strings above;
|
||||
extend the parser only where it actually fails. `Ella Anita`-style space-joined pairs are
|
||||
ambiguous — likely leave as one person unless the register says otherwise (ties to IMP-05).
|
||||
|
||||
---
|
||||
|
||||
## IMP-12 — Importer reads only the first sheet, no validation 🟡 MINOR
|
||||
|
||||
`readXlsx` does `workbook.getSheetAt(0)`. For the new xlsx that's `Familienarchiv` (✅), but
|
||||
the file also contains `Inhaltsverzeichnis grob`, `Inhaltsverzeichnis WdG`, `Tabelle4`.
|
||||
There is no header validation: if the wrong file/sheet is dropped in `/import`, the importer
|
||||
will happily map columns positionally and import nonsense. Also `findSpreadsheetFile()` picks
|
||||
the **first** spreadsheet found in `/import` — with three spreadsheets present there today,
|
||||
which one wins is filesystem-order-dependent.
|
||||
|
||||
**Proposed approach:** (a) validate the header row against expected names before importing;
|
||||
(b) make the target sheet/file explicit (config or header match) rather than "first found".
|
||||
Ties into the header-driven mapping in IMP-01(b).
|
||||
|
||||
---
|
||||
|
||||
## Summary of recommended sequencing
|
||||
|
||||
1. **Decide the importer mapping strategy** (IMP-01): positional re-config vs header-driven.
|
||||
Header-driven is the durable choice and unblocks IMP-03/12.
|
||||
2. **Build the tolerant date parser** (IMP-02) with original-string preservation + precision.
|
||||
3. **Import the Person register first** (IMP-04) and build the alias/marriage graph,
|
||||
which feeds person dedup (IMP-05).
|
||||
4. **Then import documents**, with reporting for blank-index (IMP-06), duplicates (IMP-07),
|
||||
and section rows (IMP-08).
|
||||
5. **Reconcile files** when the ~7,000 PDFs arrive (IMP-09), and decide `x`-row semantics
|
||||
(IMP-10).
|
||||
|
||||
Code-change items (→ Gitea issues when we get there): IMP-01(b), IMP-02, IMP-04, IMP-05
|
||||
(consume side), IMP-06 reporting, IMP-12. Pure-data items stay in this folder.
|
||||
417
docs/import-migration/02-normalization-spec.md
Normal file
417
docs/import-migration/02-normalization-spec.md
Normal file
@@ -0,0 +1,417 @@
|
||||
# Spec — Import Normalizer
|
||||
|
||||
> Authored in the voice of **"Elicit"**, requirements engineer (see
|
||||
> `.claude/personas/req_engineer.md`). This is a requirements artifact: it states
|
||||
> *what* the normalizer must do and *how we'll know it's done*, in problem/behaviour
|
||||
> language. Technology choices already made during brainstorming (Python, openpyxl,
|
||||
> overrides-and-rerun) are recorded as **constraints**, not re-litigated here.
|
||||
|
||||
- **Status:** Draft for review
|
||||
- **Date:** 2026-05-25
|
||||
- **Related:** [`01-findings-spreadsheet-analysis.md`](./01-findings-spreadsheet-analysis.md) (issues `IMP-01..12`), [`README.md`](./README.md)
|
||||
- **Scope boundary:** This spec covers the **offline normalizer** that turns the raw
|
||||
spreadsheets into a clean, canonical dataset + review artifacts. Wiring the canonical
|
||||
contract into the Java `MassImportService` and the `Document`/`Person` model is **Phase 2**
|
||||
and gets its own spec. This spec only *defines the contract* Phase 2 must satisfy.
|
||||
|
||||
---
|
||||
|
||||
## 1. Project Brief
|
||||
|
||||
**Vision.** Turn the family's human-curated, free-form archive spreadsheets into a clean,
|
||||
canonical dataset that imports deterministically — without hand-editing thousands of rows
|
||||
and without losing the historical nuance of how things were originally written.
|
||||
|
||||
**Problem.** The real archive (`…aktuell…xlsx`, 7,943 rows) and the person register
|
||||
(`Personendatei 2.xlsx`, 163 people) were authored for humans to read, not machines to
|
||||
import. Dates are written as they appeared in each letter (≈90% unparseable by the current
|
||||
importer), the column layout differs from what the importer expects, and the same person
|
||||
appears under many names. Importing as-is produces garbage (see `IMP-01..12`).
|
||||
|
||||
**Goal (measurable).**
|
||||
- G1 — After the automated pass, **≤ 5%** of dated rows remain `UNKNOWN`; after the
|
||||
overrides-iteration loop, **≤ 0.5%**.
|
||||
- G2 — **100%** of source rows are represented in the canonical output or in a review file —
|
||||
*zero silent drops*.
|
||||
- G3 — **100%** of original values (raw date string, raw name string, source row number)
|
||||
are preserved.
|
||||
- G4 — A full run over the current inputs completes in **< 60 s** on the dev laptop and is
|
||||
**content-deterministic** when re-run with unchanged inputs+overrides: identical canonical
|
||||
cell matrices and identical review-file contents. (Workbook metadata is pinned; literal xlsx
|
||||
byte-identity is not guaranteed because the zip container stores entry metadata.)
|
||||
|
||||
**Primary actor.** Marcel — solo owner & data steward (tech comfort 4/5). Also: a future
|
||||
agent re-running the pipeline; and the `MassImportService` as the downstream consumer.
|
||||
|
||||
**Non-Goals (explicitly out of scope).**
|
||||
- NG1 — Changing `MassImportService` or the DB schema (that is Phase 2).
|
||||
- NG2 — Uploading/attaching the ~7,000 PDFs (they arrive later; import matches by `index`).
|
||||
- NG3 — A GUI. The interface is spreadsheets in, CSVs out, an overrides file hand-edited.
|
||||
- NG4 — Perfect genealogical reconstruction. We resolve confidently-matchable people; the
|
||||
long tail stays as provisional persons.
|
||||
- NG5 — OCR/transcription content (the new xlsx has no transcription column).
|
||||
|
||||
**Key assumptions.** (A1) Sheet `Familienarchiv` is the document source of truth.
|
||||
(A2) Archive date range is **1873–1957** (drives the 2-digit-year century rule).
|
||||
(A3) `index` is the stable document key and the basis for future PDF matching.
|
||||
(A4) `Schlagwort` is a broad tag; `Inhalt` is a short summary/topic.
|
||||
|
||||
**Risks.** (R1) 2-digit/partial dates are genuinely ambiguous → mitigated by precision flag
|
||||
+ overrides. (R2) Name matching false-positives merge distinct people → mitigated by
|
||||
conservative matching + review before merge. (R3) Source spreadsheet may be re-exported with
|
||||
layout drift → mitigated by header-name-based mapping, not fixed indices.
|
||||
|
||||
---
|
||||
|
||||
## 2. Personas
|
||||
|
||||
**Marcel — Data Steward.** Role: solo owner of Familienarchiv. Context: holds the complete
|
||||
raw archive; PDFs follow. Tech comfort: 4/5 (semi-technical, reads CSV/spreadsheets fluently,
|
||||
not keen to hand-edit 7,600 rows). Primary goal: a clean, importable dataset he trusts.
|
||||
Frustrations: dates in ~20 formats; one ancestor under 4 name variants. **JTBD:** *"When I
|
||||
have raw, human-curated archive spreadsheets, I want to transform them into a clean importable
|
||||
dataset without losing how things were originally written, so I can load the archive and keep
|
||||
correcting edge cases as they surface."*
|
||||
|
||||
**The Returning Agent.** Role: a future assistant session resuming the work. Goal: re-run the
|
||||
pipeline deterministically and understand exactly what still needs human input. **JTBD:**
|
||||
*"When I pick this up cold, I want one command and a clear residue report, so I can continue
|
||||
without re-deriving context."*
|
||||
|
||||
---
|
||||
|
||||
## 3. Constraints & Decisions Already Made
|
||||
|
||||
These were settled during brainstorming and are fixed inputs to the requirements below.
|
||||
|
||||
| # | Decision | Rationale |
|
||||
| --- | --- | --- |
|
||||
| C1 | **New canonical layout** with explicit headers (not the old positional ODS shape). | Fits the new data; importer becomes header-driven in Phase 2. |
|
||||
| C2 | Dates stored as **parsed (nullable) + raw + precision**. | Historical archive; never lose the original; enable "ca. 1916". |
|
||||
| C3 | **Include person resolution** (register + alias/marriage map → canonical persons) in this effort. | Maiden-name dedup needs the register. |
|
||||
| C4 | **Overrides-file + re-run** loop for residue. | Deterministic, diffable, repeatable. |
|
||||
| C5 | Implementation: **Python 3.12 + openpyxl**, standalone tool at `tools/import-normalizer/`. | Fast iteration; no Spring rebuild / coverage gate on transform code. |
|
||||
| C6 | Century rule for archive **1873–1957**: 2-digit `00–57`→`19YY`, `73–99`→`18YY`, `58–72`→**flag**; 3-digit `DDD`→`1DDD`; never 20xx. | Stated by Marcel. Boundaries live in config. |
|
||||
| C7 | `Schlagwort`→tag, `Inhalt`→summary. | Matches importer's existing semantics. |
|
||||
| C8 | Non-register correspondents become **provisional persons**. | ~945 distinct sender strings vs 163 register people. |
|
||||
|
||||
---
|
||||
|
||||
## 4. Functional Requirements
|
||||
|
||||
Each requirement has a stable ID. User stories use Connextra + Given-When-Then; system rules
|
||||
use EARS. Traceability to findings in §8.
|
||||
|
||||
### 4.1 Ingest & layout (`FR-INGEST`, `FR-MAP`)
|
||||
|
||||
**US-MAP-01** — *As the data steward, I want each source column mapped to a named canonical
|
||||
field regardless of its position, so a re-exported spreadsheet with shifted columns still
|
||||
imports correctly.*
|
||||
- AC1 — Given the `Familienarchiv` sheet, when the normalizer reads the header row, then it
|
||||
maps columns by **header name** (not fixed index) to the canonical fields.
|
||||
- AC2 — Given a header the normalizer does not recognise, when it runs, then it records the
|
||||
unknown header in `review/summary.txt` and continues (does not crash).
|
||||
- AC3 — Given a required source header is **absent**, when it runs, then it aborts with a
|
||||
clear message naming the missing header (fail loud, before producing partial output).
|
||||
|
||||
- **REQ-INGEST-01** — The normalizer shall read only the `Familienarchiv` sheet of the
|
||||
document workbook and the `Tabelle1` sheet of the person workbook.
|
||||
- **REQ-MAP-01** — Header matching shall be case-insensitive and tolerant of internal
|
||||
multiple spaces (e.g. `"Datum des Briefes"`).
|
||||
|
||||
### 4.2 Row triage (`FR-TRIAGE`) — resolves IMP-06, IMP-07, IMP-08
|
||||
|
||||
**US-TRIAGE-01** — *As the data steward, I want rows that have data but no index surfaced
|
||||
rather than dropped, so I never lose a letter silently.*
|
||||
- AC1 — Given a row whose `index` is blank but which has any other non-empty cell, when the
|
||||
normalizer runs, then that row is written to `review/blank-index-rows.csv` with its source
|
||||
row number and is **not** emitted as a canonical document.
|
||||
- AC2 — Given a fully empty row, when it runs, then the row is skipped and counted (not
|
||||
reported as an anomaly).
|
||||
|
||||
- **REQ-TRIAGE-01** — If two or more rows resolve to the same `index`, then the normalizer
|
||||
shall emit all of them to `review/duplicate-index.csv` and mark each canonical row
|
||||
`needs_review = duplicate_index` (it shall **not** silently drop either).
|
||||
- **REQ-TRIAGE-02** — Where a row is identified as a section/banner row (blank index, text
|
||||
only in a name column), the normalizer shall classify it as such in the blank-index report.
|
||||
- **REQ-TRIAGE-03** — Rows whose `index` ends in `x` (a transcription/back-side of the base
|
||||
letter, not yet independently mappable) shall be **skipped** — not emitted as a canonical
|
||||
document — and written to `review/skipped-x-suffix.csv` with their source row and base index
|
||||
(`index` minus the trailing `x`), so they can be linked in a later pass. (Resolves IMP-10.)
|
||||
|
||||
### 4.3 Date normalization (`FR-DATE`) — resolves IMP-02, IMP-03
|
||||
|
||||
**US-DATE-01** — *As the data steward, I want every date interpreted as precisely as the
|
||||
source allows, with the original always kept, so I can sort the archive and still see what the
|
||||
letter actually said.*
|
||||
- AC1 — Given a parseable date, when normalized, then `date_iso` holds the best-effort ISO
|
||||
date, `date_raw` holds the verbatim source string, and `date_precision` ∈
|
||||
`{DAY, MONTH, SEASON, YEAR, RANGE, APPROX, UNKNOWN}`.
|
||||
- AC2 — Given an unparseable date, when normalized, then `date_iso` is empty,
|
||||
`date_precision = UNKNOWN`, `date_raw` is preserved, and the value appears in
|
||||
`review/unparsed-dates.csv`.
|
||||
- AC3 — Given the same `date_raw` appears in `overrides/dates.csv`, when normalized, then the
|
||||
override's `(iso, precision)` wins over the automatic parse.
|
||||
|
||||
- **REQ-DATE-01** — The parser shall accept, at minimum, these forms (see §10 examples):
|
||||
Excel/ISO; `D.M.YYYY`/`D.M.YY`; `D/M. YY[YY]` (slash treated as dot); Roman-numeral months
|
||||
`I–XII`; German + English month names, full and abbreviated, with or without a separating
|
||||
space; `Month YYYY`; season/holiday + year; bare `YYYY`; and start-anchored ranges.
|
||||
- **REQ-DATE-02** — Precision shall be assigned by what is known: full day → `DAY`; month+year
|
||||
→ `MONTH` (day = 1); a **named feast/holiday + year** → resolved to its **actual calendar
|
||||
date for that year** → `DAY`; a **season + year** → representative mid-season month (day = 1)
|
||||
→ `SEASON`; year only → `YEAR` (month = Jan, day = 1); a range → start date + `RANGE`; a
|
||||
value carrying an uncertainty marker (`?`, `um`, `ca`, `circa`) → `APPROX` with best-effort date.
|
||||
- **REQ-DATE-03** — Two-digit and three-digit years shall be expanded per **C6**; a 2-digit
|
||||
year in `58–72` shall yield `UNKNOWN` + a review entry rather than a guess.
|
||||
- **REQ-DATE-04** — Trailing editorial notes (e.g. `", 2. Brief"`) shall be stripped before
|
||||
parsing and preserved (kept within `date_raw`; not invented into the date).
|
||||
- **REQ-DATE-05** — The parser shall be pure and side-effect-free so it can be unit-tested in
|
||||
isolation (see NFR-TEST-01).
|
||||
- **REQ-DATE-06** — **Movable feasts are never mapped to a fixed month**; they shall be
|
||||
computed per year from Easter (Gauss/Butcher computus): Karfreitag = Easter−2, Ostern =
|
||||
Easter Sunday, Himmelfahrt = Easter+39, Pfingst(sonntag) = Easter+49, Pfingstmontag =
|
||||
Easter+50, Fronleichnam = Easter+60, 1.–4. Advent = the 4th…1st Sunday before 25 Dec. Fixed
|
||||
feasts use a lookup table (Neujahr=01-01, Heiligabend=12-24, Weihnachten=12-25,
|
||||
Silvester=12-31, …). Seasons map to representative months: Frühling/Frühjahr=Apr, Sommer=Jul,
|
||||
Herbst=Oct, Winter=Jan. The feast/season tables and Easter algorithm live in `config.py`
|
||||
(NFR-MAINT-01).
|
||||
- **REQ-DATE-07** — **Intra-month day ranges carry an end day; half-resolved ranges are
|
||||
flagged.** For a day range like `7./8. Sept.1923`, `date_iso` holds the start day, the end
|
||||
day is resolved against the shared month/year into `date_end`, and `date_precision` =
|
||||
`RANGE`. If the **start** parses but the **end day is impossible** (e.g. `10./40.1.1917`),
|
||||
the row keeps the start and `RANGE` precision, leaves `date_end` **empty**, and is flagged
|
||||
`needs_review = range_end_unparsed` — the unparseable end is dropped honestly (surfaced for
|
||||
review), never silently invented or clamped. A `RANGE` row **may** therefore legitimately
|
||||
have an empty `date_end`; the importer must treat `date_end` as optional even on a `RANGE`.
|
||||
|
||||
### 4.4 Person resolution & dedup (`FR-PERS`, `FR-DEDUP`) — resolves IMP-04, IMP-05, IMP-11
|
||||
|
||||
**US-PERS-01** — *As the data steward, I want the genealogical register turned into canonical
|
||||
people with all their known facts, so documents can link to real persons.*
|
||||
- AC1 — Given a register row, when parsed, then a canonical person is produced with
|
||||
`person_id`, name parts, `maiden_name`, birth/death (parsed + raw + place), spouse,
|
||||
generation, nickname, notes — applying the same date rules as §4.3 to birth/death dates.
|
||||
- AC2 — Given multi-value given names (`"Charlotte,Meta,Jacobi"`), when parsed, then the
|
||||
primary given name is the first; the remainder are retained as additional names/aliases.
|
||||
|
||||
**US-PERS-02** — *As the data steward, I want each sender/receiver string matched to a
|
||||
canonical person where possible and never dropped otherwise, so the correspondence graph is
|
||||
complete.*
|
||||
- AC1 — Given a sender/receiver string, when resolved, then it maps to a register
|
||||
`person_id` via the alias index (exact → normalized/casefold → conservative fuzzy).
|
||||
- AC2 — Given no confident match, when resolved, then a **provisional person** is created from
|
||||
the cleaned string, linked, and listed in `review/unmatched-names.csv` (occurrence count +
|
||||
example source rows).
|
||||
- AC3 — Given the string appears in `overrides/names.csv`, when resolved, then it maps to the
|
||||
specified `person_id` (override wins).
|
||||
- AC4 — Given a multi-person receiver cell (`"Eugenie u Walter de Gruyter"`, `"Herbert u
|
||||
Clara"`, `"…//…"`, `"Hedi und Tutu (Gruber)"`), when resolved, then it is split into
|
||||
individual people, each resolved independently; ambiguous space-joined pairs
|
||||
(`"Ella Anita"`) are emitted to `review/ambiguous-receivers.csv` rather than guessed.
|
||||
|
||||
- **REQ-DEDUP-01** — The alias index shall be derived from the register: canonical
|
||||
"First Last", maiden form (`geb als`), spouse-surname married form, nickname, and
|
||||
first-name-only **only when unambiguous** across the register.
|
||||
- **REQ-DEDUP-02** — The normalizer shall not merge two distinct strings into one person on
|
||||
fuzzy similarity alone above a configured threshold without the match being reported; merges
|
||||
must be auditable.
|
||||
- **REQ-PERS-01** — Sender cells shall be parsed for multi-person content using the same rules
|
||||
as receiver cells (today the importer parses only receivers — IMP-11).
|
||||
|
||||
### 4.5 Overrides & idempotency (`FR-OVR`) — supports the iteration loop
|
||||
|
||||
- **REQ-OVR-01** — When the normalizer runs, then it shall load `overrides/dates.csv` and
|
||||
`overrides/names.csv` if present and apply them; absence of either file shall not be an error.
|
||||
- **REQ-OVR-02** — While overrides are unchanged and inputs are unchanged, re-running shall
|
||||
produce **byte-identical** canonical outputs and review files (NFR-IDEM-01).
|
||||
- **REQ-OVR-03** — Each override application shall be counted in `review/summary.txt` (how many
|
||||
dates/names were resolved by override vs automatically).
|
||||
|
||||
### 4.6 Canonical output & provenance (`FR-OUT`, `FR-PROV`) — resolves IMP-01, IMP-09, IMP-12
|
||||
|
||||
- **REQ-OUT-01** — The normalizer shall write `out/canonical-documents.xlsx` and
|
||||
`out/canonical-persons.xlsx` with the headered schemas in §6.
|
||||
- **REQ-PROV-01** — Every canonical document row shall carry `source_row` (1-based row number
|
||||
in the source sheet) so any value can be traced back to the original.
|
||||
- **REQ-PROV-02** — Every canonical row shall carry a `needs_review` field listing zero or more
|
||||
flags (`duplicate_index`, `unparsed_date`, `unmatched_sender`, `unmatched_receiver`,
|
||||
`index_file_mismatch`, …) so the import and the UI can foreground uncertain data.
|
||||
- **REQ-OUT-02** — Where the source `Datei` path disagrees with the index-derived filename
|
||||
(IMP-09), the normalizer shall record the discrepancy in `review/index-file-mismatch.csv`
|
||||
and flag the row; it shall **not** alter the `index` (the stable key).
|
||||
|
||||
---
|
||||
|
||||
## 5. Non-Functional Requirements
|
||||
|
||||
| ID | Category | Requirement (measurable) |
|
||||
| --- | --- | --- |
|
||||
| NFR-DATA-01 | Data integrity | 100% of source rows are accounted for in output **or** a review file; 100% of original date/name strings preserved verbatim. |
|
||||
| NFR-IDEM-01 | Determinism | Identical inputs + overrides ⇒ identical *logical* output across runs/machines: identical canonical cell matrices and review-file contents. Workbook `created`/`modified` metadata is pinned to a constant; ordering of all generated rows/aliases is stable (no set-iteration leakage). xlsx byte-identity is explicitly not required — determinism is asserted on content. |
|
||||
| NFR-PERF-01 | Performance | Full run over 7,943 doc rows + 163 person rows completes in < 60 s on the dev laptop. |
|
||||
| NFR-ACCUR-01 | Date accuracy | After automated pass, `UNKNOWN` dates ≤ 5% of dated rows; after overrides iteration, ≤ 0.5%. |
|
||||
| NFR-ACCUR-02 | Name coverage | Every sender/receiver occurrence yields a linked person (register or provisional); 0 dropped. |
|
||||
| NFR-I18N-01 | Encoding | UTF-8 end-to-end; German diacritics and ß round-trip with no mojibake in any output. |
|
||||
| NFR-TEST-01 | Testability | `dates.py` and `persons.py` have pytest tests covering every format/alias category in §10 with real examples from the archive. |
|
||||
| NFR-MAINT-01 | Maintainability | Column-name map, century boundaries, season→month map, and fuzzy threshold live in `config.py`, not inline in logic. |
|
||||
| NFR-OBSERV-01 | Observability | `review/summary.txt` reports per-run stats: rows in, documents out, dates by precision, names matched vs provisional, overrides applied, anomalies by type. |
|
||||
| NFR-SAFETY-01 | Source safety | Source workbooks are opened read-only and never written. |
|
||||
|
||||
---
|
||||
|
||||
## 6. Data Dictionary (canonical contract)
|
||||
|
||||
This is the contract Phase 2 (the importer) must consume. Field-level, format-level — not a
|
||||
DB schema.
|
||||
|
||||
### 6.1 `canonical-documents.xlsx`
|
||||
|
||||
| Field | Required | Format / values | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `index` | yes | string | Stable key; basis for PDF matching. |
|
||||
| `file` | no | string | verbatim `Datei` value (e.g. `H-0730.pdf`); carried through for the importer to link the scanned PDF. |
|
||||
| `box` | no | string | from `Box`. |
|
||||
| `folder` | no | string | from `Mappe`. |
|
||||
| `sender_person_id` | no | person_id | resolved; empty if no sender. |
|
||||
| `sender_name` | no | string | canonical display name (or cleaned raw if provisional). |
|
||||
| `receiver_person_ids` | no | `id\|id\|…` | pipe-separated. |
|
||||
| `receiver_names` | no | `name\|name\|…` | pipe-separated, aligned with ids. |
|
||||
| `date_iso` | no | `YYYY-MM-DD` | best-effort; empty if `UNKNOWN`. |
|
||||
| `date_raw` | no | string | verbatim source date. |
|
||||
| `date_precision` | yes | enum | `DAY\|MONTH\|SEASON\|YEAR\|RANGE\|APPROX\|UNKNOWN`. |
|
||||
| `date_end` | no | `YYYY-MM-DD` or empty | RANGE end day (e.g. `7./8. Sept.1923` → `date_iso` = start, `date_end` = end). Empty for every non-RANGE precision **and** for a half-resolved RANGE whose end did not parse (see REQ-DATE-07). |
|
||||
| `location` | no | string | from `Ort`. |
|
||||
| `tags` | no | `tag\|tag` | from `Schlagwort`. |
|
||||
| `summary` | no | string | from `Inhalt`. |
|
||||
| `source_row` | yes | int | provenance (NFR-DATA-01). |
|
||||
| `needs_review` | yes | `flag\|flag` or empty | review flags (REQ-PROV-02). Flags include `unparsed_date`, `range_end_unparsed` (half-resolved RANGE, REQ-DATE-07), `unmatched_sender`, `unmatched_receiver`, `multi_sender`, `index_file_mismatch`, `duplicate_index`. |
|
||||
|
||||
### 6.2 `canonical-persons.xlsx`
|
||||
|
||||
| Field | Required | Format | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `person_id` | yes | slug | stable id (e.g. `de-gruyter-eugenie`); collisions suffixed. |
|
||||
| `last_name` | yes | string | from `Familienname`. |
|
||||
| `first_name` | no | string | primary given name. |
|
||||
| `maiden_name` | no | string | from `geb als` — drives dedup. |
|
||||
| `title` | no | string | e.g. honorifics if present. |
|
||||
| `nickname` | no | string | from quoted `Bemerkung`/spouse field. |
|
||||
| `birth_date` / `birth_date_raw` / `birth_place` | no | ISO / string / string | §4.3 rules. |
|
||||
| `death_date` / `death_date_raw` / `death_place` | no | ISO / string / string | §4.3 rules. |
|
||||
| `spouse` | no | person_id or name | from `verheiratet mit`. |
|
||||
| `generation` | no | string | `G 1`..`G 4`. |
|
||||
| `notes` | no | string | from `Bemerkung`. |
|
||||
| `aliases` | no | `a\|b\|c` | every surface form that maps here. |
|
||||
| `provisional` | yes | bool | true if created from a document string, not the register. |
|
||||
|
||||
### 6.3 `canonical-persons-tree.json`
|
||||
|
||||
The de-duplicated genealogical tree (family members + their relationships) the importer
|
||||
uses to seed the family graph. Each `persons[]` entry carries a `personId` that **joins
|
||||
1:1 onto** `person_id` in `canonical-persons.xlsx`.
|
||||
|
||||
| Field | Required | Format | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `personId` | yes | slug | The register's **verbatim** `person_id` (e.g. `cram-hans-1`), propagated — never re-slugified — so collision suffixes match `canonical-persons.xlsx` exactly. Every tree `personId` exists in the register; the register is the sole slug authority. |
|
||||
| `firstName` / `lastName` / `maidenName` | first/last yes | string | name parts. |
|
||||
| `birthYear` / `deathYear` | no | int or null | year only (tree granularity). |
|
||||
| `birthPlace` / `deathPlace` | no | string or null | from the register. |
|
||||
| `generation` | no | int or null | parsed from `G n`. |
|
||||
| `notes` | no | string or null | leftover Bemerkung text after relationship extraction. |
|
||||
| `familyMember` | yes | bool | always true for tree persons. |
|
||||
|
||||
A top-level `generated_at` is pinned to a fixed timestamp (`2020-01-01T00:00:00`) for
|
||||
reproducibility (NFR-IDEM-01), not a wall-clock value. `relationships[]` carry `SPOUSE_OF`
|
||||
and `PARENT_OF` edges keyed by `rowId`; `unresolved[]` lists relationship strings that did
|
||||
not match a tree person.
|
||||
|
||||
---
|
||||
|
||||
## 7. Prioritized Backlog (MoSCoW)
|
||||
|
||||
| ID | Item | MoSCoW | Effort | Depends on |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| B1 | Project scaffolding + read both workbooks (`FR-INGEST`, header map `FR-MAP`) | Must | S | — |
|
||||
| B2 | Row triage + blank/duplicate/empty reports (`FR-TRIAGE`) | Must | S | B1 |
|
||||
| B3 | Date parser + precision + century rule + Easter/feast computus + season map + tests (`FR-DATE`) | Must | L | B1 |
|
||||
| B4 | Person register parser → canonical persons (`FR-PERS` US-PERS-01) | Must | M | B1 |
|
||||
| B5 | Alias index + name resolution + multi-person split (`FR-DEDUP`, US-PERS-02) | Must | L | B4 |
|
||||
| B6 | Overrides load + apply + idempotency (`FR-OVR`) | Must | S | B3,B5 |
|
||||
| B7 | Canonical writers + provenance + review summary (`FR-OUT`, `FR-PROV`) | Must | M | B2,B3,B5 |
|
||||
| B8 | Index↔Datei mismatch report (`REQ-OUT-02`) | Should | XS | B1 |
|
||||
| B9 | Ambiguous-receiver review path (US-PERS-02 AC4) | Should | S | B5 |
|
||||
| B10 | Comma-split `Inhalt` into extra tags | Could | XS | B7 |
|
||||
| B11 | Phase-2 importer wiring (separate spec) | Won't (this spec) | — | B7 |
|
||||
|
||||
---
|
||||
|
||||
## 8. Traceability — Findings → Requirements
|
||||
|
||||
| Finding | Severity | Addressed by |
|
||||
| --- | --- | --- |
|
||||
| IMP-01 layout mismatch | blocker | C1, FR-MAP, REQ-OUT-01 |
|
||||
| IMP-02 free-text dates | blocker | FR-DATE (all), C2, C6 |
|
||||
| IMP-03 no ISO/normalized cols | blocker | FR-DATE, FR-PERS |
|
||||
| IMP-04 register unimported | major | C3, US-PERS-01, §6.2 |
|
||||
| IMP-05 name variants → dupes | major | C3, FR-DEDUP |
|
||||
| IMP-06 blank-index dropped | major | US-TRIAGE-01 |
|
||||
| IMP-07 duplicate indices | minor | REQ-TRIAGE-01 |
|
||||
| IMP-08 section rows / tags vs summary | minor | REQ-TRIAGE-02, C7 |
|
||||
| IMP-09 index↔file mismatch | minor | REQ-OUT-02, B8 |
|
||||
| IMP-10 `x`-suffix rows | minor | REQ-TRIAGE-03 (skip + log this pass) |
|
||||
| IMP-11 sender not split / ` u ` sep | minor | REQ-PERS-01, US-PERS-02 AC4 |
|
||||
| IMP-12 first-sheet, no validation | minor | REQ-INGEST-01, FR-MAP AC2/AC3 |
|
||||
|
||||
---
|
||||
|
||||
## 9. Open Questions / TBD Register
|
||||
|
||||
| ID | Question | Why it matters | Ref | Resolution |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| OQ-01 ✅ | Season/holiday → date. | Accuracy of ~70 SEASON/feast rows. | REQ-DATE-06 | **Resolved (2026-05-25):** movable feasts (Ostern, Pfingsten, Himmelfahrt, Advent, …) **computed per year from Easter — never a fixed month**; fixed feasts looked up (Weihnachten=12-25, Neujahr=01-01, …); seasons = mid-season month (Frühling=Apr, Sommer=Jul, Herbst=Oct, Winter=Jan). |
|
||||
| OQ-02 ✅ | Date ranges: start only, or start+end? | Sorting/display of ~315 range values. | REQ-DATE-02, REQ-DATE-07 | **Confirmed (updated #670):** store **start** in `date_iso`, precision `RANGE`, full text in `date_raw`, **and the resolved end day in `date_end`** for intra-month day ranges. A half-resolved range (start parsed, end impossible) keeps `date_end` empty and is flagged `range_end_unparsed`. |
|
||||
| OQ-03 ✅ | `person_id` format. | Stability across re-runs; diffability. | §6 | **Confirmed:** readable slug `lastname-firstname`, numeric suffix on collision. |
|
||||
| OQ-04 ✅ | `x`-suffix row handling. | 42 rows. | REQ-TRIAGE-03 | **Resolved (2026-05-25):** `x` rows are transcriptions of the base letter but not yet mappable → **skip this pass**, log to `review/skipped-x-suffix.csv` for later linking. |
|
||||
| OQ-05 ✅ | Importer output format. | Phase-2 reader. | B11 | **Confirmed:** `.xlsx` (openpyxl-native, headered). |
|
||||
| OQ-06 ✅ | Fuzzy-match policy. | False-positive person merges (R2). | REQ-DEDUP-02 | **Confirmed:** conservative — report all fuzzy matches; no silent merge. |
|
||||
|
||||
*All open questions resolved as of 2026-05-25. New ambiguities discovered during build go here.*
|
||||
|
||||
---
|
||||
|
||||
## 10. Glossary & Worked Examples
|
||||
|
||||
**Precision** — how exactly a date is known (`DAY` … `UNKNOWN`). **Provisional person** — a
|
||||
person created from a document name string with no register match. **Alias index** — map from
|
||||
every known surface form of a name to a canonical `person_id`. **Override** — a
|
||||
human-supplied correction applied deterministically on each run.
|
||||
|
||||
**Date examples → expected outcome:**
|
||||
|
||||
| `date_raw` | `date_iso` | `date_precision` |
|
||||
| --- | --- | --- |
|
||||
| `15.2.1888` | 1888-02-15 | DAY |
|
||||
| `6.März 1888` | 1888-03-06 | DAY |
|
||||
| `22.III.18` | 1918-03-22 | DAY |
|
||||
| `13.5.09` | 1909-05-13 | DAY |
|
||||
| `10.Oct.95` | 1895-10-10 | DAY |
|
||||
| `17/6. 1916` | 1916-06-17 | DAY |
|
||||
| `Mai 1895` | 1895-05-01 | MONTH |
|
||||
| `Pfingsten 1922` | 1922-06-04 | DAY (computed: Easter 1922 = Apr 16, +49 days) |
|
||||
| `Herbst 1913` | 1913-10-01 | SEASON |
|
||||
| `1905` | 1905-01-01 | YEAR |
|
||||
| `8.1.1916 - 15.3.1916` | 1916-01-08 | RANGE |
|
||||
| `17.Nov (?) 1887` | 1887-11-17 | APPROX |
|
||||
| `?` | *(empty)* | UNKNOWN |
|
||||
|
||||
**Name examples → expected outcome:**
|
||||
|
||||
| raw cell | resolves to |
|
||||
| --- | --- |
|
||||
| `Eugenie Müller` (+ register `geb Müller`) | `de-gruyter-eugenie` (matched via maiden alias) |
|
||||
| `Eugenie de Gruyter` | `de-gruyter-eugenie` |
|
||||
| `Herbert u Clara` | `cram-herbert` + `cram-clara` (split, surname distributed) |
|
||||
| `Hedi und Tutu (Gruber)` | `gruber-hedi` + `gruber-tutu` |
|
||||
| `Ella Anita` | → `review/ambiguous-receivers.csv` (not auto-split) |
|
||||
| `Hans Wittkopf` (not in register) | provisional `wittkopf-hans` |
|
||||
2281
docs/import-migration/03-normalizer-implementation-plan.md
Normal file
2281
docs/import-migration/03-normalizer-implementation-plan.md
Normal file
File diff suppressed because it is too large
Load Diff
502
docs/import-migration/04-unresolved-names-plan.md
Normal file
502
docs/import-migration/04-unresolved-names-plan.md
Normal file
@@ -0,0 +1,502 @@
|
||||
# Unresolved-Name Classification Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Add a focused `review/unresolved-names.csv` that isolates sender/receiver strings whose *name itself* is problematic (unknown/illegible, single-token, relational-only, collective/group, prose-in-name-column, or a genuine two-given-name pair), and fix the ambiguous-pair heuristic so a plain `First Surname` external person (e.g. `Mieze Schefold`) is no longer falsely flagged.
|
||||
|
||||
**Architecture:** A pure `classify_name(raw, given_names)` function in `persons.py` returns a `NameClass`. `ResolutionContext` classifies every *unmatched* name and records the non-`RESOLVABLE` ones in `self.unresolved`. A runtime-built given-name set (register first names + a small config supplement) lets the classifier distinguish a two-given-name pair (`Ella Anita` → two people) from a first+surname single person (`Mieze Schefold`). The orchestrator writes the aggregated report and per-category stats, replacing the noisy `ambiguous-receivers.csv`.
|
||||
|
||||
**Tech Stack:** Python 3.12, openpyxl, pytest — extends the existing `tools/import-normalizer/`.
|
||||
|
||||
**Context:** This builds on the completed normalizer (PR #663). Run all tests with CWD = the tool dir, e.g. `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_X.py -v`. Reuse the existing venv at `tools/import-normalizer/.venv` (do NOT recreate it). Commit on the current branch `docs/import-migration` (never main, never push). Each commit message ends with a trailing `Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>` line.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
tools/import-normalizer/
|
||||
├── config.py # + RELATIONAL_TERMS, COLLECTIVE_TERMS, UNKNOWN_NAME_MARKERS, PROSE_MAX_LEN, EXTRA_GIVEN_NAMES
|
||||
├── persons.py # + NameClass, classify_name(), build_given_names(); ResolutionContext gains given_names + self.unresolved
|
||||
├── normalize.py # writes unresolved-names.csv (replaces ambiguous-receivers.csv) + per-category stats
|
||||
├── README.md # + unresolved-names.csv row in the review-file table
|
||||
└── tests/
|
||||
├── test_config.py # + name-table presence test
|
||||
├── test_persons.py # + classify_name + build_given_names tests
|
||||
├── test_documents.py # ambiguous test → unresolved test (+ resolvable-pair test)
|
||||
└── test_normalize.py # integration asserts unresolved-names.csv
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Config — name-classification tables
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/config.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_config.py`
|
||||
|
||||
- [ ] **Step 1: Add the failing test** to `tests/test_config.py`
|
||||
|
||||
```python
|
||||
def test_name_classification_tables():
|
||||
assert "tante" in config.RELATIONAL_TERMS
|
||||
assert "familie" in config.COLLECTIVE_TERMS
|
||||
assert "unbekannt" in config.UNKNOWN_NAME_MARKERS
|
||||
assert config.PROSE_MAX_LEN >= 30
|
||||
assert "anita" in config.EXTRA_GIVEN_NAMES
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_config.py::test_name_classification_tables -v && cd -`
|
||||
Expected: FAIL — `AttributeError: module 'config' has no attribute 'RELATIONAL_TERMS'`.
|
||||
|
||||
- [ ] **Step 3: Implement** — append to `config.py` (after the existing tables, before/after `KNOWN_LAST_NAMES` — anywhere at module level)
|
||||
|
||||
```python
|
||||
# --- Name classification (unresolved-name review) ---
|
||||
# Relational reference terms — a sender/receiver named by relation, not a proper name.
|
||||
RELATIONAL_TERMS = {
|
||||
"tante", "onkel", "mutter", "vater", "oma", "opa", "großmutter", "grossmutter",
|
||||
"großvater", "grossvater", "schwester", "bruder", "cousin", "cousine", "kusine",
|
||||
"neffe", "nichte", "tochter", "sohn", "schwager", "schwägerin", "schwiegermutter",
|
||||
"schwiegervater", "enkel", "enkelin", "vetter", "base", "witwe", "witwer",
|
||||
}
|
||||
# Collective/group terms — not a single person. Matched against alpha-only word tokens
|
||||
# (so "Fam.Cram" -> ["fam","cram"] matches "fam"), NOT as substrings/prefixes.
|
||||
COLLECTIVE_TERMS = {
|
||||
"familie", "fam", "kinder", "eltern", "geschwister", "großeltern",
|
||||
"grosseltern", "alle", "diverse", "div", "gebrüder", "gebr",
|
||||
}
|
||||
# Markers of an unknown/illegible name (the literal "?" is handled separately in code).
|
||||
# All long enough to be safe as SUBSTRING matches — do NOT add short tokens like "nn"
|
||||
# (it occurs inside real names: Hanni, Johanna, Anna).
|
||||
UNKNOWN_NAME_MARKERS = {"unbekannt", "unbek", "unleserlich", "unklar", "unsicher"}
|
||||
# A name-column value longer than this (chars) is treated as prose/description, not a name.
|
||||
PROSE_MAX_LEN = 40
|
||||
# Common given names that may appear in two-given-name pairs (e.g. "Ella Anita") but are not
|
||||
# in the family register. Only used to detect AMBIGUOUS_PAIR — extend as review surfaces more.
|
||||
EXTRA_GIVEN_NAMES = {
|
||||
"ella", "anita", "kurt", "georg", "hanni", "mieze", "ellen", "leni", "klara",
|
||||
"margret", "gustava", "emmy", "minna", "sophie", "helga", "raymonde", "augusta",
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_config.py -v && cd -`
|
||||
Expected: PASS (all config tests).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/config.py tools/import-normalizer/tests/test_config.py
|
||||
git commit -m "feat(normalizer): config tables for name classification"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: `classify_name` + `NameClass`
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/persons.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_persons.py`
|
||||
|
||||
- [ ] **Step 1: Add failing tests** to `tests/test_persons.py`
|
||||
|
||||
```python
|
||||
from persons import NameClass
|
||||
|
||||
GIVEN = {"ella", "anita", "kurt", "georg", "clara", "eugenie"}
|
||||
|
||||
def test_classify_unknown():
|
||||
assert persons.classify_name("?", GIVEN) is NameClass.UNKNOWN
|
||||
assert persons.classify_name("A. Kredell?", GIVEN) is NameClass.UNKNOWN
|
||||
assert persons.classify_name("unbekannt", GIVEN) is NameClass.UNKNOWN
|
||||
|
||||
def test_classify_prose():
|
||||
assert persons.classify_name("Adressenliste v Clara Cram zur Kondolenz", GIVEN) is NameClass.PROSE
|
||||
assert persons.classify_name("Clara de Gruyter(*1871)", GIVEN) is NameClass.PROSE # digit
|
||||
assert persons.classify_name('"Cramiade" Gedicht', GIVEN) is NameClass.PROSE # quote
|
||||
|
||||
def test_classify_collective():
|
||||
assert persons.classify_name("Familie", GIVEN) is NameClass.COLLECTIVE
|
||||
assert persons.classify_name("Fam.Cram", GIVEN) is NameClass.COLLECTIVE
|
||||
assert persons.classify_name("Eltern Cram", GIVEN) is NameClass.COLLECTIVE
|
||||
assert persons.classify_name("seine Kinder", GIVEN) is NameClass.COLLECTIVE
|
||||
|
||||
def test_classify_relational():
|
||||
assert persons.classify_name("Cousine Emmy Haniel", GIVEN) is NameClass.RELATIONAL
|
||||
assert persons.classify_name("Schwester Hanni", GIVEN) is NameClass.RELATIONAL
|
||||
|
||||
def test_classify_single_token():
|
||||
assert persons.classify_name("Agnes", GIVEN) is NameClass.SINGLE_TOKEN
|
||||
assert persons.classify_name("A.B.", GIVEN) is NameClass.SINGLE_TOKEN
|
||||
|
||||
def test_classify_ambiguous_pair():
|
||||
assert persons.classify_name("Ella Anita", GIVEN) is NameClass.AMBIGUOUS_PAIR
|
||||
assert persons.classify_name("Kurt Georg", GIVEN) is NameClass.AMBIGUOUS_PAIR
|
||||
|
||||
def test_classify_resolvable_single_person():
|
||||
# first + surname (surname not a given name) -> one real person, NOT ambiguous
|
||||
assert persons.classify_name("Mieze Schefold", GIVEN) is NameClass.RESOLVABLE
|
||||
assert persons.classify_name("Adolf Butenandt", GIVEN) is NameClass.RESOLVABLE
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_persons.py -k classify -v && cd -`
|
||||
Expected: FAIL — `NameClass` / `classify_name` not defined.
|
||||
|
||||
- [ ] **Step 3: Implement** — add to `persons.py`. Add `from enum import StrEnum` to the imports if not present, then add:
|
||||
|
||||
```python
|
||||
class NameClass(StrEnum):
|
||||
RESOLVABLE = "resolvable"
|
||||
UNKNOWN = "unknown"
|
||||
SINGLE_TOKEN = "single_token"
|
||||
RELATIONAL = "relational"
|
||||
COLLECTIVE = "collective"
|
||||
PROSE = "prose"
|
||||
AMBIGUOUS_PAIR = "ambiguous_pair"
|
||||
|
||||
|
||||
_QUOTE_CHARS = "\"'“”„‚‘’"
|
||||
|
||||
|
||||
def classify_name(raw: str, given_names: set[str]) -> NameClass:
|
||||
"""Classify a (post-split) sender/receiver string by why it may be unresolvable.
|
||||
|
||||
Precedence (first match wins): UNKNOWN -> PROSE -> COLLECTIVE -> RELATIONAL ->
|
||||
SINGLE_TOKEN -> AMBIGUOUS_PAIR -> RESOLVABLE.
|
||||
"""
|
||||
s = raw.strip()
|
||||
if not s:
|
||||
return NameClass.RESOLVABLE
|
||||
low = s.lower()
|
||||
tokens = s.split()
|
||||
# alpha-only word tokens: "Fam.Cram" -> ["fam","cram"], so collective/relational terms
|
||||
# are matched as whole words (no substring/prefix false positives like "Allerton").
|
||||
alpha_words = re.findall(r"[a-zäöüß]+", low)
|
||||
if "?" in s or any(m in low for m in config.UNKNOWN_NAME_MARKERS):
|
||||
return NameClass.UNKNOWN
|
||||
if (len(s) > config.PROSE_MAX_LEN or any(c.isdigit() for c in s)
|
||||
or any(q in s for q in _QUOTE_CHARS) or len(tokens) > 3):
|
||||
return NameClass.PROSE
|
||||
if any(w in config.COLLECTIVE_TERMS for w in alpha_words):
|
||||
return NameClass.COLLECTIVE
|
||||
if any(w in config.RELATIONAL_TERMS for w in alpha_words):
|
||||
return NameClass.RELATIONAL
|
||||
if len(tokens) == 1:
|
||||
return NameClass.SINGLE_TOKEN
|
||||
if len(tokens) == 2 and all(_norm(t) in given_names for t in tokens):
|
||||
return NameClass.AMBIGUOUS_PAIR
|
||||
return NameClass.RESOLVABLE
|
||||
|
||||
|
||||
# Known limitation: a 4+-token name with no digits/quotes (e.g. "Anna von der Heide") is
|
||||
# classified PROSE. Such multi-particle names are rare here and usually resolve via the
|
||||
# register; if they surface in review, lower-priority than the real prose entries.
|
||||
```
|
||||
|
||||
> Note: `_norm` already exists in `persons.py` (added in the alias-index task) and strips accents + lowercases. `classify_name` uses it so given-name matching is accent-insensitive.
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_persons.py -v && cd -`
|
||||
Expected: PASS (all persons tests, including the 7 new classify tests).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/persons.py tools/import-normalizer/tests/test_persons.py
|
||||
git commit -m "feat(normalizer): classify_name + NameClass"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: `build_given_names`
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/persons.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_persons.py`
|
||||
|
||||
- [ ] **Step 1: Add failing test** to `tests/test_persons.py`
|
||||
|
||||
```python
|
||||
def test_build_given_names():
|
||||
people = persons.parse_register([
|
||||
{"last_name": "de Gruyter", "first_name": "Eugenie"},
|
||||
{"last_name": "Cram", "first_name": "Charlotte,Meta"}, # comma -> primary + extra given
|
||||
])
|
||||
g = persons.build_given_names(people, {"Anita"})
|
||||
assert "eugenie" in g
|
||||
assert "charlotte" in g and "meta" in g # primary + extra given names
|
||||
assert "anita" in g # from the extra set, normalized
|
||||
assert "schefold" not in g
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify it fails**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_persons.py::test_build_given_names -v && cd -`
|
||||
Expected: FAIL — `build_given_names` not defined.
|
||||
|
||||
- [ ] **Step 3: Implement** — add to `persons.py`
|
||||
|
||||
```python
|
||||
def build_given_names(register: list[Person], extra: set[str]) -> set[str]:
|
||||
"""Set of normalized given names from the register (first + extra given) plus a supplement.
|
||||
|
||||
Used by classify_name to tell a two-given-name pair (two people) from a first+surname.
|
||||
"""
|
||||
names: set[str] = set()
|
||||
for p in register:
|
||||
if p.first_name:
|
||||
names.add(_norm(p.first_name))
|
||||
for g in p.extra_given_names:
|
||||
names.add(_norm(g))
|
||||
for e in extra:
|
||||
names.add(_norm(e))
|
||||
return names
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run to verify it passes**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_persons.py -v && cd -`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/persons.py tools/import-normalizer/tests/test_persons.py
|
||||
git commit -m "feat(normalizer): build_given_names from register + supplement"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Integrate — ResolutionContext records unresolved; orchestrator writes the report
|
||||
|
||||
This task touches `persons.py`, `normalize.py`, and two test files together so the whole suite stays green in one commit (removing `ctx.ambiguous` requires updating its only consumer, `normalize.py`, in the same change).
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/persons.py` (ResolutionContext)
|
||||
- Modify: `tools/import-normalizer/normalize.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_documents.py`
|
||||
- Modify: `tools/import-normalizer/tests/test_normalize.py`
|
||||
|
||||
- [ ] **Step 1: Update the failing tests first**
|
||||
|
||||
In `tests/test_documents.py`, **replace** the existing `test_ambiguous_space_pair_flagged_not_split` function entirely with these two functions:
|
||||
|
||||
```python
|
||||
def test_ambiguous_pair_recorded_in_unresolved():
|
||||
people = persons.parse_register([{"last_name": "de Gruyter", "first_name": "Walter"}])
|
||||
ctx = persons.ResolutionContext(persons.AliasIndex(people), name_overrides={},
|
||||
given_names={"ella", "anita"})
|
||||
raw = documents.RawRow(source_row=7, index="C-0200", sender="", receivers="Ella Anita")
|
||||
doc = documents.to_canonical(raw, ctx, date_overrides={})
|
||||
assert len(doc.receiver_person_ids) == 1 # not split — one provisional
|
||||
assert any(name == "Ella Anita" and cat == "ambiguous_pair" for name, cat, _ in ctx.unresolved)
|
||||
|
||||
def test_resolvable_first_surname_pair_not_unresolved():
|
||||
ctx = persons.ResolutionContext(persons.AliasIndex([]), name_overrides={},
|
||||
given_names={"ella", "anita"})
|
||||
ctx.resolve_one("Mieze Schefold", source_row=1) # surname is not a given name
|
||||
assert ctx.unresolved == [] # RESOLVABLE -> not recorded
|
||||
```
|
||||
|
||||
In `tests/test_normalize.py`, in the `_doc_wb` fixture, change the `C-0001` row's receiver from empty to `"?"` so the run produces an unresolved entry. Find the line that appends the `C-0001` row and set its `EmpfängerIn` cell to `"?"`. For example the row currently reads:
|
||||
|
||||
```python
|
||||
ws.append(["C-0001", "", "", "", "Hans Wittkopf", "", "Freitag 1919", "", "", ""])
|
||||
```
|
||||
|
||||
change the 6th cell (EmpfängerIn) from `""` to `"?"`:
|
||||
|
||||
```python
|
||||
ws.append(["C-0001", "", "", "", "Hans Wittkopf", "?", "Freitag 1919", "", "", ""])
|
||||
```
|
||||
|
||||
Then add these assertions inside `test_run_end_to_end`, right after the existing `assert (review_dir / "unparsed-dates.csv").exists()` line:
|
||||
|
||||
```python
|
||||
assert (out_dir / "canonical-documents.xlsx").exists() # (keep existing asserts above)
|
||||
assert (review_dir / "unresolved-names.csv").exists()
|
||||
unresolved_text = (review_dir / "unresolved-names.csv").read_text(encoding="utf-8")
|
||||
assert "unknown" in unresolved_text and "?" in unresolved_text # the "?" receiver
|
||||
assert not (review_dir / "ambiguous-receivers.csv").exists() # replaced
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run to verify they fail**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/test_documents.py tests/test_normalize.py -v && cd -`
|
||||
Expected: FAIL — `ResolutionContext` has no `given_names`/`unresolved`; `unresolved-names.csv` not written.
|
||||
|
||||
- [ ] **Step 3a: Implement — `ResolutionContext` in `persons.py`**
|
||||
|
||||
Replace the `ResolutionContext.__init__` body's two lines (`self.ambiguous` and add `given_names`) and the relevant methods. The new `__init__`:
|
||||
|
||||
```python
|
||||
def __init__(self, alias_index: AliasIndex, name_overrides: dict[str, str],
|
||||
given_names: set[str] | None = None):
|
||||
self.index = alias_index
|
||||
self.name_overrides = name_overrides
|
||||
self.given_names = given_names or set()
|
||||
self.provisional: dict[str, Person] = {}
|
||||
self.unmatched: dict[str, list] = {}
|
||||
self.unresolved: list[tuple] = [] # (raw_name, category, source_row) for non-RESOLVABLE names
|
||||
self._raw_to_pid: dict[str, str] = {}
|
||||
self.override_hits = 0
|
||||
```
|
||||
|
||||
In `resolve_one`, the provisional branch must classify the name. Replace this existing block:
|
||||
|
||||
```python
|
||||
# provisional person (unmatched) — never reuse a register id
|
||||
self.unmatched.setdefault(name, []).append(source_row)
|
||||
if name in self._raw_to_pid:
|
||||
return self._raw_to_pid[name], name, False
|
||||
```
|
||||
|
||||
with:
|
||||
|
||||
```python
|
||||
# provisional person (unmatched) — never reuse a register id
|
||||
self.unmatched.setdefault(name, []).append(source_row)
|
||||
category = classify_name(name, self.given_names)
|
||||
if category is not NameClass.RESOLVABLE:
|
||||
self.unresolved.append((name, str(category), source_row))
|
||||
if name in self._raw_to_pid:
|
||||
return self._raw_to_pid[name], name, False
|
||||
```
|
||||
|
||||
Replace the entire `resolve_receivers` method (the ambiguous detection now lives in `resolve_one` via `classify_name`):
|
||||
|
||||
```python
|
||||
def resolve_receivers(self, raw: str, source_row: int):
|
||||
return [self.resolve_one(part, source_row) for part in split_receivers(raw)]
|
||||
```
|
||||
|
||||
- [ ] **Step 3b: Implement — `normalize.py`**
|
||||
|
||||
Find the line that builds the context:
|
||||
|
||||
```python
|
||||
ctx = persons.ResolutionContext(alias_index, name_overrides)
|
||||
```
|
||||
|
||||
replace it with (build the given-name set from the register + config supplement):
|
||||
|
||||
```python
|
||||
given_names = persons.build_given_names(register, config.EXTRA_GIVEN_NAMES)
|
||||
ctx = persons.ResolutionContext(alias_index, name_overrides, given_names=given_names)
|
||||
```
|
||||
|
||||
Replace the `ambiguous-receivers.csv` write line:
|
||||
|
||||
```python
|
||||
writers.write_review_csv(review_dir / "ambiguous-receivers.csv", ["raw", "part", "source_row"], ctx.ambiguous)
|
||||
```
|
||||
|
||||
with an aggregated unresolved-names report:
|
||||
|
||||
```python
|
||||
unresolved_agg: dict[tuple, list] = {}
|
||||
for name, category, row in ctx.unresolved:
|
||||
unresolved_agg.setdefault((category, name), []).append(row)
|
||||
unresolved_rows = sorted(
|
||||
([cat, name, len(rows), " ".join(map(str, sorted(rows)[:5]))]
|
||||
for (cat, name), rows in unresolved_agg.items()),
|
||||
key=lambda r: (r[0], -r[2], r[1]))
|
||||
writers.write_review_csv(review_dir / "unresolved-names.csv",
|
||||
["category", "raw", "count", "example_rows"], unresolved_rows)
|
||||
```
|
||||
|
||||
In the `stats` dict, replace the `"ambiguous_receivers"` line:
|
||||
|
||||
```python
|
||||
"ambiguous_receivers": len(ctx.ambiguous),
|
||||
```
|
||||
|
||||
with a per-category breakdown:
|
||||
|
||||
```python
|
||||
"unresolved_name_occurrences": len(ctx.unresolved),
|
||||
"unresolved_unknown": sum(1 for _, c, _ in ctx.unresolved if c == "unknown"),
|
||||
"unresolved_single_token": sum(1 for _, c, _ in ctx.unresolved if c == "single_token"),
|
||||
"unresolved_relational": sum(1 for _, c, _ in ctx.unresolved if c == "relational"),
|
||||
"unresolved_collective": sum(1 for _, c, _ in ctx.unresolved if c == "collective"),
|
||||
"unresolved_prose": sum(1 for _, c, _ in ctx.unresolved if c == "prose"),
|
||||
"unresolved_ambiguous_pair": sum(1 for _, c, _ in ctx.unresolved if c == "ambiguous_pair"),
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the whole suite to verify green**
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/ -q && cd -`
|
||||
Expected: PASS (all tests, no `ambiguous` references remain).
|
||||
|
||||
Also grep to confirm no dangling references:
|
||||
Run: `grep -rn "ctx.ambiguous\|ambiguous-receivers\|ambiguous_receivers\|self.ambiguous" tools/import-normalizer/*.py`
|
||||
Expected: no matches.
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/persons.py tools/import-normalizer/normalize.py tools/import-normalizer/tests/test_documents.py tools/import-normalizer/tests/test_normalize.py
|
||||
git commit -m "feat(normalizer): unresolved-names report + fix ambiguous-pair over-flagging"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: README — document the new report
|
||||
|
||||
**Files:**
|
||||
- Modify: `tools/import-normalizer/README.md`
|
||||
|
||||
- [ ] **Step 1: Update the review-file table** in `README.md`. Replace the `ambiguous-receivers.csv` row with an `unresolved-names.csv` row. Find the table row referencing `ambiguous-receivers.csv` and replace it with:
|
||||
|
||||
```markdown
|
||||
| `unresolved-names.csv` | Names whose value is itself problematic, grouped by `category`: `unknown` (`?`/illegible), `single_token` (first OR last name only), `relational` (`Tante …`), `collective` (`Familie …`), `prose` (a description landed in a name column), `ambiguous_pair` (two given names → likely two people, not auto-split). Review highest-impact categories first; add decisions to `overrides/names.csv`. |
|
||||
```
|
||||
|
||||
If the README has no such row (older version), add the row above to the review-file table.
|
||||
|
||||
- [ ] **Step 2: Add a note** to the iteration-loop section of `README.md` (after the table):
|
||||
|
||||
```markdown
|
||||
> `unresolved-names.csv` is the focused "names that need a human" list — distinct from
|
||||
> `unmatched-names.csv` (which is just non-family correspondents that got provisional persons).
|
||||
> The given-name set that drives `ambiguous_pair` detection is the register's first names plus
|
||||
> `config.EXTRA_GIVEN_NAMES` — add names there if a real two-person cell isn't being flagged.
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Verify the suite is still green** (README-only change, but confirm nothing references the old file)
|
||||
|
||||
Run: `cd tools/import-normalizer && .venv/bin/python -m pytest tests/ -q && cd -`
|
||||
Expected: PASS.
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add tools/import-normalizer/README.md
|
||||
git commit -m "docs(normalizer): document unresolved-names.csv review report"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-Review
|
||||
|
||||
**Spec coverage** (against the agreed proposal):
|
||||
- Focused report isolating problem name classes → Task 4 writes `review/unresolved-names.csv` with a `category` column; categories defined in Task 2 `classify_name`. ✓
|
||||
- Fix ambiguous over-flagging of `First Surname` → Task 2 `AMBIGUOUS_PAIR` requires *both* tokens in the given-name set; `Mieze Schefold` → `RESOLVABLE` (tested). ✓
|
||||
- Distinguish "not fully known" (unknown/single-token/relational/collective/prose) from "can't split cleanly" (ambiguous_pair) → all are `NameClass` values, each its own category column value. ✓
|
||||
- Per-category counts in summary → Task 4 stats. ✓
|
||||
- Senders covered too (not just receivers) → classification happens in `resolve_one`, which both `resolve_sender` and `resolve_receivers` call. ✓
|
||||
|
||||
**Placeholder scan:** No TBD/TODO; every code step has complete code. The README replacement gives the exact row text.
|
||||
|
||||
**Type consistency:** `NameClass` (StrEnum) defined Task 2; `classify_name(raw, given_names)` and `build_given_names(register, extra)` signatures used consistently in Task 4; `ResolutionContext(alias_index, name_overrides, given_names=…)` matches the new `__init__`; `self.unresolved` is `list[tuple]` of `(raw, category, source_row)` and read with that shape in both the report and the stats. `str(category)` yields the StrEnum value (e.g. `"ambiguous_pair"`), matching the stat comparisons and the test assertions.
|
||||
|
||||
**Cross-task green:** Task 4 deliberately bundles the `persons.py` + `normalize.py` + test changes into one commit because removing `ctx.ambiguous` breaks its consumer otherwise — no red commit is left behind (lesson from the prior build).
|
||||
|
||||
**Out of scope (future):** Spanish month names + `Mon DD-YYYY` date form (separate date-parser enhancement); promoting `unresolved` rows into a document-level `needs_review` flag; auto-splitting confirmed `ambiguous_pair` entries via overrides.
|
||||
62
docs/import-migration/README.md
Normal file
62
docs/import-migration/README.md
Normal file
@@ -0,0 +1,62 @@
|
||||
# Import Migration — Working Folder
|
||||
|
||||
This folder tracks the iterative work of mass-importing the **real, raw family archive**
|
||||
spreadsheets (≈7,600 letter rows + ~7,000 PDFs that arrive later) into Familienarchiv.
|
||||
|
||||
It is intentionally **local docs, not Gitea issues**. We only open a Gitea issue when a
|
||||
finding requires a *software* change (e.g. a new date parser). Pure data observations and
|
||||
the running plan live here so any agent can pick the work up cold.
|
||||
|
||||
## Source files (in `/import`)
|
||||
|
||||
| File | What it is | Importer support today |
|
||||
| --- | --- | --- |
|
||||
| `zzfamilienarchiv aktuell 2 - Kopie 2025-07-05.xlsx` | The **real raw archive** — 7,943 rows, sheet `Familienarchiv`. Human-readable, dates as written in the letters. | ❌ layout does **not** match importer defaults |
|
||||
| `Personendatei 2.xlsx` | Genealogical **person register** — 163 people, sheet `Tabelle1` (maiden names, birth/death, marriages, relationships). | ❌ no importer at all |
|
||||
| `zzfamilienarchiv Walter und Eugenie 2025-04-10.ods` | A small, **already-normalized** subset (Walter & Eugenie brautbriefe). 14 clean columns incl. ISO dates. | ✅ this is what `MassImportService` was built for |
|
||||
|
||||
The PDFs (~7,000) will follow later. The importer matches files by the **Index** column
|
||||
(e.g. `W-0001` → `W-0001.pdf`), and already imports metadata-only when a file is missing —
|
||||
so we can import all metadata now and the PDFs will attach on a re-run.
|
||||
|
||||
## How to inspect the spreadsheets
|
||||
|
||||
`openpyxl` is installed in the OCR service venv:
|
||||
|
||||
```bash
|
||||
/home/marcel/Desktop/familienarchiv/ocr-service/.venv/bin/python3 -c "import openpyxl; print(openpyxl.__version__)"
|
||||
```
|
||||
|
||||
## Documents in this folder
|
||||
|
||||
- [`01-findings-spreadsheet-analysis.md`](./01-findings-spreadsheet-analysis.md) — full analysis of every data-quality / importer issue found (2026-05-25). Each issue has an ID `IMP-NN`.
|
||||
- [`02-normalization-spec.md`](./02-normalization-spec.md) — requirements spec for the offline **import normalizer** (the agreed strategy: normalize the raw sheets into a clean canonical dataset before import). Requirements `FR-*`/`NFR-*`, traceable to the `IMP-NN` findings.
|
||||
- `WORKLOG.md` — running log of what each session did and what's next. **Start here when resuming.**
|
||||
|
||||
## Strategy (decided 2026-05-25)
|
||||
|
||||
Normalize **before** import. A standalone Python tool (`tools/import-normalizer/`, not yet
|
||||
built) transforms the raw xlsx + person register into a clean canonical dataset
|
||||
(`canonical-documents.xlsx`, `canonical-persons.xlsx`) plus review CSVs. Residual cases
|
||||
(unparseable dates, unmatched names) are fixed via a version-controlled overrides file and
|
||||
re-run. The Java importer is adjusted to consume the canonical contract in a later **Phase 2**.
|
||||
See the spec for the full contract.
|
||||
|
||||
## Status board
|
||||
|
||||
| ID | Issue | Severity | Status |
|
||||
| --- | --- | --- | --- |
|
||||
| IMP-01 | New xlsx column layout ≠ importer defaults | 🔴 blocker | open |
|
||||
| IMP-02 | 90% of dates are free-text the parser can't read | 🔴 blocker | open |
|
||||
| IMP-03 | No ISO/normalized date column in the new xlsx | 🔴 blocker | open |
|
||||
| IMP-04 | Person register (`Personendatei 2.xlsx`) not imported | 🟠 major | open |
|
||||
| IMP-05 | Name variations = duplicate Persons (maiden vs married) | 🟠 major | open |
|
||||
| IMP-06 | 93 data rows with blank Index are silently dropped | 🟠 major | open |
|
||||
| IMP-07 | 43 duplicate Index values | 🟡 minor | open |
|
||||
| IMP-08 | Section/title rows interleaved in data | 🟡 minor | open |
|
||||
| IMP-09 | Index↔Datei filename mismatches | 🟡 minor | open |
|
||||
| IMP-10 | `x`-suffix rows (letter backsides/enclosures) | 🟡 minor | open |
|
||||
| IMP-11 | Multi-receiver separators incl. bare `u`/`u.` | 🟡 minor | open |
|
||||
| IMP-12 | Importer reads only the first sheet, no validation | 🟡 minor | open |
|
||||
|
||||
See the findings doc for detail and proposed approach per issue.
|
||||
147
docs/import-migration/WORKLOG.md
Normal file
147
docs/import-migration/WORKLOG.md
Normal file
@@ -0,0 +1,147 @@
|
||||
# Import Migration — Worklog
|
||||
|
||||
Running log of each working session. **Resume here.** Newest entry on top.
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 5) — Unresolved-name classification
|
||||
|
||||
**Did:** Implemented [`04-unresolved-names-plan.md`](./04-unresolved-names-plan.md) subagent-driven
|
||||
(5 tasks, TDD, per-task spec + code-quality review; 67 tests pass). Added `classify_name` +
|
||||
`NameClass` + `build_given_names` in `persons.py`; `ResolutionContext` now records non-RESOLVABLE
|
||||
names in `self.unresolved`; orchestrator writes `review/unresolved-names.csv` (replaces the noisy
|
||||
`ambiguous-receivers.csv`) with per-category stats.
|
||||
|
||||
**Why:** `unmatched-names.csv` mixes boring non-family correspondents (expected) with genuinely
|
||||
unresolvable entries. The new report isolates the latter so review focuses on ~440 real cases.
|
||||
|
||||
**Real-run result:** unresolved-names.csv = single_token 191 / prose 103 / unknown 74 /
|
||||
collective 46 / relational 21 / ambiguous_pair **5** (distinct). The ambiguous over-flagging fix
|
||||
cut `ambiguous_pair` from 303 → 5 (genuine two-given-name pairs only; `Mieze Schefold` etc. now
|
||||
correctly RESOLVABLE). given-name set = register first names ∪ `config.EXTRA_GIVEN_NAMES`.
|
||||
|
||||
**Next:** populate `overrides/names.csv` from unresolved-names.csv (highest-count first); extend
|
||||
`EXTRA_GIVEN_NAMES` if a real pair isn't flagged; still-open date work (Spanish months, 58–72 band).
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 4) — Built the normalizer (subagent-driven, all 17 tasks)
|
||||
|
||||
**Did:** Executed the plan subagent-driven (implementer + spec review + code-quality review per
|
||||
task). The tool `tools/import-normalizer/` is **complete and passing (57 tests)**. Final
|
||||
opus review: **READY** — determinism verified on the real corpus (two runs → identical cell
|
||||
matrices + byte-identical review files), zero silent drops.
|
||||
|
||||
**Per-task code review caught & fixed real issues** (all in the committed code): leading
|
||||
qualifiers `nach/vor/…` now → APPROX; English month-first matcher hardened to structurally
|
||||
not shadow `Mai 1895`; person-id collision de-dup suffixes *all* members; `split_receivers`
|
||||
returns `[]` for a `geb.`-only cell; boolean cells no longer coerced to `1/0`; duplicate-index
|
||||
flags every occurrence; provisional ids never steal a register id; CSV-injection defanged.
|
||||
|
||||
**REAL DRY-RUN** (`python normalize.py` over the actual archive — outputs are gitignored):
|
||||
- documents_emitted **7,582** (+225 empty +93 blank-index +42 x-suffix = 7,942 rows read, 0 dropped)
|
||||
- register_persons **163**, provisional_persons **942**
|
||||
- dates: DAY 6,509 / MONTH 36 / RANGE 36 / APPROX 28 / YEAR 17 / SEASON 1 / UNKNOWN 955
|
||||
- **unknown_date_rate 9.2%** (of dated rows; target ≤5% pre-override, ≤0.5% after overrides)
|
||||
- duplicate_index 85, index_file_mismatches 550, ambiguous_receivers 303
|
||||
|
||||
**⚠️ Concurrency incident:** a parallel Claude session committed reader-dashboard work to this
|
||||
branch and hard-reset it mid-execution, deleting the Task 15 files and orphaning a commit.
|
||||
Recovered via reflog (`reset --hard 366b4848` + `checkout 401160e3 -- <task15 files>`); no code
|
||||
lost. Casualty: my *during-execution* edits to the plan/spec docs (02/03) for Tasks 5–14 were
|
||||
discarded — **the committed code + tests are the source of truth**, not the plan doc, which now
|
||||
reflects the pre-execution + persona-review version.
|
||||
|
||||
**Next steps (iterative refinement — the overrides loop, as designed):**
|
||||
1. Shave the 9.2% UNKNOWN cheaply: add **Spanish month names** (Enero…Diciembre) and the
|
||||
`Mon DD-YYYY` dash form to `config.MONTHS`/the parser (Mexican-branch correspondence);
|
||||
revisit the 58–72 two-digit-year band (real `…58/59/60` dates = 1958–1960, just past the
|
||||
1873–1957 window — decide whether to extend the upper bound in `config`).
|
||||
2. `?` (99×) is genuinely "date unknown" — leave UNKNOWN or add a convention.
|
||||
3. Populate `overrides/dates.csv` + `overrides/names.csv` from the review CSVs and re-run.
|
||||
4. README note: a leading `'`/`!` in a `review/*.csv` `raw` cell may be a CSV-defang artifact —
|
||||
match against the true source value when writing overrides.
|
||||
5. Phase 2 (separate spec): wire the canonical contract into the Java `MassImportService`.
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 3) — Implementation plan + persona review
|
||||
|
||||
**Did:**
|
||||
- Wrote [`03-normalizer-implementation-plan.md`](./03-normalizer-implementation-plan.md): 17
|
||||
bite-sized TDD tasks for `tools/import-normalizer/` (Python, openpyxl), bottom-up — date
|
||||
parser w/ Easter computus first, then persons/alias, ingest, mapping, orchestrator, writers.
|
||||
- Ran a 6-persona inline review (architect, developer, tester, req-engineer, security, devops;
|
||||
ui-expert too) via parallel agents. Acted on all material findings.
|
||||
|
||||
**Key fixes from review (see plan §"Review feedback incorporated"):**
|
||||
- Idempotency redefined byte-identical → **content-deterministic** (spec G4/NFR-IDEM-01);
|
||||
pinned workbook timestamps + deterministic alias ordering + a real two-run equality test.
|
||||
- Real bug: duplicate-index only reported repeats → now flags/reports every occurrence.
|
||||
- Provisional `person_id` could overwrite a register id → now suffixed.
|
||||
- Date parser gaps: invalid-calendar-date → UNKNOWN, intra-month day-range (`7./8. Sept.1923`).
|
||||
- Multi-person sender now split + flagged (REQ-PERS-01); CSV-injection defanged in review files;
|
||||
pinned deps + hardened root `.gitignore`.
|
||||
|
||||
**Next:**
|
||||
- Marcel reviews the plan. Then execute it (subagent-driven or inline) — the date parser
|
||||
(Task 3/8 + Easter computus) is the meatiest piece.
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 2) — Strategy + normalizer spec
|
||||
|
||||
**Did:**
|
||||
- Decided strategy with Marcel: **normalize the raw sheets first**, then import (higher
|
||||
leverage than making the Java importer tolerate every mess).
|
||||
- Locked design decisions (see spec §3): new canonical layout; dates = parsed + raw +
|
||||
precision; include person register + dedup in this effort; overrides-file + re-run loop;
|
||||
Python tool at `tools/import-normalizer/`.
|
||||
- Century rule fixed by Marcel: archive spans **1873–1957**; 2-digit `00–57`→19YY,
|
||||
`73–99`→18YY, `58–72`→flag; 3-digit→1DDD; never 20xx.
|
||||
- Wrote [`02-normalization-spec.md`](./02-normalization-spec.md) in the requirements-engineer
|
||||
persona (FR/NFR, Given-When-Then ACs, traceability to IMP-NN, TBD register).
|
||||
|
||||
**All 6 open questions resolved (spec §9):** OQ-01 — movable feasts (Ostern, Pfingsten, …)
|
||||
**computed per year from Easter**, never a fixed month; seasons → mid-season month
|
||||
(Sommer=Jul, Herbst=Oct). OQ-02 ranges → start+RANGE. OQ-03 slug ids. OQ-04 — `x`-suffix rows
|
||||
**skipped + logged** this pass (they're transcriptions of the base letter, not yet mappable).
|
||||
OQ-05 → `.xlsx`. OQ-06 → conservative, no silent merge.
|
||||
|
||||
**Git:** moved off the unrelated `feat/issue-356-…` branch; pulled `main`; created clean
|
||||
branch **`docs/import-migration`** and committed these docs there. (The dirty `.venv`
|
||||
pycache + `skills/implement/SKILL.md` in the tree are pre-existing/environmental noise — left
|
||||
uncommitted, not ours.)
|
||||
|
||||
**Next:**
|
||||
- Marcel reviews the spec.
|
||||
- Then writing-plans → build the normalizer at `tools/import-normalizer/` (backlog B1–B7 are
|
||||
the Musts; B3 date parser incl. Easter computus is the big one).
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-25 (session 1) — Initial analysis
|
||||
|
||||
**Did:**
|
||||
- Got the real raw archive xlsx (7,943 rows) + person register (163 people). PDFs to follow.
|
||||
- Compared the new xlsx layout against `MassImportService` defaults and the old ODS.
|
||||
- Full statistical scan of all rows: dates, indices, senders/receivers, file column.
|
||||
- Wrote [`01-findings-spreadsheet-analysis.md`](./01-findings-spreadsheet-analysis.md)
|
||||
with 12 issues (IMP-01..IMP-12) + recommended sequencing.
|
||||
- Installed `openpyxl` into the OCR service venv for inspection.
|
||||
|
||||
**Key facts established:**
|
||||
- Importer defaults match the **ODS**, not the new xlsx → wrong column mapping (IMP-01).
|
||||
- **90%** of dated rows (6,571 / 7,319) are free-text dates the ISO-only parser drops (IMP-02).
|
||||
- Person register is rich but **unimported**; holds the maiden-name dedup key (IMP-04/05).
|
||||
|
||||
**Decisions pending from Marcel (blockers for any code work):**
|
||||
1. IMP-01: positional re-config of `app.import.col.*` vs header-driven mapping rewrite?
|
||||
2. IMP-02: how to store imprecise dates — new `dateOriginal` + `precision` columns, or lossy?
|
||||
3. IMP-04/05: format for the person/alias mapping; import persons before documents?
|
||||
4. IMP-10: are `x`-suffix rows separate documents, attachments, or skipped?
|
||||
|
||||
**Next:**
|
||||
- Get Marcel's calls on the 4 decisions above.
|
||||
- Then split the code-change items into Gitea issues (IMP-01b, IMP-02, IMP-04, IMP-06, IMP-12).
|
||||
- Pure-data tasks (IMP-07 dup list, IMP-09 file reconcile) stay here.
|
||||
1329
docs/superpowers/plans/2026-05-25-personendatei-importer.md
Normal file
1329
docs/superpowers/plans/2026-05-25-personendatei-importer.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,292 @@
|
||||
# Personendatei Importer — Design Spec
|
||||
|
||||
**Date:** 2026-05-25
|
||||
**Source file:** `import/Personendatei 2.xlsx`
|
||||
**Output:** `tools/import-normalizer/out/canonical-persons-tree.json`
|
||||
**Tool location:** `tools/import-normalizer/persons_tree.py`
|
||||
|
||||
---
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
Normalize the 163-person family register in `Personendatei 2.xlsx` into a machine-readable JSON file that a future backend importer can consume to seed the `persons` and `person_relationships` tables. The tool is offline (no backend required) and produces a reviewable artifact with an explicit `unresolved[]` list for manual follow-up.
|
||||
|
||||
---
|
||||
|
||||
## 2. Source Data — Column Map
|
||||
|
||||
Sheet: `Tabelle1` (rows 2–164; row 1 is the header).
|
||||
|
||||
| Col | Header | Content | Notes |
|
||||
|-----|--------|---------|-------|
|
||||
| A | Generation | `G 0`–`G 5` | Generation relative to Herbert & Clara Cram (G 2). Inconsistent formatting: `"G3"`, `"G 0"`, `"G 2 de Gruyter"` — strip non-digit chars and parse the integer. |
|
||||
| B | Familienname | Last name | Sometimes compound: `"de Gruyter"`, `"Cram Heydrich"`, `"Burkhard- Meier"` |
|
||||
| C | Vorname | First name | Sometimes multiple: `"Charlotte,Meta,Jacobi"`, nicknames in parens: `"Otto (Herbert)"` |
|
||||
| D | geb als | Maiden name | Used as a name alias for matching |
|
||||
| E | Geburtsdatum | Birth date | **Mixed types** — see §4 |
|
||||
| F | Geburtsort | Birth place | Free-text string, stored verbatim |
|
||||
| G | Todesdatum | Death date | Same mixed types as col E |
|
||||
| H | Sterbeort | Death place | Free-text string, stored verbatim |
|
||||
| I | verheiratet mit | Spouse name | Partial name in either `"Firstname Lastname"` or `"Lastname Firstname"` order |
|
||||
| J | Bemerkung | German relationship notes | `"Sohn v Clara u Herbert"`, `"Nichte v Herbert"`, free text |
|
||||
|
||||
---
|
||||
|
||||
## 3. Two-Pass Architecture
|
||||
|
||||
### Pass 1 — Parse & Normalize (rows → person records)
|
||||
|
||||
For each row:
|
||||
1. Read all 10 columns.
|
||||
2. Assign a stable `rowId`: `"row_{i:03d}"` where `i` is the 1-based row number (e.g. `row_002`).
|
||||
3. Normalize fields per §4 and §5.
|
||||
4. Build the **name-lookup index** (see §6).
|
||||
5. Emit a person record.
|
||||
|
||||
### Pass 2 — Resolve Relationships
|
||||
|
||||
Walk every person record:
|
||||
1. Resolve col I (spouse) → emit `SPOUSE_OF` edge or `unresolved` entry.
|
||||
2. Parse col J (Bemerkung) for parent/child patterns → emit `PARENT_OF` edges or `unresolved` entries.
|
||||
3. Append unmatched Bemerkung text to `person.notes`.
|
||||
|
||||
---
|
||||
|
||||
## 4. Date Parsing
|
||||
|
||||
Both col E (birth) and col G (death) arrive as either an Excel numeric serial or a string.
|
||||
|
||||
### Excel serial conversion
|
||||
When the cell value is an integer (or a float with no string representation):
|
||||
```
|
||||
date = datetime(1899, 12, 30) + timedelta(days=int(value))
|
||||
year = date.year
|
||||
```
|
||||
Excel's epoch is 1899-12-30 (accounts for the Lotus 1-2-3 leap-year bug).
|
||||
|
||||
### String fallback — reuse existing `dates.parse_date()`
|
||||
Pass the raw string to the existing `tools/import-normalizer/dates.parse_date()`. It already handles:
|
||||
- `DD.MM.YYYY` and `D.M.YY`
|
||||
- Year-only (`1930`)
|
||||
- Month + year (`August 1941`, `Sept. 1913`)
|
||||
- Partial/approximate markers
|
||||
|
||||
Extract `.year` from the returned `ParsedDate.iso` if `iso` is not `None`.
|
||||
|
||||
### Unresolvable dates
|
||||
If both paths yield `None` (e.g. `"2.9.196"`, `"4.3.1023"`, `".12.1955"`):
|
||||
- Set `birthYear`/`deathYear` to `null`.
|
||||
- Append the raw value to `person.notes` as `"[Geburtsdatum: <raw>]"` or `"[Todesdatum: <raw>]"` for human review.
|
||||
|
||||
---
|
||||
|
||||
## 5. Person Record Normalization
|
||||
|
||||
### Name fields
|
||||
- **lastName** = col B, stripped.
|
||||
- **firstName** = col C. Keep as-is (including multi-name strings and parenthetical nicknames) — the backend can split later.
|
||||
- **maidenName** = col D, stripped. Stored in the JSON; the backend maps this to a `PersonNameAlias` of type `BIRTH_NAME`.
|
||||
- **alias** = `null` (the tool does not invent aliases; maiden name is the alias).
|
||||
|
||||
### Generation
|
||||
Extract the first digit sequence from col A:
|
||||
```python
|
||||
import re
|
||||
m = re.search(r"\d+", raw_generation)
|
||||
generation = int(m.group()) if m else None
|
||||
```
|
||||
Handles all observed variants: `"G 3"`, `"G3"`, `"G 0"`, `"G 2 de Gruyter"`, `"G 0"`.
|
||||
Stored as `generation: int | null` in the JSON (informational; not mapped to a backend field directly).
|
||||
|
||||
### familyMember
|
||||
Set `true` for all records. Every person in this register is part of the family network. The backend can refine this.
|
||||
|
||||
### notes
|
||||
Constructed by concatenation:
|
||||
1. Unmatched Bemerkung text (after relationship pattern is stripped).
|
||||
2. Unresolvable date raw values (prefixed with field name).
|
||||
|
||||
---
|
||||
|
||||
## 6. Name Lookup Index
|
||||
|
||||
After pass 1, build a `dict[str, list[str]]` mapping normalized name keys → list of `rowId`s.
|
||||
|
||||
### Normalization function `_norm(s) -> str`
|
||||
1. Lowercase.
|
||||
2. Strip surrounding `"` and `'`.
|
||||
3. Remove parenthetical substrings: `r"\([^)]*\)"`.
|
||||
4. Collapse internal whitespace.
|
||||
5. Strip geographic/honorific suffixes: `aachen`, `mex.`, `mexiko`, `sen`, `jun`, `jr`.
|
||||
6. Strip trailing commas, dots.
|
||||
|
||||
### Keys indexed per person
|
||||
For a person with firstName `F`, lastName `L`, maidenName `M`:
|
||||
- `_norm(f"{F} {L}")` — canonical order
|
||||
- `_norm(f"{L} {F}")` — reversed order (col I uses this heavily)
|
||||
- `_norm(f"{F} {M}")` if maidenName is set — maiden-name reference
|
||||
- `_norm(L)` alone — single-token fallback
|
||||
|
||||
### Match resolution
|
||||
Given a raw name string from col I or col J:
|
||||
1. `_norm(raw)` → look up in index.
|
||||
2. **Exactly one hit** → match confirmed, use that `rowId`.
|
||||
3. **Zero hits** → `reason: "not_found"` → `unresolved[]`.
|
||||
4. **Multiple hits** → `reason: "ambiguous"` → `unresolved[]`.
|
||||
|
||||
---
|
||||
|
||||
## 7. Relationship Extraction
|
||||
|
||||
### 7.1 SPOUSE_OF (col I — `verheiratet mit`)
|
||||
|
||||
1. Normalize col I value.
|
||||
2. Resolve via name index (§6).
|
||||
3. If matched: emit one edge `{ personId, relatedPersonId, type: "SPOUSE_OF", source: "verheiratet_mit" }`.
|
||||
- Skip if an identical edge (regardless of direction) already exists in the relationship list.
|
||||
4. If unresolved: add to `unresolved[]`.
|
||||
|
||||
### 7.2 PARENT_OF (col J — `Bemerkung`)
|
||||
|
||||
Apply these regex patterns in order, case-insensitive, with optional whitespace:
|
||||
|
||||
| Pattern | Direction | Note |
|
||||
|---------|-----------|------|
|
||||
| `(Sohn\|Tochter)\s+v(?:on)?\s+(.+)` | Named person(s) → this person | "Sohn v Clara u Herbert" |
|
||||
| `(Vater\|Mutter)\s+v(?:on)?\s+(.+)` | This person → named person(s) | "Vater v Herbert" |
|
||||
|
||||
**Multi-parent extraction:** The parent string may contain two parents joined by `\s+u(?:nd)?\s+`. Split on this pattern, resolve each part independently.
|
||||
|
||||
**Emit** one `PARENT_OF` edge per resolved parent:
|
||||
```json
|
||||
{
|
||||
"personId": "<parent_rowId>",
|
||||
"relatedPersonId": "<child_rowId>",
|
||||
"type": "PARENT_OF",
|
||||
"source": "bemerkung",
|
||||
"rawBemerkung": "<original col J value>"
|
||||
}
|
||||
```
|
||||
|
||||
**Skip** (do not emit, do not add to `unresolved[]`, leave in notes):
|
||||
- Patterns starting with `Neffe`, `Nichte`, `Enkel`, `Enkelin`, `Urenkel`, `Urenkelin` — too indirect.
|
||||
- Patterns starting with `Bruder`, `Schwester` — SIBLING_OF is out of scope for this tool.
|
||||
- Any other Bemerkung text that does not match the parent patterns.
|
||||
|
||||
**After extraction:** the matched portion of the Bemerkung is removed; the remainder goes into `person.notes`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Output JSON Schema
|
||||
|
||||
File: `tools/import-normalizer/out/canonical-persons-tree.json`
|
||||
|
||||
```json
|
||||
{
|
||||
"generated_at": "<ISO-8601 timestamp>",
|
||||
"source": "Personendatei 2.xlsx",
|
||||
"stats": {
|
||||
"persons": 163,
|
||||
"relationships": 87,
|
||||
"unresolved": 12
|
||||
},
|
||||
"persons": [
|
||||
{
|
||||
"rowId": "row_002",
|
||||
"firstName": "Elsgard",
|
||||
"lastName": "Allemeyer",
|
||||
"maidenName": "Wöhler",
|
||||
"alias": null,
|
||||
"notes": "Nichte von Herbert",
|
||||
"birthYear": 1920,
|
||||
"deathYear": 1999,
|
||||
"birthPlace": "Garz",
|
||||
"deathPlace": "Espelkamp",
|
||||
"generation": 3,
|
||||
"familyMember": true
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"personId": "row_002",
|
||||
"relatedPersonId": "row_003",
|
||||
"type": "SPOUSE_OF",
|
||||
"source": "verheiratet_mit"
|
||||
},
|
||||
{
|
||||
"personId": "row_019",
|
||||
"relatedPersonId": "row_021",
|
||||
"type": "PARENT_OF",
|
||||
"source": "bemerkung",
|
||||
"rawBemerkung": "Tochter v Clara u Herbert"
|
||||
}
|
||||
],
|
||||
"unresolved": [
|
||||
{
|
||||
"rowId": "row_007",
|
||||
"field": "verheiratet_mit",
|
||||
"raw": "\"Tante Lolly\"",
|
||||
"reason": "not_found"
|
||||
},
|
||||
{
|
||||
"rowId": "row_042",
|
||||
"field": "bemerkung",
|
||||
"raw": "Zwillingsbruder v Herbert",
|
||||
"reason": "not_found"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. CLI Interface
|
||||
|
||||
```
|
||||
python3 persons_tree.py [--input PATH] [--output PATH] [--dry-run]
|
||||
```
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--input` | `../../import/Personendatei 2.xlsx` | Source Excel file |
|
||||
| `--output` | `out/canonical-persons-tree.json` | Output JSON file |
|
||||
| `--dry-run` | off | Print stats + first 5 unresolved entries; do not write file |
|
||||
|
||||
On success, print:
|
||||
```
|
||||
✓ 163 persons parsed
|
||||
✓ 87 relationships emitted (52 SPOUSE_OF, 35 PARENT_OF)
|
||||
⚠ 12 unresolved (see unresolved[] in output)
|
||||
→ out/canonical-persons-tree.json
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Module Reuse
|
||||
|
||||
| Existing module | What we reuse |
|
||||
|-----------------|---------------|
|
||||
| `dates.parse_date()` | String date parsing — handles DD.MM.YYYY, year-only, month+year, approximate markers |
|
||||
| `config.MONTHS` | Month name → integer mapping (German + Spanish month names already present) |
|
||||
|
||||
The Excel serial conversion is new logic added directly in `persons_tree.py` (3 lines).
|
||||
|
||||
---
|
||||
|
||||
## 11. What This Tool Does NOT Do
|
||||
|
||||
- Does not call the backend API or touch the database.
|
||||
- Does not create `PersonNameAlias` records — it emits `maidenName` as a field; the future backend importer maps it.
|
||||
- Does not infer SIBLING_OF edges (requires symmetric lookup across multiple rows — deferred).
|
||||
- Does not deduplicate persons that appear in both this file and `canonical-persons.xlsx` — deduplication is the backend importer's responsibility.
|
||||
- Does produce `birthPlace` / `deathPlace` as top-level fields in the JSON (see §8) — they are free-text strings and informational only. The `Person` entity has no corresponding columns; the future backend importer decides whether to add columns or fold the values into `notes`.
|
||||
|
||||
---
|
||||
|
||||
## 12. Resolved Decisions
|
||||
|
||||
| OQ | Question | Decision |
|
||||
|----|----------|----------|
|
||||
| OQ-01 | Duplicate rows (127/138 — Christa Schütz; 129/139 — Christoph Seils). | **Tool deduplicates.** On pass 1, after building the person list, detect rows with identical `(firstName, lastName, birthYear)` and keep only the first occurrence. Log skipped row ids to stdout. |
|
||||
| OQ-02 | `birthPlace` / `deathPlace` absent from `Person` entity. | **Keep as separate top-level fields** in the JSON (`birthPlace`, `deathPlace`). The future backend importer may add columns to the `persons` table; the field is preserved here to avoid data loss. |
|
||||
| OQ-03 | `firstName` = `"Charlotte,Meta,Jacobi"` (multi-name comma string). | **Store verbatim as `firstName`.** No splitting. |
|
||||
@@ -28,7 +28,7 @@ src/
|
||||
│ ├── +layout.server.ts # Loads current user, injects auth cookie
|
||||
│ ├── +page.svelte # Home / document search dashboard
|
||||
│ ├── documents/ # Document CRUD, detail, edit, upload
|
||||
│ ├── persons/ # Person directory, detail, edit, merge
|
||||
│ ├── persons/ # Person directory (filtered, paginated), detail, edit, merge, review (triage)
|
||||
│ ├── briefwechsel/ # Bilateral conversation timeline
|
||||
│ ├── aktivitaeten/ # Unified activity feed (Chronik)
|
||||
│ ├── admin/ # User, group, tag, OCR, system management
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"error_file_too_large": "Die Datei ist zu groß (max. 50 MB).",
|
||||
"error_user_not_found": "Der Benutzer wurde nicht gefunden.",
|
||||
"error_import_already_running": "Ein Import läuft bereits. Bitte warten Sie, bis dieser abgeschlossen ist.",
|
||||
"error_import_artifact_invalid": "Eine Importdatei fehlt oder ist ungültig. Bitte führen Sie den Normalizer erneut aus.",
|
||||
"error_invalid_credentials": "E-Mail-Adresse oder Passwort ist falsch.",
|
||||
"error_session_expired": "Ihre Sitzung ist abgelaufen. Bitte melden Sie sich erneut an.",
|
||||
"error_session_expired_explainer": "Aus Sicherheitsgründen werden Sitzungen nach 8 Stunden Inaktivität automatisch beendet.",
|
||||
@@ -99,6 +100,9 @@
|
||||
"docs_list_summary": "Zusammenfassung",
|
||||
"docs_list_unknown": "Unbekannt",
|
||||
"docs_group_undated": "Undatiert",
|
||||
"docs_filter_undated_only": "Nur undatierte",
|
||||
"docs_filter_undated_count_label": "{count} undatierte Dokumente",
|
||||
"docs_range_excludes_undated": "Ein Datumsfilter schließt undatierte Dokumente aus, da sie keinem Zeitraum zugeordnet werden können.",
|
||||
"docs_group_unknown": "Unbekannt",
|
||||
"doc_section_who_when": "Wer & Wann",
|
||||
"doc_section_description": "Beschreibung",
|
||||
@@ -129,6 +133,31 @@
|
||||
"persons_search_placeholder": "Namen suchen...",
|
||||
"persons_empty_heading": "Keine Personen gefunden.",
|
||||
"persons_empty_text": "Versuchen Sie einen anderen Suchbegriff.",
|
||||
"persons_empty_filtered": "Keine Personen für diese Filter.",
|
||||
"persons_filter_group_label": "Filter",
|
||||
"persons_filter_type_person": "Person",
|
||||
"persons_filter_type_group": "Gruppe",
|
||||
"persons_filter_type_institution": "Institution",
|
||||
"persons_filter_family_only": "Nur Familie",
|
||||
"persons_filter_has_documents": "Mit Dokumenten",
|
||||
"persons_toggle_show_all": "Alle anzeigen",
|
||||
"persons_toggle_needs_review": "Zu prüfen ({count})",
|
||||
"person_badge_unconfirmed": "unbestätigt",
|
||||
"persons_review_heading": "Personen prüfen",
|
||||
"persons_review_intro": "Vom Import erzeugte, noch nicht bestätigte Personen. Zusammenführen, umbenennen, bestätigen oder löschen.",
|
||||
"persons_review_action_merge": "Zusammenführen",
|
||||
"persons_review_action_rename": "Umbenennen",
|
||||
"persons_review_action_confirm": "Bestätigen",
|
||||
"persons_review_action_delete": "Löschen",
|
||||
"persons_review_action_cancel": "Abbrechen",
|
||||
"persons_review_action_save": "Speichern",
|
||||
"persons_review_empty": "Keine Personen zu prüfen.",
|
||||
"persons_review_delete_confirm_title": "Person löschen",
|
||||
"persons_review_delete_confirm_text": "Diese Person wird endgültig gelöscht. Dokumentverweise bleiben erhalten, verlieren aber diese Person.",
|
||||
"persons_review_delete_confirm_button": "Person löschen",
|
||||
"persons_review_merge_label": "Mit welcher Person zusammenführen?",
|
||||
"persons_field_first_name": "Vorname",
|
||||
"persons_field_last_name": "Nachname",
|
||||
"persons_new_heading": "Neue Person",
|
||||
"persons_section_details": "Angaben zur Person",
|
||||
"person_edit_heading": "Person bearbeiten",
|
||||
@@ -260,6 +289,24 @@
|
||||
"doc_preview_iframe_title": "Dokumentvorschau",
|
||||
"doc_image_alt": "Original-Scan",
|
||||
"doc_no_date": "Kein Datum",
|
||||
"date_precision_unknown": "Datum unbekannt",
|
||||
"date_precision_approx_prefix": "ca.",
|
||||
"date_range_open_prefix": "ab",
|
||||
"date_season_spring": "Frühling",
|
||||
"date_season_summer": "Sommer",
|
||||
"date_season_autumn": "Herbst",
|
||||
"date_season_winter": "Winter",
|
||||
"date_original_label": "Originaltext:",
|
||||
"date_unknown_icon_label": "Datum unbekannt",
|
||||
"form_label_date_precision": "Datumsgenauigkeit",
|
||||
"form_label_date_end": "Enddatum",
|
||||
"date_precision_option_day": "Genauer Tag",
|
||||
"date_precision_option_month": "Monat",
|
||||
"date_precision_option_season": "Jahreszeit",
|
||||
"date_precision_option_year": "Jahr",
|
||||
"date_precision_option_range": "Zeitraum",
|
||||
"date_precision_option_approx": "Ungefähr",
|
||||
"date_precision_option_unknown": "Unbekannt",
|
||||
"person_merge_will_be_deleted": "wird gelöscht.",
|
||||
"comp_typeahead_placeholder": "Namen tippen...",
|
||||
"comp_typeahead_loading": "Suche...",
|
||||
@@ -356,11 +403,13 @@
|
||||
"admin_system_import_status_done_label": "Dokumente verarbeitet",
|
||||
"admin_system_import_skipped_label": "übersprungen",
|
||||
"import_reason_invalid_pdf_signature": "Keine gültige PDF-Signatur",
|
||||
"import_reason_path_traversal": "Ungültiger Dateiname (Pfad)",
|
||||
"import_reason_file_read_error": "Fehler beim Lesen der Datei",
|
||||
"import_reason_s3_upload_failed": "Upload-Fehler (S3)",
|
||||
"import_reason_already_exists": "Bereits importiert",
|
||||
"admin_system_import_status_failed": "Import fehlgeschlagen",
|
||||
"admin_system_import_failed_no_spreadsheet": "Keine Tabellendatei gefunden.",
|
||||
"admin_system_import_failed_artifact": "Eine Importdatei fehlt oder ist ungültig.",
|
||||
"admin_system_import_failed_internal": "Interner Fehler beim Import.",
|
||||
"admin_system_thumbnails_heading": "Thumbnails erzeugen",
|
||||
"admin_system_thumbnails_description": "Erzeugt Vorschaubilder für Dokumente ohne Thumbnail (z. B. nach dem Massenimport).",
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"error_file_too_large": "The file is too large (max. 50 MB).",
|
||||
"error_user_not_found": "User not found.",
|
||||
"error_import_already_running": "An import is already running. Please wait for it to finish.",
|
||||
"error_import_artifact_invalid": "A canonical import file is missing or invalid. Please re-run the normalizer.",
|
||||
"error_invalid_credentials": "Email address or password is incorrect.",
|
||||
"error_session_expired": "Your session has expired. Please sign in again.",
|
||||
"error_session_expired_explainer": "For security reasons, sessions are automatically ended after 8 hours of inactivity.",
|
||||
@@ -99,6 +100,9 @@
|
||||
"docs_list_summary": "Summary",
|
||||
"docs_list_unknown": "Unknown",
|
||||
"docs_group_undated": "Undated",
|
||||
"docs_filter_undated_only": "Undated only",
|
||||
"docs_filter_undated_count_label": "{count} undated documents",
|
||||
"docs_range_excludes_undated": "A date range filter excludes undated documents, because they cannot belong to any time span.",
|
||||
"docs_group_unknown": "Unknown",
|
||||
"doc_section_who_when": "Who & When",
|
||||
"doc_section_description": "Description",
|
||||
@@ -129,6 +133,31 @@
|
||||
"persons_search_placeholder": "Search names...",
|
||||
"persons_empty_heading": "No persons found.",
|
||||
"persons_empty_text": "Try a different search term.",
|
||||
"persons_empty_filtered": "No persons match these filters.",
|
||||
"persons_filter_group_label": "Filter",
|
||||
"persons_filter_type_person": "Person",
|
||||
"persons_filter_type_group": "Group",
|
||||
"persons_filter_type_institution": "Institution",
|
||||
"persons_filter_family_only": "Family only",
|
||||
"persons_filter_has_documents": "With documents",
|
||||
"persons_toggle_show_all": "Show all",
|
||||
"persons_toggle_needs_review": "Needs review ({count})",
|
||||
"person_badge_unconfirmed": "unconfirmed",
|
||||
"persons_review_heading": "Review persons",
|
||||
"persons_review_intro": "Import-generated persons not yet confirmed. Merge, rename, confirm or delete.",
|
||||
"persons_review_action_merge": "Merge",
|
||||
"persons_review_action_rename": "Rename",
|
||||
"persons_review_action_confirm": "Confirm",
|
||||
"persons_review_action_delete": "Delete",
|
||||
"persons_review_action_cancel": "Cancel",
|
||||
"persons_review_action_save": "Save",
|
||||
"persons_review_empty": "No persons to review.",
|
||||
"persons_review_delete_confirm_title": "Delete person",
|
||||
"persons_review_delete_confirm_text": "This person will be permanently deleted. Document references are kept but lose this person.",
|
||||
"persons_review_delete_confirm_button": "Delete person",
|
||||
"persons_review_merge_label": "Merge into which person?",
|
||||
"persons_field_first_name": "First name",
|
||||
"persons_field_last_name": "Last name",
|
||||
"persons_new_heading": "New person",
|
||||
"persons_section_details": "Person details",
|
||||
"person_edit_heading": "Edit person",
|
||||
@@ -260,6 +289,24 @@
|
||||
"doc_preview_iframe_title": "Document Preview",
|
||||
"doc_image_alt": "Original scan",
|
||||
"doc_no_date": "No date",
|
||||
"date_precision_unknown": "Date unknown",
|
||||
"date_precision_approx_prefix": "c.",
|
||||
"date_range_open_prefix": "from",
|
||||
"date_season_spring": "Spring",
|
||||
"date_season_summer": "Summer",
|
||||
"date_season_autumn": "Autumn",
|
||||
"date_season_winter": "Winter",
|
||||
"date_original_label": "Original:",
|
||||
"date_unknown_icon_label": "Date unknown",
|
||||
"form_label_date_precision": "Date precision",
|
||||
"form_label_date_end": "End date",
|
||||
"date_precision_option_day": "Exact day",
|
||||
"date_precision_option_month": "Month",
|
||||
"date_precision_option_season": "Season",
|
||||
"date_precision_option_year": "Year",
|
||||
"date_precision_option_range": "Range",
|
||||
"date_precision_option_approx": "Approximate",
|
||||
"date_precision_option_unknown": "Unknown",
|
||||
"person_merge_will_be_deleted": "will be deleted.",
|
||||
"comp_typeahead_placeholder": "Type a name...",
|
||||
"comp_typeahead_loading": "Searching...",
|
||||
@@ -356,11 +403,13 @@
|
||||
"admin_system_import_status_done_label": "Documents processed",
|
||||
"admin_system_import_skipped_label": "skipped",
|
||||
"import_reason_invalid_pdf_signature": "Invalid PDF signature",
|
||||
"import_reason_path_traversal": "Invalid filename (path)",
|
||||
"import_reason_file_read_error": "File read error",
|
||||
"import_reason_s3_upload_failed": "Upload error (S3)",
|
||||
"import_reason_already_exists": "Already imported",
|
||||
"admin_system_import_status_failed": "Import failed",
|
||||
"admin_system_import_failed_no_spreadsheet": "No spreadsheet file found.",
|
||||
"admin_system_import_failed_artifact": "A canonical import file is missing or invalid.",
|
||||
"admin_system_import_failed_internal": "Import failed due to an internal error.",
|
||||
"admin_system_thumbnails_heading": "Generate thumbnails",
|
||||
"admin_system_thumbnails_description": "Generates preview images for documents without a thumbnail (e.g. after the mass import).",
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"error_file_too_large": "El archivo es demasiado grande (máx. 50 MB).",
|
||||
"error_user_not_found": "Usuario no encontrado.",
|
||||
"error_import_already_running": "Ya hay una importación en curso. Por favor, espere a que finalice.",
|
||||
"error_import_artifact_invalid": "Falta un archivo de importación canónico o no es válido. Vuelva a ejecutar el normalizador.",
|
||||
"error_invalid_credentials": "El correo electrónico o la contraseña son incorrectos.",
|
||||
"error_session_expired": "Su sesión ha expirado. Por favor, inicie sesión de nuevo.",
|
||||
"error_session_expired_explainer": "Por razones de seguridad, las sesiones se terminan automáticamente tras 8 horas de inactividad.",
|
||||
@@ -99,6 +100,9 @@
|
||||
"docs_list_summary": "Resumen",
|
||||
"docs_list_unknown": "Desconocido",
|
||||
"docs_group_undated": "Sin fecha",
|
||||
"docs_filter_undated_only": "Solo sin fecha",
|
||||
"docs_filter_undated_count_label": "{count} documentos sin fecha",
|
||||
"docs_range_excludes_undated": "Un filtro de intervalo de fechas excluye los documentos sin fecha, ya que no pueden pertenecer a ningún periodo.",
|
||||
"docs_group_unknown": "Desconocido",
|
||||
"doc_section_who_when": "Quién & Cuándo",
|
||||
"doc_section_description": "Descripción",
|
||||
@@ -129,6 +133,31 @@
|
||||
"persons_search_placeholder": "Buscar nombres...",
|
||||
"persons_empty_heading": "No se encontraron personas.",
|
||||
"persons_empty_text": "Pruebe con otro término de búsqueda.",
|
||||
"persons_empty_filtered": "Ninguna persona coincide con estos filtros.",
|
||||
"persons_filter_group_label": "Filtro",
|
||||
"persons_filter_type_person": "Persona",
|
||||
"persons_filter_type_group": "Grupo",
|
||||
"persons_filter_type_institution": "Institución",
|
||||
"persons_filter_family_only": "Solo familia",
|
||||
"persons_filter_has_documents": "Con documentos",
|
||||
"persons_toggle_show_all": "Mostrar todo",
|
||||
"persons_toggle_needs_review": "Por revisar ({count})",
|
||||
"person_badge_unconfirmed": "sin confirmar",
|
||||
"persons_review_heading": "Revisar personas",
|
||||
"persons_review_intro": "Personas generadas por la importación aún sin confirmar. Fusionar, renombrar, confirmar o eliminar.",
|
||||
"persons_review_action_merge": "Fusionar",
|
||||
"persons_review_action_rename": "Renombrar",
|
||||
"persons_review_action_confirm": "Confirmar",
|
||||
"persons_review_action_delete": "Eliminar",
|
||||
"persons_review_action_cancel": "Cancelar",
|
||||
"persons_review_action_save": "Guardar",
|
||||
"persons_review_empty": "No hay personas por revisar.",
|
||||
"persons_review_delete_confirm_title": "Eliminar persona",
|
||||
"persons_review_delete_confirm_text": "Esta persona se eliminará de forma permanente. Las referencias de documentos se conservan pero pierden a esta persona.",
|
||||
"persons_review_delete_confirm_button": "Eliminar persona",
|
||||
"persons_review_merge_label": "¿Fusionar con qué persona?",
|
||||
"persons_field_first_name": "Nombre",
|
||||
"persons_field_last_name": "Apellido",
|
||||
"persons_new_heading": "Nueva persona",
|
||||
"persons_section_details": "Datos de la persona",
|
||||
"person_edit_heading": "Editar persona",
|
||||
@@ -260,6 +289,24 @@
|
||||
"doc_preview_iframe_title": "Vista previa del documento",
|
||||
"doc_image_alt": "Escaneado original",
|
||||
"doc_no_date": "Sin fecha",
|
||||
"date_precision_unknown": "Fecha desconocida",
|
||||
"date_precision_approx_prefix": "ca.",
|
||||
"date_range_open_prefix": "desde",
|
||||
"date_season_spring": "Primavera",
|
||||
"date_season_summer": "Verano",
|
||||
"date_season_autumn": "Otoño",
|
||||
"date_season_winter": "Invierno",
|
||||
"date_original_label": "Texto original:",
|
||||
"date_unknown_icon_label": "Fecha desconocida",
|
||||
"form_label_date_precision": "Precisión de la fecha",
|
||||
"form_label_date_end": "Fecha final",
|
||||
"date_precision_option_day": "Día exacto",
|
||||
"date_precision_option_month": "Mes",
|
||||
"date_precision_option_season": "Estación",
|
||||
"date_precision_option_year": "Año",
|
||||
"date_precision_option_range": "Periodo",
|
||||
"date_precision_option_approx": "Aproximada",
|
||||
"date_precision_option_unknown": "Desconocida",
|
||||
"person_merge_will_be_deleted": "será eliminado.",
|
||||
"comp_typeahead_placeholder": "Escriba un nombre...",
|
||||
"comp_typeahead_loading": "Buscando...",
|
||||
@@ -356,11 +403,13 @@
|
||||
"admin_system_import_status_done_label": "Documentos procesados",
|
||||
"admin_system_import_skipped_label": "omitidos",
|
||||
"import_reason_invalid_pdf_signature": "Firma PDF no válida",
|
||||
"import_reason_path_traversal": "Nombre de archivo no válido (ruta)",
|
||||
"import_reason_file_read_error": "Error al leer el archivo",
|
||||
"import_reason_s3_upload_failed": "Error de carga (S3)",
|
||||
"import_reason_already_exists": "Ya importado",
|
||||
"admin_system_import_status_failed": "Importación fallida",
|
||||
"admin_system_import_failed_no_spreadsheet": "No se encontró ninguna hoja de cálculo.",
|
||||
"admin_system_import_failed_artifact": "Falta un archivo de importación canónico o no es válido.",
|
||||
"admin_system_import_failed_internal": "Error interno durante la importación.",
|
||||
"admin_system_thumbnails_heading": "Generar miniaturas",
|
||||
"admin_system_thumbnails_description": "Genera imágenes de vista previa para documentos sin miniatura (p. ej. tras la importación masiva).",
|
||||
|
||||
@@ -44,6 +44,17 @@ describe('ChronikRow', () => {
|
||||
expect(link).not.toBeNull();
|
||||
});
|
||||
|
||||
// --- #668 negative guarantee: Chronik never fabricates a letter date ---
|
||||
it('renders the activity timestamp, not a letter date, and no undated badge', async () => {
|
||||
// The row shows the relative activity time (happenedAt), never the letter's
|
||||
// documentDate — ActivityFeedItemDTO carries no date surface to badge.
|
||||
render(ChronikRow, { item: baseItem });
|
||||
// No undated badge is introduced into a Chronik row.
|
||||
expect(document.querySelector('[data-testid="undated-badge"]')).toBeNull();
|
||||
// No fabricated "Datum unbekannt" letter-date label appears.
|
||||
await expect.element(page.getByText('Datum unbekannt')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
// --- simple variant ---
|
||||
it('renders simple variant when count === 1 and not a mention', async () => {
|
||||
render(ChronikRow, { item: baseItem });
|
||||
|
||||
70
frontend/src/lib/document/DocumentDate.svelte
Normal file
70
frontend/src/lib/document/DocumentDate.svelte
Normal file
@@ -0,0 +1,70 @@
|
||||
<script lang="ts">
|
||||
import { formatDocumentDate, type DatePrecision } from '$lib/shared/utils/documentDate';
|
||||
import { getLocale } from '$lib/paraglide/runtime.js';
|
||||
import { m } from '$lib/paraglide/messages.js';
|
||||
|
||||
type Props = {
|
||||
iso?: string | null;
|
||||
precision?: DatePrecision | null;
|
||||
end?: string | null;
|
||||
raw?: string | null;
|
||||
/** Show the verbatim "Originaltext: …" secondary line when raw is present. */
|
||||
showRaw?: boolean;
|
||||
};
|
||||
|
||||
let { iso = null, precision = null, end = null, raw = null, showRaw = true }: Props = $props();
|
||||
|
||||
const effectivePrecision = $derived<DatePrecision>(precision ?? (iso ? 'DAY' : 'UNKNOWN'));
|
||||
const label = $derived(formatDocumentDate(iso, effectivePrecision, end, raw, getLocale()));
|
||||
const isUnknown = $derived(effectivePrecision === 'UNKNOWN' || !iso);
|
||||
// Only show the verbatim raw line where it adds information the label can't: the
|
||||
// season word's source, or the original cell behind an "unknown"/approx date.
|
||||
const showRawLine = $derived(
|
||||
showRaw &&
|
||||
!!raw &&
|
||||
raw.trim().length > 0 &&
|
||||
(isUnknown || effectivePrecision === 'SEASON' || effectivePrecision === 'APPROX')
|
||||
);
|
||||
</script>
|
||||
|
||||
<span class="inline-flex flex-col">
|
||||
{#if isUnknown}
|
||||
<!--
|
||||
Neutral metadata chip (#668): an undated letter is an absence, NOT an error,
|
||||
so the chip is neutral (text-ink-3 on bg-surface, ≥4.5:1 in both themes) —
|
||||
never red/amber. Matches the archive-metadata chip pattern in the row. The
|
||||
non-color cue (WCAG 1.4.1) is the calendar-with-question glyph; the visible
|
||||
"Datum unbekannt" text is the redundant textual cue and IS the accessibility,
|
||||
so it is announced inline (never aria-hidden) while the icon is decorative.
|
||||
-->
|
||||
<span
|
||||
data-testid="undated-badge"
|
||||
class="inline-flex items-center gap-1 rounded border border-line px-1.5 py-0.5 font-sans text-xs tracking-widest text-ink-3 uppercase"
|
||||
>
|
||||
<svg
|
||||
class="h-3 w-3 shrink-0 text-ink-3"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect x="3" y="4" width="18" height="18" rx="2" />
|
||||
<path d="M3 10h18" />
|
||||
<path d="M9 16a1.5 1.5 0 0 1 3 0c0 1-1.5 1.2-1.5 2.2" />
|
||||
<path d="M10.5 21h.01" />
|
||||
</svg>
|
||||
{label}
|
||||
</span>
|
||||
{:else}
|
||||
<span>{label}</span>
|
||||
{/if}
|
||||
{#if showRawLine}
|
||||
<!-- Visible secondary line (WCAG 1.4.13 — not tooltip-only). raw is untrusted
|
||||
verbatim spreadsheet text; rendered via default Svelte interpolation, which
|
||||
HTML-escapes it (never {@html}; CWE-79). -->
|
||||
<span class="font-sans text-xs text-ink-2">{m.date_original_label()} {raw}</span>
|
||||
{/if}
|
||||
</span>
|
||||
35
frontend/src/lib/document/DocumentDate.svelte.test.ts
Normal file
35
frontend/src/lib/document/DocumentDate.svelte.test.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { cleanup, render } from 'vitest-browser-svelte';
|
||||
import { page } from 'vitest/browser';
|
||||
import DocumentDate from './DocumentDate.svelte';
|
||||
|
||||
// Browser-project (Playwright) tests — CI only.
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
describe('DocumentDate', () => {
|
||||
it('renders a DAY date as a full long date', async () => {
|
||||
render(DocumentDate, { props: { iso: '1943-12-24', precision: 'DAY' } });
|
||||
await expect.element(page.getByText('24. Dezember 1943')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders MONTH precision as month + year, never a day', async () => {
|
||||
render(DocumentDate, { props: { iso: '1916-06-01', precision: 'MONTH', raw: 'Juni 1916' } });
|
||||
await expect.element(page.getByText('Juni 1916')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows the verbatim raw cell as a visible secondary line for UNKNOWN (not tooltip-only)', async () => {
|
||||
render(DocumentDate, { props: { iso: null, precision: 'UNKNOWN', raw: 'Sommer?' } });
|
||||
// Real, visible text — not hidden behind a title attribute.
|
||||
await expect.element(page.getByText('Datum unbekannt')).toBeInTheDocument();
|
||||
await expect.element(page.getByText(/Sommer\?/)).toBeVisible();
|
||||
});
|
||||
|
||||
it('renders a malicious raw value as inert escaped text (no element injected)', async () => {
|
||||
const malicious = '<img src=x onerror="alert(1)">';
|
||||
render(DocumentDate, { props: { iso: null, precision: 'UNKNOWN', raw: malicious } });
|
||||
// The payload appears as literal text, and no <img> is created in the DOM.
|
||||
await expect.element(page.getByText(/<img/)).toBeInTheDocument();
|
||||
expect(document.querySelector('img')).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,7 @@ import WhoWhenSection from '$lib/document/WhoWhenSection.svelte';
|
||||
import DescriptionSection from '$lib/document/DescriptionSection.svelte';
|
||||
import type { Tag } from '$lib/tag/TagInput.svelte';
|
||||
import type { components } from '$lib/generated/api';
|
||||
import type { DatePrecision } from '$lib/shared/utils/documentDate';
|
||||
|
||||
type Person = components['schemas']['Person'];
|
||||
type Doc = components['schemas']['Document'];
|
||||
@@ -26,6 +27,8 @@ let {
|
||||
senderId = $bindable(''),
|
||||
selectedReceivers = $bindable<Person[]>([]),
|
||||
dateIso = $bindable(''),
|
||||
datePrecision = $bindable<DatePrecision>('DAY'),
|
||||
dateEndIso = $bindable(''),
|
||||
currentTitle = $bindable(''),
|
||||
topbar,
|
||||
actionbar
|
||||
@@ -38,6 +41,8 @@ let {
|
||||
senderId?: string;
|
||||
selectedReceivers?: Person[];
|
||||
dateIso?: string;
|
||||
datePrecision?: DatePrecision;
|
||||
dateEndIso?: string;
|
||||
currentTitle?: string;
|
||||
topbar: Snippet;
|
||||
actionbar: Snippet;
|
||||
@@ -47,6 +52,8 @@ tags = untrack(() => (doc.tags as Tag[]) ?? []);
|
||||
senderId = untrack(() => doc.sender?.id ?? '');
|
||||
selectedReceivers = untrack(() => (doc.receivers as Person[]) ?? []);
|
||||
dateIso = untrack(() => doc.documentDate ?? '');
|
||||
datePrecision = untrack(() => doc.metaDatePrecision ?? (doc.documentDate ? 'DAY' : 'UNKNOWN'));
|
||||
dateEndIso = untrack(() => doc.metaDateEnd ?? '');
|
||||
currentTitle = untrack(() => doc.title ?? '');
|
||||
|
||||
const fileLoader = createFileLoader();
|
||||
@@ -199,6 +206,9 @@ async function handleReplaceFile(e: Event) {
|
||||
bind:senderId={senderId}
|
||||
bind:selectedReceivers={selectedReceivers}
|
||||
bind:dateIso={dateIso}
|
||||
bind:precision={datePrecision}
|
||||
bind:endDateIso={dateEndIso}
|
||||
rawDate={doc.metaDateRaw ?? ''}
|
||||
initialDateIso={doc.documentDate ?? ''}
|
||||
initialLocation={doc.location ?? ''}
|
||||
initialSenderName={doc.sender?.displayName ?? ''}
|
||||
|
||||
@@ -4,6 +4,8 @@ import { formatDate } from '$lib/shared/utils/date';
|
||||
import { formatDocumentStatus } from '$lib/document/documentStatusLabel';
|
||||
import { getInitials, personAvatarColor } from '$lib/person/personFormat';
|
||||
import RelationshipPill from '$lib/person/relationship/RelationshipPill.svelte';
|
||||
import DocumentDate from './DocumentDate.svelte';
|
||||
import type { DatePrecision } from '$lib/shared/utils/documentDate';
|
||||
|
||||
type Person = { id: string; firstName?: string | null; lastName: string; displayName: string };
|
||||
type Tag = { id: string; name: string };
|
||||
@@ -16,6 +18,9 @@ type GeschichteSummary = {
|
||||
|
||||
type Props = {
|
||||
documentDate: string | null;
|
||||
metaDatePrecision?: DatePrecision | null;
|
||||
metaDateEnd?: string | null;
|
||||
metaDateRaw?: string | null;
|
||||
location: string | null;
|
||||
status: string;
|
||||
sender: Person | null;
|
||||
@@ -29,6 +34,9 @@ type Props = {
|
||||
|
||||
let {
|
||||
documentDate,
|
||||
metaDatePrecision = null,
|
||||
metaDateEnd = null,
|
||||
metaDateRaw = null,
|
||||
location,
|
||||
status,
|
||||
sender,
|
||||
@@ -59,7 +67,6 @@ function formatGeschichteDate(g: GeschichteSummary): string {
|
||||
return formatDate(g.publishedAt.slice(0, 10), 'short');
|
||||
}
|
||||
|
||||
const formattedDate = $derived(documentDate ? formatDate(documentDate) : '—');
|
||||
const displayLocation = $derived(location ?? '—');
|
||||
const statusLabel = $derived(formatDocumentStatus(status));
|
||||
const visibleReceivers = $derived(receivers.slice(0, VISIBLE_RECEIVER_LIMIT));
|
||||
@@ -105,7 +112,18 @@ function getFullName(person: Person): string {
|
||||
<dl class="space-y-3 font-serif text-sm">
|
||||
<div>
|
||||
<dt class="font-sans text-xs font-medium text-ink-3">{m.doc_details_field_date()}</dt>
|
||||
<dd class="text-ink">{formattedDate}</dd>
|
||||
<dd class="text-ink">
|
||||
{#if documentDate || metaDateRaw}
|
||||
<DocumentDate
|
||||
iso={documentDate}
|
||||
precision={metaDatePrecision}
|
||||
end={metaDateEnd}
|
||||
raw={metaDateRaw}
|
||||
/>
|
||||
{:else}
|
||||
—
|
||||
{/if}
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt class="font-sans text-xs font-medium text-ink-3">{m.form_label_location()}</dt>
|
||||
|
||||
@@ -2,13 +2,23 @@
|
||||
import type { components } from '$lib/generated/api';
|
||||
import { m } from '$lib/paraglide/messages.js';
|
||||
import { clickOutside } from '$lib/shared/actions/clickOutside';
|
||||
import { formatDate } from '$lib/shared/utils/date';
|
||||
import { formatDocumentDate, type DatePrecision } from '$lib/shared/utils/documentDate';
|
||||
import { getLocale } from '$lib/paraglide/runtime.js';
|
||||
|
||||
type Document = components['schemas']['Document'];
|
||||
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||
|
||||
/**
|
||||
* Exactly the fields this picker reads — id for selection/dedup, the rest for
|
||||
* the honest date label. A full `Document` and a `DocumentListItem` are both
|
||||
* structurally assignable, so the search results need no cast.
|
||||
*/
|
||||
type DocumentOption = Pick<
|
||||
DocumentListItem,
|
||||
'id' | 'title' | 'documentDate' | 'metaDatePrecision' | 'metaDateEnd'
|
||||
>;
|
||||
|
||||
interface Props {
|
||||
selectedDocuments?: Document[];
|
||||
selectedDocuments?: DocumentOption[];
|
||||
placeholder?: string;
|
||||
hiddenInputName?: string;
|
||||
}
|
||||
@@ -20,7 +30,7 @@ let {
|
||||
}: Props = $props();
|
||||
|
||||
let searchTerm = $state('');
|
||||
let results: Document[] = $state([]);
|
||||
let results: DocumentOption[] = $state([]);
|
||||
let showDropdown = $state(false);
|
||||
let loading = $state(false);
|
||||
let debounceTimer: ReturnType<typeof setTimeout>;
|
||||
@@ -46,11 +56,13 @@ function handleInput() {
|
||||
const res = await fetch(`/api/documents/search?q=${encodeURIComponent(searchTerm)}&size=10`);
|
||||
if (res.ok) {
|
||||
const body: { items: DocumentListItem[] } = await res.json();
|
||||
const docs = body.items.map((it) => ({
|
||||
const docs: DocumentOption[] = body.items.map((it) => ({
|
||||
id: it.id,
|
||||
title: it.title,
|
||||
documentDate: it.documentDate
|
||||
})) as unknown as Document[];
|
||||
documentDate: it.documentDate,
|
||||
metaDatePrecision: it.metaDatePrecision,
|
||||
metaDateEnd: it.metaDateEnd
|
||||
}));
|
||||
results = docs.filter((d) => !selectedDocuments.some((s) => s.id === d.id));
|
||||
}
|
||||
} catch {
|
||||
@@ -61,7 +73,7 @@ function handleInput() {
|
||||
}, 300);
|
||||
}
|
||||
|
||||
function selectDocument(doc: Document) {
|
||||
function selectDocument(doc: DocumentOption) {
|
||||
selectedDocuments = [...selectedDocuments, doc];
|
||||
searchTerm = '';
|
||||
showDropdown = false;
|
||||
@@ -72,9 +84,16 @@ function removeDocument(id: string | undefined) {
|
||||
selectedDocuments = selectedDocuments.filter((d) => d.id !== id);
|
||||
}
|
||||
|
||||
function formatDocLabel(doc: Document): string {
|
||||
if (doc.documentDate) return `${doc.title} · ${formatDate(doc.documentDate, 'short')}`;
|
||||
return doc.title;
|
||||
function formatDocLabel(doc: DocumentOption): string {
|
||||
if (!doc.documentDate) return doc.title;
|
||||
const label = formatDocumentDate(
|
||||
doc.documentDate,
|
||||
doc.metaDatePrecision as DatePrecision,
|
||||
doc.metaDateEnd,
|
||||
null,
|
||||
getLocale()
|
||||
);
|
||||
return `${doc.title} · ${label}`;
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ const docFactory = (id: string, title: string, date = '1880-01-01') => ({
|
||||
id,
|
||||
title,
|
||||
documentDate: date,
|
||||
metaDatePrecision: 'DAY' as const,
|
||||
originalFilename: `${title}.pdf`,
|
||||
receivers: [],
|
||||
tags: [],
|
||||
@@ -55,7 +56,8 @@ describe('DocumentMultiSelect — rendering', () => {
|
||||
selectedDocuments: [docFactory('d1', 'Brief vom 1. Mai', '1882-05-01')]
|
||||
});
|
||||
await expect.element(page.getByText(/Brief vom 1\. Mai/)).toBeInTheDocument();
|
||||
await expect.element(page.getByText(/01\.05\.1882/)).toBeInTheDocument();
|
||||
// DAY precision renders the honest long date (formatDocumentDate), not 01.05.1882.
|
||||
await expect.element(page.getByText(/1\. Mai 1882/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('emits a hidden documentIds input for each pre-selected document', async () => {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import { goto } from '$app/navigation';
|
||||
import type { components } from '$lib/generated/api';
|
||||
import { applyOffsets } from '$lib/document/search';
|
||||
import { formatDate } from '$lib/shared/utils/date';
|
||||
import DocumentDate from './DocumentDate.svelte';
|
||||
import * as m from '$lib/paraglide/messages.js';
|
||||
import { bulkSelectionStore } from '$lib/document/bulkSelection.svelte';
|
||||
import ProgressRing from '$lib/shared/primitives/ProgressRing.svelte';
|
||||
@@ -164,7 +164,16 @@ function safeTagColor(color: string | null | undefined): string {
|
||||
<!-- Mobile-only metadata -->
|
||||
<div class="mt-3 grid grid-cols-2 gap-x-4 gap-y-1 font-sans text-xs text-ink-2 sm:hidden">
|
||||
<div>
|
||||
{doc.documentDate ? formatDate(doc.documentDate) : '—'}
|
||||
<!-- Product decision (#666): raw provenance (meta_date_raw) is shown on the
|
||||
document DETAIL page, never in list/search rows — list rows surface only the
|
||||
honest label to keep scan-rows compact. showRaw={false} enforces this; the
|
||||
DocumentListItem payload also intentionally omits metaDateRaw. -->
|
||||
<DocumentDate
|
||||
iso={doc.documentDate}
|
||||
precision={doc.metaDatePrecision}
|
||||
end={doc.metaDateEnd}
|
||||
showRaw={false}
|
||||
/>
|
||||
</div>
|
||||
<div class="flex items-start gap-2">
|
||||
<ProgressRing percentage={item.completionPercentage} />
|
||||
@@ -178,7 +187,15 @@ function safeTagColor(color: string | null | undefined): string {
|
||||
<!-- Right column — desktop only -->
|
||||
<div class="hidden flex-col gap-2 pl-4 font-sans text-sm text-ink-2 sm:flex sm:w-44 lg:w-56">
|
||||
<div>
|
||||
{doc.documentDate ? formatDate(doc.documentDate) : '—'}
|
||||
<!-- An undated document is an absence, not an error (#668): DocumentDate
|
||||
defensively maps a null date to the "Datum unbekannt" badge, so we
|
||||
always render it — never a bare em-dash fallback. -->
|
||||
<DocumentDate
|
||||
iso={doc.documentDate}
|
||||
precision={doc.metaDatePrecision}
|
||||
end={doc.metaDateEnd}
|
||||
showRaw={false}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<span class="font-bold tracking-wide text-ink-3 uppercase">{m.docs_list_from()}</span>
|
||||
|
||||
@@ -22,6 +22,7 @@ function makeItem(overrides: Partial<DocumentListItem> = {}): DocumentListItem {
|
||||
title: 'Testbrief',
|
||||
originalFilename: 'testbrief.pdf',
|
||||
documentDate: '2024-03-15',
|
||||
metaDatePrecision: 'DAY',
|
||||
sender: undefined,
|
||||
receivers: [],
|
||||
tags: [],
|
||||
@@ -72,6 +73,35 @@ describe('DocumentRow – title', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Date rendering (#668) ──────────────────────────────────────────────────
|
||||
|
||||
describe('DocumentRow – date rendering', () => {
|
||||
it('renders a "Datum unbekannt" badge for an undated document', async () => {
|
||||
const item = makeItem({ documentDate: undefined, metaDatePrecision: 'UNKNOWN' });
|
||||
render(DocumentRow, { item });
|
||||
// The badge text appears (once per breakpoint block).
|
||||
await expect.element(page.getByText('Datum unbekannt').first()).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('does not render a bare em-dash for an undated document', async () => {
|
||||
const item = makeItem({ documentDate: undefined, metaDatePrecision: 'UNKNOWN' });
|
||||
render(DocumentRow, { item });
|
||||
await expect.element(page.getByText('—', { exact: true }).first()).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders the full date for a day-precision document', async () => {
|
||||
const item = makeItem({ documentDate: '1943-12-24', metaDatePrecision: 'DAY' });
|
||||
render(DocumentRow, { item });
|
||||
await expect.element(page.getByText(/24\. Dezember 1943/).first()).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders month precision honestly without fabricating a day', async () => {
|
||||
const item = makeItem({ documentDate: '1916-06-01', metaDatePrecision: 'MONTH' });
|
||||
render(DocumentRow, { item });
|
||||
await expect.element(page.getByText(/Juni 1916/).first()).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Snippet ──────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('DocumentRow – snippet', () => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user