feat(normalizer): generate structured tags from Schlagwort + Inhalt fields
Adds tags.py module implementing a three-outcome heuristic:
- Individual-to-individual correspondence tags ("Clara an Herbert") → dropped
- Group/collective correspondence ("Clara an Kinder", "Walter an Geschwister") → Briefwechsel/<value>
- Semantic/event tags ("Brautbriefe", "Alltag", "zur Hochzeit") → Themen/<value>
Three correspondence patterns detected: space-an-space, starts-with-"an ",
and abbreviated-sender form ("Maria W.an Clara").
COLLECTIVE_TERMS in config.py extended with 17 plural/group relational terms
(söhne, brüder, schwiegereltern, cousinen, etc.) confirmed against the full Excel.
Also adds two-phase summary mining: every run emits review/tag-candidates.csv;
subsequent runs apply keywords from overrides/approved-themes.csv as Themen tags.
Outputs: canonical-documents.xlsx gets pipe-separated "Parent/Child" tag paths;
canonical-tag-tree.xlsx provides the full tag hierarchy for backend pre-import.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
|
||||
import dates as _dates
|
||||
import tags as _tags
|
||||
|
||||
|
||||
class Triage(Enum):
|
||||
@@ -88,7 +89,7 @@ def index_file_mismatch(index: str, file_path: str) -> bool:
|
||||
return stem != index
|
||||
|
||||
|
||||
def to_canonical(raw, ctx, date_overrides: dict) -> CanonicalDocument:
|
||||
def to_canonical(raw, ctx, date_overrides: dict, approved_themes: frozenset = frozenset()) -> CanonicalDocument:
|
||||
pd = _dates.parse_date(raw.date, date_overrides)
|
||||
flags = []
|
||||
|
||||
@@ -113,6 +114,6 @@ def to_canonical(raw, ctx, date_overrides: dict) -> CanonicalDocument:
|
||||
receiver_person_ids=[r[0] for r in receivers],
|
||||
receiver_names=[r[1] for r in receivers],
|
||||
date_iso=pd.iso or "", date_raw=raw.date, date_precision=str(pd.precision),
|
||||
location=raw.location, tags=[raw.tags] if raw.tags else [], summary=raw.summary,
|
||||
location=raw.location, tags=_tags.generate_tags(raw.tags, raw.summary, approved_themes), summary=raw.summary,
|
||||
source_row=raw.source_row, needs_review=flags,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user