feat(normalizer): scaffold tool + config tables

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Marcel
2026-05-25 13:18:52 +02:00
parent 6f7aa643c9
commit 8f6f4f2d62
6 changed files with 124 additions and 0 deletions

4
.gitignore vendored
View File

@@ -26,3 +26,7 @@ node_modules/
# Repo uses npm; yarn.lock is ignored to avoid double-lockfile drift.
frontend/yarn.lock
**/.venv/
**/__pycache__/
*.pyc

5
tools/import-normalizer/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
.venv/
out/
review/
__pycache__/
*.pyc

View File

@@ -0,0 +1,100 @@
"""Tunables for the import normalizer. No logic here — only data tables."""
from pathlib import Path
# --- Paths ---
BASE_DIR = Path(__file__).resolve().parent
REPO_ROOT = BASE_DIR.parent.parent
IMPORT_DIR = REPO_ROOT / "import"
DOCUMENT_WORKBOOK = IMPORT_DIR / "zzfamilienarchiv aktuell 2 - Kopie 2025-07-05.xlsx"
DOCUMENT_SHEET = "Familienarchiv"
PERSON_WORKBOOK = IMPORT_DIR / "Personendatei 2.xlsx"
PERSON_SHEET = "Tabelle1"
OUT_DIR = BASE_DIR / "out"
REVIEW_DIR = BASE_DIR / "review"
OVERRIDES_DIR = BASE_DIR / "overrides"
# --- Header text (lowercased, whitespace-collapsed) -> canonical field ---
DOCUMENT_HEADER_MAP = {
"index": "index",
"datei": "file",
"box": "box",
"mappe": "folder",
"briefeschreiberin": "sender",
"empfängerin": "receivers",
"datum des briefes": "date",
"ort": "location",
"schlagwort": "tags",
"inhalt": "summary",
}
DOCUMENT_REQUIRED_FIELDS = {"index"}
PERSON_HEADER_MAP = {
"generation": "generation",
"familienname": "last_name",
"vorname": "first_name",
"geb als": "maiden_name",
"geburtsdatum": "birth_date",
"geburtsort": "birth_place",
"todesdatum": "death_date",
"sterbeort": "death_place",
"verheiratet mit": "spouse",
"bemerkung": "notes",
}
PERSON_REQUIRED_FIELDS = {"last_name"}
# --- Century rule (archive 18731957) ---
TWO_DIGIT_19XX_MAX = 57 # 00..57 -> 1900+yy
TWO_DIGIT_18XX_MIN = 73 # 73..99 -> 1800+yy ; 58..72 -> ambiguous -> UNKNOWN
# --- Seasons -> representative month (day = 1) ---
SEASON_MONTHS = {
"frühling": 4, "fruehling": 4, "frühjahr": 4, "fruehjahr": 4,
"sommer": 7, "herbst": 10, "winter": 1,
}
# --- Fixed feasts -> (month, day) ---
FIXED_FEASTS = {
"neujahr": (1, 1),
"heiligabend": (12, 24), "heiliger abend": (12, 24),
"weihnachten": (12, 25), "weihnacht": (12, 25), "1. weihnachtstag": (12, 25),
"silvester": (12, 31), "sylvester": (12, 31),
}
# --- Movable feasts -> day offset from Easter Sunday ---
MOVABLE_FEASTS = {
"karfreitag": -2,
"ostern": 0, "ostersonntag": 0, "ostermontag": 1,
"himmelfahrt": 39, "christi himmelfahrt": 39,
"pfingsten": 49, "pfingstsonntag": 49, "pfingstmontag": 50,
"fronleichnam": 60,
}
# --- Month names -> number (German + English, full + abbreviations) ---
MONTHS = {
"januar": 1, "jan": 1, "january": 1,
"februar": 2, "feb": 2, "febr": 2, "february": 2,
"märz": 3, "maerz": 3, "mär": 3, "mar": 3, "march": 3,
"april": 4, "apr": 4,
"mai": 5, "may": 5,
"juni": 6, "jun": 6, "june": 6,
"juli": 7, "jul": 7, "july": 7,
"august": 8, "aug": 8,
"september": 9, "sep": 9, "sept": 9,
"oktober": 10, "okt": 10, "oct": 10, "october": 10,
"november": 11, "nov": 11,
"dezember": 12, "dez": 12, "dec": 12, "december": 12,
}
ROMAN_MONTHS = {
"i": 1, "ii": 2, "iii": 3, "iv": 4, "v": 5, "vi": 6,
"vii": 7, "viii": 8, "ix": 9, "x": 10, "xi": 11, "xii": 12,
}
# --- Person matching ---
KNOWN_LAST_NAMES = [
"von der Heide", "von Massenbach", "von Geldern", "von Gelden", "von Staa",
"de Gruyter", "Dieckmann", "Gruber", "Müller", "Wolff", "Cram",
]
FUZZY_SUGGEST_THRESHOLD = 0.82 # difflib ratio; suggestions only, never auto-applied

View File

@@ -0,0 +1,2 @@
openpyxl==3.1.5
pytest==8.3.4

View File

@@ -0,0 +1,13 @@
import config
def test_century_boundaries():
assert config.TWO_DIGIT_19XX_MAX == 57
assert config.TWO_DIGIT_18XX_MIN == 73
def test_header_maps_cover_required_fields():
assert "index" in config.DOCUMENT_HEADER_MAP.values()
assert "last_name" in config.PERSON_HEADER_MAP.values()
def test_feast_tables_present():
assert config.MOVABLE_FEASTS["pfingsten"] == 49
assert config.SEASON_MONTHS["herbst"] == 10