From 482a1c28634cc716cdca01b0cab7f2975386c089 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 10:17:07 +0200 Subject: [PATCH 01/51] feat(nlp-service): spaCy model loading with get_nlp/load_all_models --- nlp-service/extractor.py | 33 ++++++++++++++++++++++++++++ nlp-service/test_extractor.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 nlp-service/extractor.py diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py new file mode 100644 index 00000000..ce752bef --- /dev/null +++ b/nlp-service/extractor.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import re +from datetime import date + +import dateparser +import spacy +from spacy.language import Language + +from models import ParseResponse + +# ── Language model registry ────────────────────────────────────────────────── + +_MODEL_NAMES: dict[str, str] = { + "de": "de_core_news_sm", + "en": "en_core_web_sm", + "es": "es_core_news_sm", +} + +_nlp_cache: dict[str, Language] = {} + + +def get_nlp(lang: str) -> Language: + if lang not in _MODEL_NAMES: + raise ValueError(f"Unsupported language: {lang!r}. Valid: {list(_MODEL_NAMES)}") + if lang not in _nlp_cache: + _nlp_cache[lang] = spacy.load(_MODEL_NAMES[lang]) + return _nlp_cache[lang] + + +def load_all_models() -> None: + for lang in _MODEL_NAMES: + get_nlp(lang) diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py index 0b80d0b4..2ef7c0a6 100644 --- a/nlp-service/test_extractor.py +++ b/nlp-service/test_extractor.py @@ -31,3 +31,44 @@ def test_parse_response_serializes_nulls(): assert data["dateFrom"] is None assert data["dateTo"] == "1920-12-31" assert data["personRole"] == "sender" + + +# ── Model loading ──────────────────────────────────────────────────────────── + +@pytest.fixture(scope="session") +def nlp_de(): + from extractor import get_nlp + return get_nlp("de") + + +@pytest.fixture(scope="session") +def nlp_en(): + from extractor import get_nlp + return get_nlp("en") + + +@pytest.fixture(scope="session") +def nlp_es(): + from extractor import get_nlp + return get_nlp("es") + + +def test_get_nlp_de_loads(nlp_de): + doc = nlp_de("Test") + assert doc is not None + + +def test_get_nlp_en_loads(nlp_en): + doc = nlp_en("Test") + assert doc is not None + + +def test_get_nlp_es_loads(nlp_es): + doc = nlp_es("Prueba") + assert doc is not None + + +def test_get_nlp_unknown_lang_raises(): + from extractor import get_nlp + with pytest.raises(ValueError, match="Unsupported language"): + get_nlp("fr") -- 2.49.1 From deea34c797ae62efc0f13f05aa6adb9a09417067 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 10:21:16 +0200 Subject: [PATCH 02/51] feat(nlp-service): NER person name extraction --- nlp-service/extractor.py | 7 ++++++ nlp-service/test_extractor.py | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index ce752bef..ca9c197e 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -31,3 +31,10 @@ def get_nlp(lang: str) -> Language: def load_all_models() -> None: for lang in _MODEL_NAMES: get_nlp(lang) + + +# ── Step 1: Person name extraction ────────────────────────────────────────── + +def extract_person_names(doc) -> list[str]: + """Return PER entity texts in left-to-right span order.""" + return [ent.text for ent in doc.ents if ent.label_ == "PER"] diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py index 2ef7c0a6..60890936 100644 --- a/nlp-service/test_extractor.py +++ b/nlp-service/test_extractor.py @@ -72,3 +72,48 @@ def test_get_nlp_unknown_lang_raises(): from extractor import get_nlp with pytest.raises(ValueError, match="Unsupported language"): get_nlp("fr") + + +# ── Person name extraction ─────────────────────────────────────────────────── + +def _make_doc_with_ents(nlp, text: str, char_ents: list[tuple[int, int, str]]): + """Create a Doc with manually injected entity spans (no NER model needed).""" + doc = nlp.make_doc(text) + spans = [doc.char_span(s, e, label=lbl) for s, e, lbl in char_ents] + doc.ents = [sp for sp in spans if sp is not None] + return doc + + +def test_extract_person_names_two_persons(nlp_de): + from extractor import extract_person_names + # "Briefe von Opa Hermann an Marie" + # "Opa Hermann" = chars 11..22, "Marie" = chars 26..31 + doc = _make_doc_with_ents(nlp_de, "Briefe von Opa Hermann an Marie", [ + (11, 22, "PER"), + (26, 31, "PER"), + ]) + assert extract_person_names(doc) == ["Opa Hermann", "Marie"] + + +def test_extract_person_names_preserves_order(nlp_de): + from extractor import extract_person_names + # "Marie von Opa" — Marie comes first in text + # "Marie" = 0..5, "Opa" = 10..13 + doc = _make_doc_with_ents(nlp_de, "Marie von Opa", [ + (0, 5, "PER"), + (10, 13, "PER"), + ]) + assert extract_person_names(doc) == ["Marie", "Opa"] + + +def test_extract_person_names_empty(nlp_de): + from extractor import extract_person_names + doc = _make_doc_with_ents(nlp_de, "Briefe aus dem Krieg", []) + assert extract_person_names(doc) == [] + + +def test_extract_person_names_ignores_non_per(nlp_de): + from extractor import extract_person_names + # DATE entity should not appear in personNames + doc = _make_doc_with_ents(nlp_de, "Briefe 1920", [(7, 11, "DATE")]) + assert extract_person_names(doc) == [] -- 2.49.1 From 8ed2a6d95b573c8b6a9fb5f14b997b5d99c0ad13 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 10:22:14 +0200 Subject: [PATCH 03/51] feat(nlp-service): role detection (sender/receiver/any) --- nlp-service/extractor.py | 48 ++++++++++++++++++++++++++ nlp-service/test_extractor.py | 65 +++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index ca9c197e..7ca84f82 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -38,3 +38,51 @@ def load_all_models() -> None: def extract_person_names(doc) -> list[str]: """Return PER entity texts in left-to-right span order.""" return [ent.text for ent in doc.ents if ent.label_ == "PER"] + + +# ── Step 2: Role detection ─────────────────────────────────────────────────── + +_SENDER_PREPS: dict[str, frozenset[str]] = { + "de": frozenset({"von", "vom"}), + "en": frozenset({"from", "by"}), + "es": frozenset({"de", "por"}), +} + +_RECEIVER_PREPS: dict[str, frozenset[str]] = { + "de": frozenset({"an", "nach", "für"}), + "en": frozenset({"to", "for"}), + "es": frozenset({"para", "a"}), +} + + +def detect_person_role(doc, per_spans: list, lang: str) -> str: + """Return 'sender', 'receiver', or 'any'. + + Only meaningful for single-PER queries — two-person queries always return + 'any' because Java derives direction from list position. + """ + if len(per_spans) != 1: + return "any" + + span = per_spans[0] + root = span.root + sender = _SENDER_PREPS[lang] + receiver = _RECEIVER_PREPS[lang] + + # Primary: dependency-tree children of the PER root + for child in root.children: + if child.dep_ in ("case", "prep", "mo"): + if child.lower_ in sender: + return "sender" + if child.lower_ in receiver: + return "receiver" + + # Fallback: token immediately before the span start + if span.start > 0: + prev = doc[span.start - 1] + if prev.lower_ in sender: + return "sender" + if prev.lower_ in receiver: + return "receiver" + + return "any" diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py index 60890936..fe14d4b1 100644 --- a/nlp-service/test_extractor.py +++ b/nlp-service/test_extractor.py @@ -117,3 +117,68 @@ def test_extract_person_names_ignores_non_per(nlp_de): # DATE entity should not appear in personNames doc = _make_doc_with_ents(nlp_de, "Briefe 1920", [(7, 11, "DATE")]) assert extract_person_names(doc) == [] + + +# ── Role detection ─────────────────────────────────────────────────────────── + +def test_role_sender_von(nlp_de): + from extractor import detect_person_role + # "Briefe von Marie" — "von" immediately before "Marie" + # "Marie" = chars 11..16 + doc = _make_doc_with_ents(nlp_de, "Briefe von Marie", [(11, 16, "PER")]) + per_spans = list(doc.ents) + assert detect_person_role(doc, per_spans, "de") == "sender" + + +def test_role_receiver_an(nlp_de): + from extractor import detect_person_role + # "Briefe an Marie" — "an" immediately before "Marie" + # "Marie" = chars 10..15 + doc = _make_doc_with_ents(nlp_de, "Briefe an Marie", [(10, 15, "PER")]) + per_spans = list(doc.ents) + assert detect_person_role(doc, per_spans, "de") == "receiver" + + +def test_role_two_persons_returns_any(nlp_de): + from extractor import detect_person_role + # "von Opa an Marie" — two PER spans → always "any" + # "Opa" = chars 4..7, "Marie" = chars 11..16 + doc = _make_doc_with_ents(nlp_de, "von Opa an Marie", [ + (4, 7, "PER"), + (11, 16, "PER"), + ]) + per_spans = list(doc.ents) + assert detect_person_role(doc, per_spans, "de") == "any" + + +def test_role_no_prep_returns_any(nlp_de): + from extractor import detect_person_role + # "Briefe Marie" — no preposition + # "Marie" = chars 7..12 + doc = _make_doc_with_ents(nlp_de, "Briefe Marie", [(7, 12, "PER")]) + per_spans = list(doc.ents) + assert detect_person_role(doc, per_spans, "de") == "any" + + +def test_role_empty_returns_any(nlp_de): + from extractor import detect_person_role + doc = _make_doc_with_ents(nlp_de, "Briefe 1920", []) + assert detect_person_role(doc, [], "de") == "any" + + +def test_role_sender_from_english(nlp_en): + from extractor import detect_person_role + # "letters from Marie" — "from" before "Marie" + # "Marie" = chars 13..18 + doc = _make_doc_with_ents(nlp_en, "letters from Marie", [(13, 18, "PER")]) + per_spans = list(doc.ents) + assert detect_person_role(doc, per_spans, "en") == "sender" + + +def test_role_receiver_to_english(nlp_en): + from extractor import detect_person_role + # "letters to Marie" — "to" before "Marie" + # "letters" = 0..7, " " = 7, "to" = 8..10, " " = 10, "Marie" = 11..16 + doc = _make_doc_with_ents(nlp_en, "letters to Marie", [(11, 16, "PER")]) + per_spans = list(doc.ents) + assert detect_person_role(doc, per_spans, "en") == "receiver" -- 2.49.1 From 3f74deda8c4c54a20f57555e9d9d1df9025db66a Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 10:23:33 +0200 Subject: [PATCH 04/51] feat(nlp-service): date range extraction with direction detection --- nlp-service/extractor.py | 82 +++++++++++++++++++++++++++++++++++ nlp-service/test_extractor.py | 69 +++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index 7ca84f82..19543af5 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -86,3 +86,85 @@ def detect_person_role(doc, per_spans: list, lang: str) -> str: return "receiver" return "any" + + +# ── Step 3: Date parsing ───────────────────────────────────────────────────── + +_YEAR_RE = re.compile(r"^\d{4}$") + +_DATE_BEFORE: dict[str, frozenset[str]] = { + "de": frozenset({"vor"}), + "en": frozenset({"before"}), + "es": frozenset({"antes"}), +} + +_DATE_AFTER: dict[str, frozenset[str]] = { + "de": frozenset({"nach"}), + "en": frozenset({"after"}), + "es": frozenset({"después", "despues"}), +} + +_DATE_BETWEEN: dict[str, frozenset[str]] = { + "de": frozenset({"zwischen"}), + "en": frozenset({"between"}), + "es": frozenset({"entre"}), +} + + +def _parse_date_text(text: str, lang: str) -> date | None: + text = text.strip() + if _YEAR_RE.match(text): + year = int(text) + if 1000 < year < 3000: + return date(year, 1, 1) + parsed = dateparser.parse( + text, + languages=[lang], + settings={"PREFER_DAY_OF_MONTH": "first", "RETURN_AS_TIMEZONE_AWARE": False}, + ) + return parsed.date() if parsed else None + + +def _year_end(d: date) -> date: + """If d is Jan 1, return Dec 31 of the same year (year-only boundary).""" + if d.month == 1 and d.day == 1: + return date(d.year, 12, 31) + return d + + +def extract_dates(doc, lang: str) -> tuple[str | None, str | None]: + """Return (date_from, date_to) as ISO strings or None.""" + date_spans = [ent for ent in doc.ents if ent.label_ == "DATE"] + if not date_spans: + return None, None + + between_tokens = _DATE_BETWEEN[lang] + before_tokens = _DATE_BEFORE[lang] + after_tokens = _DATE_AFTER[lang] + + # "zwischen X und Y" / "between X and Y" — two DATE spans form a range + has_between = any(tok.lower_ in between_tokens for tok in doc) + if has_between and len(date_spans) >= 2: + parsed = [] + for span in date_spans[:2]: + d = _parse_date_text(span.text, lang) + if d: + parsed.append(d) + if len(parsed) == 2: + parsed.sort() + return parsed[0].isoformat(), _year_end(parsed[1]).isoformat() + + # Single DATE span — use direction token + span = date_spans[0] + d = _parse_date_text(span.text, lang) + if not d: + return None, None + + prev_lower = doc[span.start - 1].lower_ if span.start > 0 else "" + + if prev_lower in before_tokens: + return None, _year_end(d).isoformat() + if prev_lower in after_tokens: + return d.isoformat(), None + # Bare year/date — closed year-range + return d.isoformat(), _year_end(d).isoformat() diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py index fe14d4b1..b79d188c 100644 --- a/nlp-service/test_extractor.py +++ b/nlp-service/test_extractor.py @@ -182,3 +182,72 @@ def test_role_receiver_to_english(nlp_en): doc = _make_doc_with_ents(nlp_en, "letters to Marie", [(11, 16, "PER")]) per_spans = list(doc.ents) assert detect_person_role(doc, per_spans, "en") == "receiver" + + +# ── Date parsing ───────────────────────────────────────────────────────────── + +def test_date_vor_1920(nlp_de): + from extractor import extract_dates + # "Briefe vor 1920" — "1920" at chars 11..15 + doc = _make_doc_with_ents(nlp_de, "Briefe vor 1920", [(11, 15, "DATE")]) + date_from, date_to = extract_dates(doc, "de") + assert date_from is None + assert date_to == "1920-12-31" + + +def test_date_nach_1900(nlp_de): + from extractor import extract_dates + # "Briefe nach 1900" — "1900" at chars 12..16 + doc = _make_doc_with_ents(nlp_de, "Briefe nach 1900", [(12, 16, "DATE")]) + date_from, date_to = extract_dates(doc, "de") + assert date_from == "1900-01-01" + assert date_to is None + + +def test_date_zwischen_1900_und_1920(nlp_de): + from extractor import extract_dates + # "zwischen 1900 und 1920" + # "1900" = chars 9..13, "1920" = chars 18..22 + doc = _make_doc_with_ents(nlp_de, "zwischen 1900 und 1920", [ + (9, 13, "DATE"), + (18, 22, "DATE"), + ]) + date_from, date_to = extract_dates(doc, "de") + assert date_from == "1900-01-01" + assert date_to == "1920-12-31" + + +def test_date_bare_year_makes_range(nlp_de): + from extractor import extract_dates + # "Briefe 1920" — no direction token → year-range + # "1920" = chars 7..11 + doc = _make_doc_with_ents(nlp_de, "Briefe 1920", [(7, 11, "DATE")]) + date_from, date_to = extract_dates(doc, "de") + assert date_from == "1920-01-01" + assert date_to == "1920-12-31" + + +def test_date_no_date_entity(nlp_de): + from extractor import extract_dates + doc = _make_doc_with_ents(nlp_de, "Briefe von Opa", []) + date_from, date_to = extract_dates(doc, "de") + assert date_from is None + assert date_to is None + + +def test_date_before_english(nlp_en): + from extractor import extract_dates + # "letters before 1920" — "1920" at chars 15..19 + doc = _make_doc_with_ents(nlp_en, "letters before 1920", [(15, 19, "DATE")]) + date_from, date_to = extract_dates(doc, "en") + assert date_from is None + assert date_to == "1920-12-31" + + +def test_date_after_english(nlp_en): + from extractor import extract_dates + # "letters after 1900" — "1900" at chars 14..18 + doc = _make_doc_with_ents(nlp_en, "letters after 1900", [(14, 18, "DATE")]) + date_from, date_to = extract_dates(doc, "en") + assert date_from == "1900-01-01" + assert date_to is None -- 2.49.1 From 702a72d575591f0250fa1ab36ce2986f995186af Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 10:24:35 +0200 Subject: [PATCH 05/51] feat(nlp-service): keyword extraction (POS-filtered, deduped lemmas) --- nlp-service/extractor.py | 27 ++++++++++++++++++++ nlp-service/test_extractor.py | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index 19543af5..5d9b230b 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -168,3 +168,30 @@ def extract_dates(doc, lang: str) -> tuple[str | None, str | None]: return d.isoformat(), None # Bare year/date — closed year-range return d.isoformat(), _year_end(d).isoformat() + + +# ── Step 4: Keyword extraction ─────────────────────────────────────────────── + +def extract_keywords(doc, excluded_spans: list) -> list[str]: + """Return lowercased lemmas of content words not inside any NER span.""" + excluded_indices: set[int] = set() + for span in excluded_spans: + excluded_indices.update(range(span.start, span.end)) + + seen: set[str] = set() + keywords: list[str] = [] + for token in doc: + if token.i in excluded_indices: + continue + if token.pos_ not in ("NOUN", "PROPN"): + continue + if token.is_stop: + continue + lemma = token.lemma_.lower() + if len(lemma) < 3: + continue + if lemma not in seen: + seen.add(lemma) + keywords.append(lemma) + + return keywords diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py index b79d188c..8d623178 100644 --- a/nlp-service/test_extractor.py +++ b/nlp-service/test_extractor.py @@ -251,3 +251,50 @@ def test_date_after_english(nlp_en): date_from, date_to = extract_dates(doc, "en") assert date_from == "1900-01-01" assert date_to is None + + +# ── Keyword extraction ─────────────────────────────────────────────────────── + +def test_keywords_extracts_nouns(nlp_de): + from extractor import extract_keywords + # Use real NLP for POS tags; disable NER to avoid interference + doc = nlp_de("Briefe aus dem Krieg", disable=["ner"]) + keywords = extract_keywords(doc, []) + # "Brief" (NOUN) and "Krieg" (NOUN) should appear as lemmas + assert "brief" in keywords + assert "krieg" in keywords + + +def test_keywords_excludes_stopwords(nlp_de): + from extractor import extract_keywords + doc = nlp_de("Briefe aus dem Krieg", disable=["ner"]) + keywords = extract_keywords(doc, []) + # "dem" is a stopword article — must not appear + assert "dem" not in keywords + + +def test_keywords_excludes_per_ner_spans(nlp_de): + from extractor import extract_keywords + # Run full NLP for POS tags, then inject a PER span over "Hermann" + # "Briefe von Hermann": B=0..6, ' '=6, v=7..10, ' '=10, H=11..18 + doc = nlp_de("Briefe von Hermann") + per_span = doc.char_span(11, 18, label="PER") + if per_span: + doc.ents = [per_span] + keywords = extract_keywords(doc, list(doc.ents)) + assert "hermann" not in keywords + + +def test_keywords_excludes_short_lemmas(nlp_de): + from extractor import extract_keywords + doc = nlp_de("Briefe an ihn", disable=["ner"]) + keywords = extract_keywords(doc, []) + # "ihn" is 3 chars but is a stopword pronoun; "an" is 2 chars + assert "an" not in keywords + + +def test_keywords_deduplicates(nlp_de): + from extractor import extract_keywords + doc = nlp_de("Brief Brief Krieg", disable=["ner"]) + keywords = extract_keywords(doc, []) + assert keywords.count("brief") == 1 -- 2.49.1 From 3ddb2b278b40104ef21d14a1f641f4154a8cc833 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 10:28:40 +0200 Subject: [PATCH 06/51] =?UTF-8?q?feat(nlp-service):=20full=20extract()=20p?= =?UTF-8?q?ipeline=20=E2=80=94=20assembles=20all=20steps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also adds regex year-fallback in extract_dates() for de/es spaCy small models that don't tag bare 4-digit years as DATE entities, and widens the direction-token window to 2 tokens back to handle Spanish "antes de". Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/extractor.py | 54 +++++++++++++++++++++++++++++++++-- nlp-service/test_extractor.py | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index 5d9b230b..5d6bb629 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -132,9 +132,28 @@ def _year_end(d: date) -> date: return d +def _find_year_spans(doc) -> list: + """Fallback: find tokens that look like 4-digit years (1000–2999) when NER + produces no DATE entities. Returns a list of single-token pseudo-spans + (spaCy Span objects) labelled 'DATE'.""" + spans = [] + for token in doc: + if _YEAR_RE.match(token.text): + year = int(token.text) + if 1000 < year < 3000: + span = doc[token.i : token.i + 1] + spans.append(span) + return spans + + def extract_dates(doc, lang: str) -> tuple[str | None, str | None]: """Return (date_from, date_to) as ISO strings or None.""" date_spans = [ent for ent in doc.ents if ent.label_ == "DATE"] + + # Fallback: some spaCy small models (de, es) don't tag bare years as DATE + if not date_spans: + date_spans = _find_year_spans(doc) + if not date_spans: return None, None @@ -160,11 +179,16 @@ def extract_dates(doc, lang: str) -> tuple[str | None, str | None]: if not d: return None, None - prev_lower = doc[span.start - 1].lower_ if span.start > 0 else "" + # Check up to 2 tokens before the date span to handle multi-word prepositions + # like Spanish "antes de 1920" where the keyword is 2 tokens back. + prev_tokens = [ + doc[span.start - i].lower_ + for i in range(1, min(3, span.start + 1)) + ] - if prev_lower in before_tokens: + if any(t in before_tokens for t in prev_tokens): return None, _year_end(d).isoformat() - if prev_lower in after_tokens: + if any(t in after_tokens for t in prev_tokens): return d.isoformat(), None # Bare year/date — closed year-range return d.isoformat(), _year_end(d).isoformat() @@ -195,3 +219,27 @@ def extract_keywords(doc, excluded_spans: list) -> list[str]: keywords.append(lemma) return keywords + + +# ── Step 5: Assembly ───────────────────────────────────────────────────────── + +def extract(query: str, lang: str) -> ParseResponse: + """Run the full NLP pipeline and return a ParseResponse.""" + nlp = get_nlp(lang) + doc = nlp(query) + + per_spans = [ent for ent in doc.ents if ent.label_ == "PER"] + + person_names = extract_person_names(doc) + person_role = detect_person_role(doc, per_spans, lang) + date_from, date_to = extract_dates(doc, lang) + keywords = extract_keywords(doc, list(doc.ents)) + + return ParseResponse( + personNames=person_names, + personRole=person_role, + dateFrom=date_from, + dateTo=date_to, + keywords=keywords, + rawQuery=query, + ) diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py index 8d623178..fb117884 100644 --- a/nlp-service/test_extractor.py +++ b/nlp-service/test_extractor.py @@ -298,3 +298,53 @@ def test_keywords_deduplicates(nlp_de): doc = nlp_de("Brief Brief Krieg", disable=["ner"]) keywords = extract_keywords(doc, []) assert keywords.count("brief") == 1 + + +# ── Full extract() pipeline ────────────────────────────────────────────────── + +def test_extract_dates_de(): + from extractor import extract + result = extract("Briefe vor 1920", "de") + assert result.dateFrom is None + assert result.dateTo == "1920-12-31" + assert result.rawQuery == "Briefe vor 1920" + assert result.personNames == [] + assert result.personRole == "any" + + +def test_extract_keywords_from_topic_de(): + from extractor import extract + result = extract("Briefe aus dem Krieg", "de") + assert "krieg" in result.keywords + assert result.dateFrom is None + assert result.dateTo is None + + +def test_extract_dates_en(): + from extractor import extract + result = extract("letters before 1920", "en") + assert result.dateTo == "1920-12-31" + assert result.dateFrom is None + + +def test_extract_dates_es(): + from extractor import extract + result = extract("cartas antes de 1920", "es") + assert result.dateTo == "1920-12-31" + assert result.dateFrom is None + + +def test_extract_rawquery_echoed(): + from extractor import extract + q = "Texte über Weihnachten" + result = extract(q, "de") + assert result.rawQuery == q + + +def test_extract_response_fields_are_complete(): + from extractor import extract + result = extract("Briefe 1900", "de") + assert isinstance(result.personNames, list) + assert result.personRole in ("sender", "receiver", "any") + assert isinstance(result.keywords, list) + assert result.rawQuery == "Briefe 1900" -- 2.49.1 From 740428413027e5f637346352b211e6b8fbe937fe Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 10:29:32 +0200 Subject: [PATCH 07/51] feat(nlp-service): FastAPI app with /parse and /health endpoints Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/main.py | 34 ++++++++++++++++++++++++++ nlp-service/test_main.py | 52 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 nlp-service/main.py create mode 100644 nlp-service/test_main.py diff --git a/nlp-service/main.py b/nlp-service/main.py new file mode 100644 index 00000000..c440a1b0 --- /dev/null +++ b/nlp-service/main.py @@ -0,0 +1,34 @@ +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI, HTTPException + +from extractor import extract, load_all_models +from models import ParseRequest, ParseResponse + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("Loading spaCy models...") + load_all_models() + logger.info("All models ready.") + yield + + +app = FastAPI(lifespan=lifespan) + + +@app.get("/health") +def health() -> dict: + return {"status": "ok"} + + +@app.post("/parse", response_model=ParseResponse) +def parse(request: ParseRequest) -> ParseResponse: + try: + return extract(request.query, request.lang) + except Exception as exc: + logger.exception("Extraction pipeline failed") + raise HTTPException(status_code=500, detail=str(exc)) from exc diff --git a/nlp-service/test_main.py b/nlp-service/test_main.py new file mode 100644 index 00000000..d9382e2d --- /dev/null +++ b/nlp-service/test_main.py @@ -0,0 +1,52 @@ +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture(scope="session") +def client(): + from main import app + with TestClient(app) as c: + yield c + + +def test_health(client): + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_parse_returns_200_with_all_fields(client): + response = client.post("/parse", json={"query": "Briefe vor 1920", "lang": "de"}) + assert response.status_code == 200 + data = response.json() + assert "personNames" in data + assert "personRole" in data + assert data["personRole"] in ("sender", "receiver", "any") + assert "dateFrom" in data + assert "dateTo" in data + assert "keywords" in data + assert "rawQuery" in data + assert data["rawQuery"] == "Briefe vor 1920" + assert data["dateTo"] == "1920-12-31" + + +def test_parse_unknown_lang_returns_422(client): + response = client.post("/parse", json={"query": "test", "lang": "fr"}) + assert response.status_code == 422 + + +def test_parse_missing_query_returns_422(client): + response = client.post("/parse", json={"lang": "de"}) + assert response.status_code == 422 + + +def test_parse_all_languages(client): + cases = [ + ("de", "Briefe vor 1920"), + ("en", "letters before 1920"), + ("es", "cartas antes de 1920"), + ] + for lang, query in cases: + response = client.post("/parse", json={"query": query, "lang": lang}) + assert response.status_code == 200, f"Failed for lang={lang}" + assert response.json()["dateTo"] == "1920-12-31", f"Wrong dateTo for lang={lang}" -- 2.49.1 From aa200bf3c5062b8af895900c195badb5a3d6a747 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 10:31:18 +0200 Subject: [PATCH 08/51] =?UTF-8?q?feat(nlp-service):=20Dockerfile=20?= =?UTF-8?q?=E2=80=94=20python:3.11-slim,=20models=20baked=20in?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nlp-service/Dockerfile | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 nlp-service/Dockerfile diff --git a/nlp-service/Dockerfile b/nlp-service/Dockerfile new file mode 100644 index 00000000..b47a71be --- /dev/null +++ b/nlp-service/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Bake models into the image — no volume needed, ~350 MB total +RUN python -m spacy download de_core_news_sm \ + && python -m spacy download en_core_web_sm \ + && python -m spacy download es_core_news_sm + +COPY . . + +RUN useradd --no-create-home --shell /usr/sbin/nologin --uid 1001 nlp \ + && chown -R nlp:nlp /app + +USER nlp + +EXPOSE 8001 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ + CMD curl -f http://localhost:8001/health || exit 1 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001"] -- 2.49.1 From 03d7d44e57603f2e17c5dbfd7b0187842e148079 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 11:00:03 +0200 Subject: [PATCH 09/51] feat(nlp-service): replace spaCy NER with DB-backed PersonMatcher Rule-based pipeline: persons matched via rapidfuzz against all known names loaded from DB at startup. Fixes first-name-only extraction (Eugenie, Herbert), merged-span bug (Herbert + Eugenie de Gruyter), false positives on compound nouns, and EN/ES model failures. Date extraction unchanged (regex). No spaCy models required. Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/CLAUDE.md | 25 +- nlp-service/extractor.py | 381 ++++++++++--------- nlp-service/main.py | 35 +- nlp-service/person_matcher.py | 164 ++++++++ nlp-service/requirements.txt | 3 +- nlp-service/test_extractor.py | 683 +++++++++++++++++----------------- nlp-service/test_main.py | 73 ++-- nlp-service/test_sentences.md | 126 +++++++ 8 files changed, 939 insertions(+), 551 deletions(-) create mode 100644 nlp-service/person_matcher.py create mode 100644 nlp-service/test_sentences.md diff --git a/nlp-service/CLAUDE.md b/nlp-service/CLAUDE.md index 4b5300ea..f7579b7d 100644 --- a/nlp-service/CLAUDE.md +++ b/nlp-service/CLAUDE.md @@ -5,23 +5,30 @@ replacing Ollama for the Familienarchiv NL search feature. ## Stack -- Python 3.11, FastAPI 0.115, spaCy 3.8, dateparser 1.2 +- Python 3.11, FastAPI 0.115, rapidfuzz 3.x, dateparser 1.2, psycopg2-binary + +No ML models — persons are matched against the live DB via fuzzy lookup. ## Endpoints - `POST /parse` — parse a free-text query, return extraction matching `OllamaExtraction` contract -- `GET /health` — returns `{"status": "ok"}` when all models are loaded +- `GET /health` — returns `{"status": "ok", "persons_loaded": N}` ## Running locally ```bash pip install -r requirements.txt -python -m spacy download de_core_news_sm en_core_web_sm es_core_news_sm + +# Without DB (empty person matcher — dates and keywords still work): uvicorn main:app --reload --port 8001 +# With DB (full person matching): +DATABASE_URL=postgresql://archive_user:secret@localhost:5432/family_archive_db \ + uvicorn main:app --reload --port 8001 + curl -X POST http://localhost:8001/parse \ -H "Content-Type: application/json" \ - -d '{"query": "Briefe von Opa Hermann an Marie vor 1920", "lang": "de"}' + -d '{"query": "Briefe von Clara Cram an Walter de Gruyter vor 1920", "lang": "de"}' ``` ## Testing @@ -30,6 +37,14 @@ curl -X POST http://localhost:8001/parse \ pytest -v ``` +No DB required for tests — fixture pre-seeds the PersonMatcher with a small test corpus. + +## Architecture + +- `person_matcher.py` — DB-backed name lookup: loads all persons at startup, fuzzy-matches query tokens after person prepositions +- `extractor.py` — pipeline: persons → role → dates (regex) → keywords (stopword filter) +- `main.py` — FastAPI app; reads `DATABASE_URL` env var at startup + ## Design spec See `docs/superpowers/specs/2026-06-07-spacy-nlp-service-design.md`. @@ -39,3 +54,5 @@ See `docs/superpowers/specs/2026-06-07-spacy-nlp-service-design.md`. This is a **prototype** for extraction quality evaluation. No docker-compose integration or Java-side changes in this iteration. The extraction contract matches `OllamaExtraction` in `backend/src/main/java/org/raddatz/familienarchiv/search/`. + +Test sentences for manual evaluation are in `test_sentences.md`. diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index 5d6bb629..b23a58b0 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -1,46 +1,33 @@ +"""Rule-based NLP pipeline: dates via regex, persons via DB-backed matcher.""" from __future__ import annotations import re from datetime import date +from typing import TYPE_CHECKING import dateparser -import spacy -from spacy.language import Language from models import ParseResponse +from person_matcher import PersonMatcher -# ── Language model registry ────────────────────────────────────────────────── +if TYPE_CHECKING: + pass -_MODEL_NAMES: dict[str, str] = { - "de": "de_core_news_sm", - "en": "en_core_web_sm", - "es": "es_core_news_sm", -} +# ── Module-level PersonMatcher (set at startup) ─────────────────────────────── -_nlp_cache: dict[str, Language] = {} +_matcher: PersonMatcher | None = None -def get_nlp(lang: str) -> Language: - if lang not in _MODEL_NAMES: - raise ValueError(f"Unsupported language: {lang!r}. Valid: {list(_MODEL_NAMES)}") - if lang not in _nlp_cache: - _nlp_cache[lang] = spacy.load(_MODEL_NAMES[lang]) - return _nlp_cache[lang] +def set_person_matcher(m: PersonMatcher) -> None: + global _matcher + _matcher = m -def load_all_models() -> None: - for lang in _MODEL_NAMES: - get_nlp(lang) +def get_person_matcher() -> PersonMatcher | None: + return _matcher -# ── Step 1: Person name extraction ────────────────────────────────────────── - -def extract_person_names(doc) -> list[str]: - """Return PER entity texts in left-to-right span order.""" - return [ent.text for ent in doc.ents if ent.label_ == "PER"] - - -# ── Step 2: Role detection ─────────────────────────────────────────────────── +# ── Preposition sets ────────────────────────────────────────────────────────── _SENDER_PREPS: dict[str, frozenset[str]] = { "de": frozenset({"von", "vom"}), @@ -54,43 +41,12 @@ _RECEIVER_PREPS: dict[str, frozenset[str]] = { "es": frozenset({"para", "a"}), } +_ALL_PERSON_PREPS: dict[str, frozenset[str]] = { + lang: _SENDER_PREPS[lang] | _RECEIVER_PREPS[lang] + for lang in ("de", "en", "es") +} -def detect_person_role(doc, per_spans: list, lang: str) -> str: - """Return 'sender', 'receiver', or 'any'. - - Only meaningful for single-PER queries — two-person queries always return - 'any' because Java derives direction from list position. - """ - if len(per_spans) != 1: - return "any" - - span = per_spans[0] - root = span.root - sender = _SENDER_PREPS[lang] - receiver = _RECEIVER_PREPS[lang] - - # Primary: dependency-tree children of the PER root - for child in root.children: - if child.dep_ in ("case", "prep", "mo"): - if child.lower_ in sender: - return "sender" - if child.lower_ in receiver: - return "receiver" - - # Fallback: token immediately before the span start - if span.start > 0: - prev = doc[span.start - 1] - if prev.lower_ in sender: - return "sender" - if prev.lower_ in receiver: - return "receiver" - - return "any" - - -# ── Step 3: Date parsing ───────────────────────────────────────────────────── - -_YEAR_RE = re.compile(r"^\d{4}$") +# ── Date direction tokens ───────────────────────────────────────────────────── _DATE_BEFORE: dict[str, frozenset[str]] = { "de": frozenset({"vor"}), @@ -110,130 +66,219 @@ _DATE_BETWEEN: dict[str, frozenset[str]] = { "es": frozenset({"entre"}), } +# ── Stopword lists ──────────────────────────────────────────────────────────── -def _parse_date_text(text: str, lang: str) -> date | None: - text = text.strip() - if _YEAR_RE.match(text): - year = int(text) - if 1000 < year < 3000: - return date(year, 1, 1) - parsed = dateparser.parse( - text, - languages=[lang], - settings={"PREFER_DAY_OF_MONTH": "first", "RETURN_AS_TIMEZONE_AWARE": False}, - ) - return parsed.date() if parsed else None +_STOPWORDS: dict[str, frozenset[str]] = { + "de": frozenset({ + "der", "die", "das", "des", "dem", "den", + "ein", "eine", "einem", "einen", "einer", "eines", + "er", "sie", "es", "wir", "ihr", "ich", "du", + "und", "oder", "aber", "doch", "auch", "noch", "nur", + "in", "an", "auf", "aus", "bei", "mit", "nach", "von", "vom", + "vor", "zu", "zur", "zum", "durch", "für", "über", "unter", + "zwischen", "gegen", "ohne", "um", "bis", "seit", "wegen", + "ist", "sind", "war", "waren", "wird", "werden", + "hat", "haben", "hatte", "hatten", + "sein", "seine", "seinen", "seiner", "seines", + "ihre", "ihren", "ihrer", "ihrem", "ihres", + "nicht", "kein", "keine", "keinen", "keinem", "keines", + "so", "wie", "als", "da", "hier", "dort", "wo", "wer", "was", + "im", "am", "beim", "ins", "ans", + "ja", "nein", "denn", "wenn", "weil", "dass", "ob", "damit", + "alle", "alles", "mehr", "sehr", "viel", "wenig", + "diesem", "dieser", "dieses", "diese", "diesen", + "jetzt", "dann", "nun", "schon", "wohl", "wurde", "wurden", + "worden", "geschrieben", "seinen", "ihrer", + "beim", "nach", "zum", "zur", "dem", "den", + "seine", "ihrem", "Jahr", "Jahren", "jahre", "jahr", + }), + "en": frozenset({ + "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", + "for", "of", "with", "by", "from", "about", "as", "into", + "through", "is", "are", "was", "were", "be", "been", "being", + "have", "has", "had", "do", "does", "did", "will", "would", + "could", "should", "may", "might", "must", "shall", "can", + "i", "you", "he", "she", "it", "we", "they", "their", "our", + "his", "her", "its", "my", "your", + "this", "that", "these", "those", "all", "not", "no", "nor", + "very", "more", "most", "much", "many", "some", "any", + "before", "after", "between", "during", "since", "until", + "when", "where", "who", "which", "what", "how", + }), + "es": frozenset({ + "el", "la", "los", "las", "un", "una", "unos", "unas", + "y", "o", "pero", "sin", "con", "en", "de", "del", "al", + "a", "ante", "bajo", "desde", "entre", "hacia", "hasta", + "para", "por", "sobre", "tras", + "es", "son", "era", "eran", "fue", "fueron", "ser", "estar", + "ha", "han", "he", "tener", "tiene", + "yo", "su", "sus", "mi", "tu", + "este", "esta", "estos", "estas", "ese", "esa", + "no", "muy", "todo", "todos", "toda", + "que", "cuando", "donde", "como", + "antes", "después", "durante", "desde", "hasta", + }), +} + +# ── Year regex ──────────────────────────────────────────────────────────────── + +_YEAR_RE = re.compile(r"\b(\d{4})\b") +_WORD_RE = re.compile(r"\b[^\W\d_]{3,}\b", re.UNICODE) -def _year_end(d: date) -> date: - """If d is Jan 1, return Dec 31 of the same year (year-only boundary).""" - if d.month == 1 and d.day == 1: - return date(d.year, 12, 31) - return d +# ── Step 1 + 2: Person extraction and role detection ───────────────────────── + +def _extract_persons_and_role( + query: str, + lang: str, +) -> tuple[list[str], str]: + """Return (person_names, role) using the DB-backed PersonMatcher.""" + m = _matcher + if m is None or len(m) == 0: + return [], "any" + + preps = _ALL_PERSON_PREPS[lang] + stops = preps | _DATE_BEFORE[lang] | _DATE_AFTER[lang] | _DATE_BETWEEN[lang] + matches = m.find_in_query(query, preps, stop_tokens=stops) + + person_names = [text for text, _ in matches] + + if len(matches) != 1: + return person_names, "any" + + _, prep = matches[0] + if prep is None: + return person_names, "any" + if prep in _SENDER_PREPS[lang]: + return person_names, "sender" + if prep in _RECEIVER_PREPS[lang]: + return person_names, "receiver" + return person_names, "any" -def _find_year_spans(doc) -> list: - """Fallback: find tokens that look like 4-digit years (1000–2999) when NER - produces no DATE entities. Returns a list of single-token pseudo-spans - (spaCy Span objects) labelled 'DATE'.""" - spans = [] - for token in doc: - if _YEAR_RE.match(token.text): - year = int(token.text) - if 1000 < year < 3000: - span = doc[token.i : token.i + 1] - spans.append(span) - return spans +# ── Step 3: Date extraction ─────────────────────────────────────────────────── - -def extract_dates(doc, lang: str) -> tuple[str | None, str | None]: - """Return (date_from, date_to) as ISO strings or None.""" - date_spans = [ent for ent in doc.ents if ent.label_ == "DATE"] - - # Fallback: some spaCy small models (de, es) don't tag bare years as DATE - if not date_spans: - date_spans = _find_year_spans(doc) - - if not date_spans: - return None, None - - between_tokens = _DATE_BETWEEN[lang] - before_tokens = _DATE_BEFORE[lang] - after_tokens = _DATE_AFTER[lang] - - # "zwischen X und Y" / "between X and Y" — two DATE spans form a range - has_between = any(tok.lower_ in between_tokens for tok in doc) - if has_between and len(date_spans) >= 2: - parsed = [] - for span in date_spans[:2]: - d = _parse_date_text(span.text, lang) - if d: - parsed.append(d) - if len(parsed) == 2: - parsed.sort() - return parsed[0].isoformat(), _year_end(parsed[1]).isoformat() - - # Single DATE span — use direction token - span = date_spans[0] - d = _parse_date_text(span.text, lang) - if not d: - return None, None - - # Check up to 2 tokens before the date span to handle multi-word prepositions - # like Spanish "antes de 1920" where the keyword is 2 tokens back. - prev_tokens = [ - doc[span.start - i].lower_ - for i in range(1, min(3, span.start + 1)) +def _find_years(query: str) -> list[tuple[int, int, int]]: + """Return list of (start, end, year_int) for valid 4-digit year tokens.""" + return [ + (m.start(), m.end(), int(m.group())) + for m in _YEAR_RE.finditer(query) + if 1000 < int(m.group()) < 3000 ] - if any(t in before_tokens for t in prev_tokens): - return None, _year_end(d).isoformat() - if any(t in after_tokens for t in prev_tokens): - return d.isoformat(), None - # Bare year/date — closed year-range - return d.isoformat(), _year_end(d).isoformat() + +def _direction_before_year( + query: str, + year_start: int, + lang: str, + person_names: list[str], +) -> str: + """Classify direction of the date span as 'before', 'after', or 'bare'. + + Looks at the two tokens immediately preceding the year. If the closer + token is a matched person name part, the direction word belongs to that + person — not to the year — so we return 'bare'. + """ + prefix_words = query[:year_start].split() + if not prefix_words: + return "bare" + + person_tokens = {w.lower() for name in person_names for w in name.split()} + recent = [w.lower() for w in prefix_words[-2:]] + + before_set = _DATE_BEFORE[lang] + after_set = _DATE_AFTER[lang] + + for direction_tok in reversed(recent): # closest first + if direction_tok in before_set: + # Only use this if the word immediately before the year is not a person + if recent[-1] in person_tokens: + return "bare" + return "before" + if direction_tok in after_set: + if recent[-1] in person_tokens: + return "bare" + return "after" + + return "bare" -# ── Step 4: Keyword extraction ─────────────────────────────────────────────── +def extract_dates( + query: str, + lang: str, + person_names: list[str] | None = None, +) -> tuple[str | None, str | None]: + """Return (date_from, date_to) as ISO strings or None.""" + if person_names is None: + person_names = [] -def extract_keywords(doc, excluded_spans: list) -> list[str]: - """Return lowercased lemmas of content words not inside any NER span.""" - excluded_indices: set[int] = set() - for span in excluded_spans: - excluded_indices.update(range(span.start, span.end)) + year_spans = _find_years(query) + if not year_spans: + return None, None + # "zwischen X und Y" / "between X and Y" — two years form a range + query_lower = query.lower() + if any(w in query_lower.split() for w in _DATE_BETWEEN[lang]) and len(year_spans) >= 2: + years = sorted([y for _, _, y in year_spans[:2]]) + return date(years[0], 1, 1).isoformat(), date(years[1], 12, 31).isoformat() + + start, end, year = year_spans[0] + direction = _direction_before_year(query, start, lang, person_names) + + if direction == "before": + return None, date(year, 12, 31).isoformat() + if direction == "after": + return date(year, 1, 1).isoformat(), None + # bare year → closed year range + return date(year, 1, 1).isoformat(), date(year, 12, 31).isoformat() + + +# ── Step 4: Keyword extraction ──────────────────────────────────────────────── + +def extract_keywords( + query: str, + lang: str, + person_spans: list[str], + year_strings: list[str], +) -> list[str]: + """Return lowercased content words after removing persons, years, stopwords.""" + text = query + + # Remove matched person spans (longest first to avoid partial replacements) + for span in sorted(person_spans, key=len, reverse=True): + text = re.sub( + r"(? ParseResponse: - """Run the full NLP pipeline and return a ParseResponse.""" - nlp = get_nlp(lang) - doc = nlp(query) - - per_spans = [ent for ent in doc.ents if ent.label_ == "PER"] - - person_names = extract_person_names(doc) - person_role = detect_person_role(doc, per_spans, lang) - date_from, date_to = extract_dates(doc, lang) - keywords = extract_keywords(doc, list(doc.ents)) + """Run the full rule-based pipeline and return a ParseResponse.""" + person_names, person_role = _extract_persons_and_role(query, lang) + year_strings = [str(y) for _, _, y in _find_years(query)] + date_from, date_to = extract_dates(query, lang, person_names) + keywords = extract_keywords(query, lang, person_names, year_strings) return ParseResponse( personNames=person_names, diff --git a/nlp-service/main.py b/nlp-service/main.py index c440a1b0..7163c8ac 100644 --- a/nlp-service/main.py +++ b/nlp-service/main.py @@ -1,19 +1,38 @@ -import logging +"""FastAPI app — /parse and /health endpoints.""" +from __future__ import annotations + +import os from contextlib import asynccontextmanager from fastapi import FastAPI, HTTPException -from extractor import extract, load_all_models +from extractor import extract, get_person_matcher, set_person_matcher from models import ParseRequest, ParseResponse +from person_matcher import PersonMatcher -logger = logging.getLogger(__name__) + +def _load_persons_from_db(db_url: str) -> list[tuple[str | None, str | None]]: + import psycopg2 # deferred — not available in test environments without a DB + + conn = psycopg2.connect(db_url) + try: + cur = conn.cursor() + cur.execute("SELECT first_name, last_name FROM persons") + return cur.fetchall() + finally: + conn.close() @asynccontextmanager async def lifespan(app: FastAPI): - logger.info("Loading spaCy models...") - load_all_models() - logger.info("All models ready.") + # Only initialise the matcher when nothing was pre-seeded (e.g., by tests). + if get_person_matcher() is None: + m = PersonMatcher() + db_url = os.environ.get("DATABASE_URL") + if db_url: + rows = _load_persons_from_db(db_url) + m.load(rows) + set_person_matcher(m) yield @@ -22,7 +41,8 @@ app = FastAPI(lifespan=lifespan) @app.get("/health") def health() -> dict: - return {"status": "ok"} + m = get_person_matcher() + return {"status": "ok", "persons_loaded": len(m) if m else 0} @app.post("/parse", response_model=ParseResponse) @@ -30,5 +50,4 @@ def parse(request: ParseRequest) -> ParseResponse: try: return extract(request.query, request.lang) except Exception as exc: - logger.exception("Extraction pipeline failed") raise HTTPException(status_code=500, detail=str(exc)) from exc diff --git a/nlp-service/person_matcher.py b/nlp-service/person_matcher.py new file mode 100644 index 00000000..5e6f69c7 --- /dev/null +++ b/nlp-service/person_matcher.py @@ -0,0 +1,164 @@ +"""DB-backed person name matcher with fuzzy search.""" +from __future__ import annotations + +import re + +from rapidfuzz import fuzz, process + +_PUNCT_RE = re.compile(r"[^\w\s\-]", re.UNICODE) +_YEAR_PAT = re.compile(r"^\d{4}$") + + +class PersonMatcher: + """Match person name fragments from free-text queries against known persons. + + Loaded once at startup from (first_name, last_name) DB rows. At query + time, scans for tokens following person-indicator prepositions and fuzzy- + matches them against the loaded name variants. Returns the original query + text (not the resolved DB name) so the Java resolveNames() mechanism can + do its own disambiguation. + """ + + def __init__(self) -> None: + self._names: list[str] = [] # lowercase name variants + + # ── Loading ─────────────────────────────────────────────────────────────── + + def load(self, rows: list[tuple[str | None, str | None]]) -> None: + """Populate from DB rows of (first_name, last_name).""" + seen: set[str] = set() + for first, last in rows: + first = (first or "").strip() + last = (last or "").strip() + for variant in _name_variants(first, last): + key = variant.lower() + if key not in seen: + seen.add(key) + self._names.append(key) + + def __len__(self) -> int: + return len(self._names) + + # ── Query-time matching ─────────────────────────────────────────────────── + + def find_in_query( + self, + query: str, + prepositions: frozenset[str], + stop_tokens: frozenset[str] | None = None, + threshold: int = 80, + ) -> list[tuple[str, str | None]]: + """Find person name spans in *query*. + + Returns a list of ``(original_query_text, anchoring_prep_or_None)`` + in left-to-right order. + + Parameters + ---------- + prepositions: + Person-indicator prepositions for the query language (triggers a + scan for the tokens that follow). + stop_tokens: + Tokens that terminate a name span (prepositions + date-direction + words). "de" is a special exception: when immediately followed by + a capitalised word it is treated as a name connector (e.g. + "de Gruyter") rather than a stop. + threshold: + Minimum rapidfuzz token_sort_ratio score to accept a match. + + Strategy + -------- + Pass 1 — prep-anchored: for each person-indicator preposition found in + the token list, collect up to 3 consecutive non-stop, non-year tokens + after it and fuzzy-match the resulting span against loaded names. + Longest match wins. + + Pass 2 — full-name scan: scan positions not yet consumed for exact + multi-word full-name matches (no preposition anchor required). + """ + tokens = query.split() + clean = [_PUNCT_RE.sub("", t) for t in tokens] + lower = [t.lower() for t in clean] + + # Prepositions always terminate a name span, even without explicit stop_tokens. + stops = (stop_tokens or frozenset()) | prepositions + consumed: set[int] = set() + hits: list[tuple[int, str, str | None]] = [] # (position, text, prep) + + # Pass 1 — prep-anchored + for i, ltok in enumerate(lower): + if ltok not in prepositions or i + 1 >= len(tokens): + continue + + # Build candidate span — stop at stop tokens or 4-digit years. + # Exception: "de" before a capitalised word is a name connector. + span_indices: list[int] = [] + j = i + 1 + while j < len(tokens) and len(span_indices) < 3: + if j in consumed: + break + t = lower[j] + if t in stops or _YEAR_PAT.match(clean[j]): + # Allow "de" when the *next* token starts with a capital — + # e.g. "Walter de Gruyter". + next_clean = clean[j + 1] if j + 1 < len(tokens) else "" + if t == "de" and next_clean[:1].isupper(): + pass # connector — keep going + else: + break + span_indices.append(j) + j += 1 + + # Try longest match first, then shorter spans + for span_len in range(len(span_indices), 0, -1): + idx = span_indices[:span_len] + span_lower = " ".join(lower[k] for k in idx) + if self._is_match(span_lower, threshold): + hits.append((idx[0], " ".join(tokens[k] for k in idx), ltok)) + consumed.update(idx) + break + + # Pass 2 — full multi-word name scan (exact only, no preposition needed) + for span_len in (3, 2): + for i in range(len(tokens) - span_len + 1): + span_idx = range(i, i + span_len) + if any(j in consumed for j in span_idx): + continue + span_lower = " ".join(lower[i : i + span_len]) + if span_lower in self._names: + hits.append((i, " ".join(tokens[i : i + span_len]), None)) + consumed.update(span_idx) + + hits.sort(key=lambda h: h[0]) + return [(text, prep) for _, text, prep in hits] + + # ── Internal helpers ────────────────────────────────────────────────────── + + def _is_match(self, text: str, threshold: int) -> bool: + """Return True if *text* fuzzy-matches any loaded name at >= threshold.""" + if not self._names or len(text.strip()) < 3: + return False + text_lower = text.strip().lower() + if text_lower in self._names: + return True # exact match — fast path + result = process.extractOne( + text_lower, + self._names, + scorer=fuzz.token_sort_ratio, + score_cutoff=threshold, + ) + return result is not None + + +# ── helpers ─────────────────────────────────────────────────────────────────── + +def _name_variants(first: str, last: str) -> list[str]: + """Return the name variants to index for a single person.""" + variants = [] + if first and last: + variants.append(f"{first} {last}") + if first: + variants.append(first) + if last: + variants.append(last) + return variants diff --git a/nlp-service/requirements.txt b/nlp-service/requirements.txt index 14c14462..253b7ed0 100644 --- a/nlp-service/requirements.txt +++ b/nlp-service/requirements.txt @@ -1,6 +1,7 @@ fastapi[standard]==0.115.6 uvicorn[standard]==0.34.0 -spacy>=3.8,<4.0 dateparser>=1.2,<2.0 +rapidfuzz>=3.0,<4.0 +psycopg2-binary>=2.9,<3.0 pytest>=8.0,<9.0 httpx>=0.28,<1.0 diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py index fb117884..ac7490f1 100644 --- a/nlp-service/test_extractor.py +++ b/nlp-service/test_extractor.py @@ -1,350 +1,337 @@ +"""Tests for the rule-based extractor and PersonMatcher.""" import pytest -from pydantic import ValidationError - -# ── Models ────────────────────────────────────────────────────────────────── - -def test_parse_request_valid(): - from models import ParseRequest - req = ParseRequest(query="Briefe von Opa", lang="de") - assert req.query == "Briefe von Opa" - assert req.lang == "de" - - -def test_parse_request_rejects_unknown_lang(): - from models import ParseRequest - with pytest.raises(ValidationError): - ParseRequest(query="Letters from grandpa", lang="fr") - - -def test_parse_response_serializes_nulls(): - from models import ParseResponse - resp = ParseResponse( - personNames=["Opa"], - personRole="sender", - dateFrom=None, - dateTo="1920-12-31", - keywords=["brief"], - rawQuery="Briefe von Opa", - ) - data = resp.model_dump() - assert data["dateFrom"] is None - assert data["dateTo"] == "1920-12-31" - assert data["personRole"] == "sender" - - -# ── Model loading ──────────────────────────────────────────────────────────── - -@pytest.fixture(scope="session") -def nlp_de(): - from extractor import get_nlp - return get_nlp("de") - - -@pytest.fixture(scope="session") -def nlp_en(): - from extractor import get_nlp - return get_nlp("en") - - -@pytest.fixture(scope="session") -def nlp_es(): - from extractor import get_nlp - return get_nlp("es") - - -def test_get_nlp_de_loads(nlp_de): - doc = nlp_de("Test") - assert doc is not None - - -def test_get_nlp_en_loads(nlp_en): - doc = nlp_en("Test") - assert doc is not None - - -def test_get_nlp_es_loads(nlp_es): - doc = nlp_es("Prueba") - assert doc is not None - - -def test_get_nlp_unknown_lang_raises(): - from extractor import get_nlp - with pytest.raises(ValueError, match="Unsupported language"): - get_nlp("fr") - - -# ── Person name extraction ─────────────────────────────────────────────────── - -def _make_doc_with_ents(nlp, text: str, char_ents: list[tuple[int, int, str]]): - """Create a Doc with manually injected entity spans (no NER model needed).""" - doc = nlp.make_doc(text) - spans = [doc.char_span(s, e, label=lbl) for s, e, lbl in char_ents] - doc.ents = [sp for sp in spans if sp is not None] - return doc - - -def test_extract_person_names_two_persons(nlp_de): - from extractor import extract_person_names - # "Briefe von Opa Hermann an Marie" - # "Opa Hermann" = chars 11..22, "Marie" = chars 26..31 - doc = _make_doc_with_ents(nlp_de, "Briefe von Opa Hermann an Marie", [ - (11, 22, "PER"), - (26, 31, "PER"), - ]) - assert extract_person_names(doc) == ["Opa Hermann", "Marie"] - - -def test_extract_person_names_preserves_order(nlp_de): - from extractor import extract_person_names - # "Marie von Opa" — Marie comes first in text - # "Marie" = 0..5, "Opa" = 10..13 - doc = _make_doc_with_ents(nlp_de, "Marie von Opa", [ - (0, 5, "PER"), - (10, 13, "PER"), - ]) - assert extract_person_names(doc) == ["Marie", "Opa"] - - -def test_extract_person_names_empty(nlp_de): - from extractor import extract_person_names - doc = _make_doc_with_ents(nlp_de, "Briefe aus dem Krieg", []) - assert extract_person_names(doc) == [] - - -def test_extract_person_names_ignores_non_per(nlp_de): - from extractor import extract_person_names - # DATE entity should not appear in personNames - doc = _make_doc_with_ents(nlp_de, "Briefe 1920", [(7, 11, "DATE")]) - assert extract_person_names(doc) == [] - - -# ── Role detection ─────────────────────────────────────────────────────────── - -def test_role_sender_von(nlp_de): - from extractor import detect_person_role - # "Briefe von Marie" — "von" immediately before "Marie" - # "Marie" = chars 11..16 - doc = _make_doc_with_ents(nlp_de, "Briefe von Marie", [(11, 16, "PER")]) - per_spans = list(doc.ents) - assert detect_person_role(doc, per_spans, "de") == "sender" - - -def test_role_receiver_an(nlp_de): - from extractor import detect_person_role - # "Briefe an Marie" — "an" immediately before "Marie" - # "Marie" = chars 10..15 - doc = _make_doc_with_ents(nlp_de, "Briefe an Marie", [(10, 15, "PER")]) - per_spans = list(doc.ents) - assert detect_person_role(doc, per_spans, "de") == "receiver" - - -def test_role_two_persons_returns_any(nlp_de): - from extractor import detect_person_role - # "von Opa an Marie" — two PER spans → always "any" - # "Opa" = chars 4..7, "Marie" = chars 11..16 - doc = _make_doc_with_ents(nlp_de, "von Opa an Marie", [ - (4, 7, "PER"), - (11, 16, "PER"), - ]) - per_spans = list(doc.ents) - assert detect_person_role(doc, per_spans, "de") == "any" - - -def test_role_no_prep_returns_any(nlp_de): - from extractor import detect_person_role - # "Briefe Marie" — no preposition - # "Marie" = chars 7..12 - doc = _make_doc_with_ents(nlp_de, "Briefe Marie", [(7, 12, "PER")]) - per_spans = list(doc.ents) - assert detect_person_role(doc, per_spans, "de") == "any" - - -def test_role_empty_returns_any(nlp_de): - from extractor import detect_person_role - doc = _make_doc_with_ents(nlp_de, "Briefe 1920", []) - assert detect_person_role(doc, [], "de") == "any" - - -def test_role_sender_from_english(nlp_en): - from extractor import detect_person_role - # "letters from Marie" — "from" before "Marie" - # "Marie" = chars 13..18 - doc = _make_doc_with_ents(nlp_en, "letters from Marie", [(13, 18, "PER")]) - per_spans = list(doc.ents) - assert detect_person_role(doc, per_spans, "en") == "sender" - - -def test_role_receiver_to_english(nlp_en): - from extractor import detect_person_role - # "letters to Marie" — "to" before "Marie" - # "letters" = 0..7, " " = 7, "to" = 8..10, " " = 10, "Marie" = 11..16 - doc = _make_doc_with_ents(nlp_en, "letters to Marie", [(11, 16, "PER")]) - per_spans = list(doc.ents) - assert detect_person_role(doc, per_spans, "en") == "receiver" - - -# ── Date parsing ───────────────────────────────────────────────────────────── - -def test_date_vor_1920(nlp_de): - from extractor import extract_dates - # "Briefe vor 1920" — "1920" at chars 11..15 - doc = _make_doc_with_ents(nlp_de, "Briefe vor 1920", [(11, 15, "DATE")]) - date_from, date_to = extract_dates(doc, "de") - assert date_from is None - assert date_to == "1920-12-31" - - -def test_date_nach_1900(nlp_de): - from extractor import extract_dates - # "Briefe nach 1900" — "1900" at chars 12..16 - doc = _make_doc_with_ents(nlp_de, "Briefe nach 1900", [(12, 16, "DATE")]) - date_from, date_to = extract_dates(doc, "de") - assert date_from == "1900-01-01" - assert date_to is None - - -def test_date_zwischen_1900_und_1920(nlp_de): - from extractor import extract_dates - # "zwischen 1900 und 1920" - # "1900" = chars 9..13, "1920" = chars 18..22 - doc = _make_doc_with_ents(nlp_de, "zwischen 1900 und 1920", [ - (9, 13, "DATE"), - (18, 22, "DATE"), - ]) - date_from, date_to = extract_dates(doc, "de") - assert date_from == "1900-01-01" - assert date_to == "1920-12-31" - - -def test_date_bare_year_makes_range(nlp_de): - from extractor import extract_dates - # "Briefe 1920" — no direction token → year-range - # "1920" = chars 7..11 - doc = _make_doc_with_ents(nlp_de, "Briefe 1920", [(7, 11, "DATE")]) - date_from, date_to = extract_dates(doc, "de") - assert date_from == "1920-01-01" - assert date_to == "1920-12-31" - - -def test_date_no_date_entity(nlp_de): - from extractor import extract_dates - doc = _make_doc_with_ents(nlp_de, "Briefe von Opa", []) - date_from, date_to = extract_dates(doc, "de") - assert date_from is None - assert date_to is None - - -def test_date_before_english(nlp_en): - from extractor import extract_dates - # "letters before 1920" — "1920" at chars 15..19 - doc = _make_doc_with_ents(nlp_en, "letters before 1920", [(15, 19, "DATE")]) - date_from, date_to = extract_dates(doc, "en") - assert date_from is None - assert date_to == "1920-12-31" - - -def test_date_after_english(nlp_en): - from extractor import extract_dates - # "letters after 1900" — "1900" at chars 14..18 - doc = _make_doc_with_ents(nlp_en, "letters after 1900", [(14, 18, "DATE")]) - date_from, date_to = extract_dates(doc, "en") - assert date_from == "1900-01-01" - assert date_to is None - - -# ── Keyword extraction ─────────────────────────────────────────────────────── - -def test_keywords_extracts_nouns(nlp_de): - from extractor import extract_keywords - # Use real NLP for POS tags; disable NER to avoid interference - doc = nlp_de("Briefe aus dem Krieg", disable=["ner"]) - keywords = extract_keywords(doc, []) - # "Brief" (NOUN) and "Krieg" (NOUN) should appear as lemmas - assert "brief" in keywords - assert "krieg" in keywords - - -def test_keywords_excludes_stopwords(nlp_de): - from extractor import extract_keywords - doc = nlp_de("Briefe aus dem Krieg", disable=["ner"]) - keywords = extract_keywords(doc, []) - # "dem" is a stopword article — must not appear - assert "dem" not in keywords - - -def test_keywords_excludes_per_ner_spans(nlp_de): - from extractor import extract_keywords - # Run full NLP for POS tags, then inject a PER span over "Hermann" - # "Briefe von Hermann": B=0..6, ' '=6, v=7..10, ' '=10, H=11..18 - doc = nlp_de("Briefe von Hermann") - per_span = doc.char_span(11, 18, label="PER") - if per_span: - doc.ents = [per_span] - keywords = extract_keywords(doc, list(doc.ents)) - assert "hermann" not in keywords - - -def test_keywords_excludes_short_lemmas(nlp_de): - from extractor import extract_keywords - doc = nlp_de("Briefe an ihn", disable=["ner"]) - keywords = extract_keywords(doc, []) - # "ihn" is 3 chars but is a stopword pronoun; "an" is 2 chars - assert "an" not in keywords - - -def test_keywords_deduplicates(nlp_de): - from extractor import extract_keywords - doc = nlp_de("Brief Brief Krieg", disable=["ner"]) - keywords = extract_keywords(doc, []) - assert keywords.count("brief") == 1 - - -# ── Full extract() pipeline ────────────────────────────────────────────────── - -def test_extract_dates_de(): - from extractor import extract - result = extract("Briefe vor 1920", "de") - assert result.dateFrom is None - assert result.dateTo == "1920-12-31" - assert result.rawQuery == "Briefe vor 1920" - assert result.personNames == [] - assert result.personRole == "any" - - -def test_extract_keywords_from_topic_de(): - from extractor import extract - result = extract("Briefe aus dem Krieg", "de") - assert "krieg" in result.keywords - assert result.dateFrom is None - assert result.dateTo is None - - -def test_extract_dates_en(): - from extractor import extract - result = extract("letters before 1920", "en") - assert result.dateTo == "1920-12-31" - assert result.dateFrom is None - - -def test_extract_dates_es(): - from extractor import extract - result = extract("cartas antes de 1920", "es") - assert result.dateTo == "1920-12-31" - assert result.dateFrom is None - - -def test_extract_rawquery_echoed(): - from extractor import extract - q = "Texte über Weihnachten" - result = extract(q, "de") - assert result.rawQuery == q - - -def test_extract_response_fields_are_complete(): - from extractor import extract - result = extract("Briefe 1900", "de") - assert isinstance(result.personNames, list) - assert result.personRole in ("sender", "receiver", "any") - assert isinstance(result.keywords, list) - assert result.rawQuery == "Briefe 1900" +from extractor import extract, extract_dates, extract_keywords, set_person_matcher +from person_matcher import PersonMatcher + +# ── Shared test fixture ─────────────────────────────────────────────────────── + +_TEST_PERSONS = [ + ("Clara", "Cram"), + ("Herbert", "Cram"), + ("Eugenie", "de Gruyter"), + ("Walter", "de Gruyter"), + ("Marie", "Cram"), + ("Juan", "Cram"), + ("Hilde", "de Gruyter"), + ("Hans", "de Gruyter"), + ("Albert", "de Gruyter"), + ("Anita", "Wöhler"), + ("Else", "Bohrmann"), + ("Lili", "Duvenbeck"), +] + + +@pytest.fixture(scope="session", autouse=True) +def seeded_matcher(): + """Load test persons into the global matcher before any test runs.""" + m = PersonMatcher() + m.load(_TEST_PERSONS) + set_person_matcher(m) + return m + + +# ── PersonMatcher unit tests ────────────────────────────────────────────────── + +class TestPersonMatcher: + DE_PREPS = frozenset({"von", "vom", "an", "nach", "für"}) + + def test_load_populates_names(self, seeded_matcher): + assert len(seeded_matcher) > 0 + + def test_exact_full_name_match(self, seeded_matcher): + hits = seeded_matcher.find_in_query("Briefe von Clara Cram", self.DE_PREPS) + assert hits == [("Clara Cram", "von")] + + def test_exact_first_name_only(self, seeded_matcher): + hits = seeded_matcher.find_in_query("Briefe von Eugenie", self.DE_PREPS) + assert hits == [("Eugenie", "von")] + + def test_exact_first_name_receiver(self, seeded_matcher): + hits = seeded_matcher.find_in_query("Briefe an Herbert", self.DE_PREPS) + assert hits == [("Herbert", "an")] + + def test_fuzzy_typo(self, seeded_matcher): + hits = seeded_matcher.find_in_query("Briefe von Herrbert Cram", self.DE_PREPS) + assert len(hits) == 1 + assert hits[0][1] == "von" + + def test_two_persons_extracted(self, seeded_matcher): + hits = seeded_matcher.find_in_query( + "Briefe von Clara Cram an Herbert Cram", self.DE_PREPS + ) + assert len(hits) == 2 + assert hits[0][0] == "Clara Cram" + assert hits[0][1] == "von" + assert hits[1][0] == "Herbert Cram" + assert hits[1][1] == "an" + + def test_no_match_for_place_name(self, seeded_matcher): + hits = seeded_matcher.find_in_query("Reise nach Mexiko", self.DE_PREPS) + assert hits == [] + + def test_no_match_for_topic_word(self, seeded_matcher): + hits = seeded_matcher.find_in_query("Briefe aus dem Krieg", self.DE_PREPS) + assert hits == [] + + def test_first_name_eugenie_regression(self, seeded_matcher): + # spaCy NER missed standalone first names + hits = seeded_matcher.find_in_query("Briefe von Eugenie", self.DE_PREPS) + assert len(hits) == 1 + + def test_merged_names_regression(self, seeded_matcher): + # spaCy NER merged "Herbert an Eugenie de Gruyter" into one PER span + hits = seeded_matcher.find_in_query( + "Briefe von Herbert an Eugenie de Gruyter nach 1914", self.DE_PREPS + ) + assert len(hits) == 2 + names = [h[0] for h in hits] + assert "Herbert" in names + assert "Eugenie de Gruyter" in names + + def test_english_preps(self, seeded_matcher): + en_preps = frozenset({"from", "by", "to", "for"}) + hits = seeded_matcher.find_in_query( + "Letters from Clara Cram to Walter de Gruyter in 1920", en_preps + ) + assert len(hits) == 2 + assert hits[0][0] == "Clara Cram" + assert hits[1][0] == "Walter de Gruyter" + + def test_double_preposition_de(self, seeded_matcher): + hits = seeded_matcher.find_in_query( + "Briefe von Clara nach Herbert", self.DE_PREPS + ) + assert len(hits) == 2 + names = [h[0] for h in hits] + assert "Clara" in names + assert "Herbert" in names + + +# ── Date extraction tests ───────────────────────────────────────────────────── + +class TestExtractDates: + def test_bare_year_gives_range(self): + assert extract_dates("Briefe 1920", "de") == ("1920-01-01", "1920-12-31") + + def test_im_jahr(self): + assert extract_dates("Schriften im Jahr 1905", "de") == ( + "1905-01-01", "1905-12-31" + ) + + def test_vor_year(self): + assert extract_dates("Briefe vor 1920", "de") == (None, "1920-12-31") + + def test_nach_year(self): + assert extract_dates("Schriften nach 1920", "de") == ("1920-01-01", None) + + def test_zwischen(self): + assert extract_dates("Dokumente zwischen 1914 und 1918", "de") == ( + "1914-01-01", "1918-12-31" + ) + + def test_before_en(self): + assert extract_dates("Letters before 1918", "en") == (None, "1918-12-31") + + def test_after_en(self): + assert extract_dates("Letters after 1939", "en") == ("1939-01-01", None) + + def test_between_en(self): + assert extract_dates("Letters between 1914 and 1918", "en") == ( + "1914-01-01", "1918-12-31" + ) + + def test_antes_de_es(self): + assert extract_dates("Cartas antes de 1900", "es") == (None, "1900-12-31") + + def test_entre_es(self): + assert extract_dates("entre 1915 y 1920", "es") == ( + "1915-01-01", "1920-12-31" + ) + + def test_no_year(self): + assert extract_dates("Briefe aus dem Krieg", "de") == (None, None) + + def test_nach_before_person_then_year(self): + # "nach Marie 1920" — "nach" belongs to person, not date + date_from, date_to = extract_dates("Briefe nach Marie 1920", "de", ["Marie"]) + assert date_from == "1920-01-01" + assert date_to == "1920-12-31" + + def test_bare_year_alone(self): + assert extract_dates("1918", "de") == ("1918-01-01", "1918-12-31") + + +# ── Keyword extraction tests ────────────────────────────────────────────────── + +class TestExtractKeywords: + def test_basic_topic_words(self): + kws = extract_keywords("Briefe aus dem Krieg", "de", [], []) + assert "krieg" in kws + + def test_stopwords_excluded(self): + kws = extract_keywords("von der nach dem aus", "de", [], []) + for sw in ("von", "der", "nach", "dem", "aus"): + assert sw not in kws + + def test_person_spans_excluded(self): + kws = extract_keywords( + "Briefe von Clara Cram nach Herbert", "de", + ["Clara Cram", "Herbert"], [] + ) + assert "clara" not in kws + assert "cram" not in kws + assert "herbert" not in kws + + def test_years_excluded(self): + kws = extract_keywords("Schriften 1920 über Reise", "de", [], ["1920"]) + assert "1920" not in kws + + def test_deduplication(self): + kws = extract_keywords("Krieg Krieg Krieg", "de", [], []) + assert kws.count("krieg") == 1 + + def test_en_stopwords(self): + kws = extract_keywords("Letters about the war", "en", [], []) + assert "the" not in kws + assert "war" in kws + + def test_short_words_excluded(self): + kws = extract_keywords("ab cd ef xy", "de", [], []) + assert all(len(k) >= 3 for k in kws) + + +# ── Full pipeline integration tests ────────────────────────────────────────── + +class TestExtract: + def test_full_sentence_de(self): + r = extract("Briefe von Clara Cram an Walter de Gruyter im Jahr 1920", "de") + assert "Clara Cram" in r.personNames + assert "Walter de Gruyter" in r.personNames + assert r.personRole == "any" + assert r.dateFrom == "1920-01-01" + assert r.dateTo == "1920-12-31" + + def test_sender_role_de(self): + r = extract("Briefe von Clara Cram vor 1910", "de") + assert r.personNames == ["Clara Cram"] + assert r.personRole == "sender" + assert r.dateTo == "1910-12-31" + assert r.dateFrom is None + + def test_receiver_role_de(self): + r = extract("Briefe an Walter de Gruyter", "de") + assert r.personNames == ["Walter de Gruyter"] + assert r.personRole == "receiver" + + def test_first_name_only_eugenie(self): + r = extract("Briefe von Eugenie", "de") + assert "Eugenie" in r.personNames + assert r.personRole == "sender" + + def test_first_name_only_herbert(self): + r = extract("Kriegsbriefe von Herbert", "de") + assert "Herbert" in r.personNames + + def test_merged_names_bug_fixed(self): + r = extract("Briefe von Herbert an Eugenie de Gruyter nach 1914", "de") + assert "Herbert" in r.personNames + assert "Eugenie de Gruyter" in r.personNames + assert r.dateFrom == "1914-01-01" + + def test_topic_only_krieg(self): + r = extract("Briefe aus dem Krieg", "de") + assert r.personNames == [] + assert "krieg" in r.keywords + + def test_topic_only_single_word(self): + r = extract("Kriegspost", "de") + assert r.personNames == [] + + def test_date_range_only(self): + r = extract("Dokumente zwischen 1914 und 1918", "de") + assert r.personNames == [] + assert r.dateFrom == "1914-01-01" + assert r.dateTo == "1918-12-31" + + def test_colloquial_von(self): + r = extract("von Clara", "de") + assert r.personNames == ["Clara"] + assert r.personRole == "sender" + + def test_colloquial_an(self): + r = extract("an Walter", "de") + assert r.personNames == ["Walter"] + assert r.personRole == "receiver" + + def test_bare_year_alone(self): + r = extract("1918", "de") + assert r.dateFrom == "1918-01-01" + assert r.dateTo == "1918-12-31" + assert r.personNames == [] + + def test_english_full_sentence(self): + r = extract("Letters from Clara Cram to Walter de Gruyter in 1920", "en") + assert "Clara Cram" in r.personNames + assert "Walter de Gruyter" in r.personNames + assert r.dateFrom == "1920-01-01" + + def test_english_receiver_with_date(self): + r = extract("Letters to Herbert Cram after 1939", "en") + assert "Herbert Cram" in r.personNames + assert r.personRole == "receiver" + assert r.dateFrom == "1939-01-01" + + def test_english_birthday(self): + r = extract("Birthday greetings from Anita Wöhler", "en") + assert "Anita Wöhler" in r.personNames + assert r.personRole == "sender" + + def test_english_between_dates(self): + r = extract("Letters between 1914 and 1918", "en") + assert r.dateFrom == "1914-01-01" + assert r.dateTo == "1918-12-31" + + def test_spanish_full_sentence(self): + r = extract("Cartas de Clara Cram a Walter de Gruyter en 1920", "es") + assert "Clara Cram" in r.personNames + assert "Walter de Gruyter" in r.personNames + assert r.dateFrom == "1920-01-01" + + def test_spanish_before(self): + r = extract("Cartas antes de 1900", "es") + assert r.dateTo == "1900-12-31" + assert r.dateFrom is None + + def test_rawquery_echoed(self): + q = "test query" + r = extract(q, "de") + assert r.rawQuery == q + + def test_false_positive_compound_noun_regression(self): + # spaCy tagged "Geburtstagsglückwünsche" as a PER entity + r = extract("Geburtstagsglückwünsche", "de") + assert r.personNames == [] + + def test_question_phrasing(self): + r = extract("Wer hat an Herbert Cram 1918 geschrieben?", "de") + assert "Herbert Cram" in r.personNames + assert r.personRole == "receiver" + assert r.dateFrom == "1918-01-01" + + def test_lowercase_query(self): + r = extract("briefe von clara cram an herbert 1920", "de") + # Should still find persons despite lowercase + assert len(r.personNames) >= 1 + + def test_empty_matcher_returns_no_persons(self): + # Temporarily use an empty matcher + from extractor import set_person_matcher + empty = PersonMatcher() + set_person_matcher(empty) + r = extract("Briefe von Clara Cram", "de") + assert r.personNames == [] + # Restore seeded matcher + m = PersonMatcher() + m.load(_TEST_PERSONS) + set_person_matcher(m) diff --git a/nlp-service/test_main.py b/nlp-service/test_main.py index d9382e2d..0b169c34 100644 --- a/nlp-service/test_main.py +++ b/nlp-service/test_main.py @@ -1,43 +1,72 @@ +"""Integration tests for the FastAPI app.""" import pytest from fastapi.testclient import TestClient +from extractor import set_person_matcher +from person_matcher import PersonMatcher + +_TEST_PERSONS = [ + ("Clara", "Cram"), + ("Herbert", "Cram"), + ("Eugenie", "de Gruyter"), + ("Walter", "de Gruyter"), + ("Marie", "Cram"), + ("Anita", "Wöhler"), +] + @pytest.fixture(scope="session") def client(): + # Pre-seed the matcher so the lifespan doesn't overwrite it with an empty one. + m = PersonMatcher() + m.load(_TEST_PERSONS) + set_person_matcher(m) from main import app with TestClient(app) as c: yield c def test_health(client): - response = client.get("/health") - assert response.status_code == 200 - assert response.json() == {"status": "ok"} + r = client.get("/health") + assert r.status_code == 200 + assert r.json()["status"] == "ok" + assert r.json()["persons_loaded"] > 0 def test_parse_returns_200_with_all_fields(client): - response = client.post("/parse", json={"query": "Briefe vor 1920", "lang": "de"}) - assert response.status_code == 200 - data = response.json() - assert "personNames" in data - assert "personRole" in data - assert data["personRole"] in ("sender", "receiver", "any") - assert "dateFrom" in data - assert "dateTo" in data - assert "keywords" in data - assert "rawQuery" in data - assert data["rawQuery"] == "Briefe vor 1920" - assert data["dateTo"] == "1920-12-31" + r = client.post("/parse", json={"query": "Briefe vor 1920", "lang": "de"}) + assert r.status_code == 200 + d = r.json() + assert "personNames" in d + assert d["personRole"] in ("sender", "receiver", "any") + assert "dateFrom" in d + assert "dateTo" in d + assert "keywords" in d + assert d["rawQuery"] == "Briefe vor 1920" + assert d["dateTo"] == "1920-12-31" + + +def test_parse_person_with_date(client): + r = client.post( + "/parse", + json={"query": "Briefe von Clara Cram an Walter de Gruyter im Jahr 1920", "lang": "de"}, + ) + assert r.status_code == 200 + d = r.json() + assert "Clara Cram" in d["personNames"] + assert "Walter de Gruyter" in d["personNames"] + assert d["dateFrom"] == "1920-01-01" + assert d["dateTo"] == "1920-12-31" def test_parse_unknown_lang_returns_422(client): - response = client.post("/parse", json={"query": "test", "lang": "fr"}) - assert response.status_code == 422 + r = client.post("/parse", json={"query": "test", "lang": "fr"}) + assert r.status_code == 422 def test_parse_missing_query_returns_422(client): - response = client.post("/parse", json={"lang": "de"}) - assert response.status_code == 422 + r = client.post("/parse", json={"lang": "de"}) + assert r.status_code == 422 def test_parse_all_languages(client): @@ -47,6 +76,6 @@ def test_parse_all_languages(client): ("es", "cartas antes de 1920"), ] for lang, query in cases: - response = client.post("/parse", json={"query": query, "lang": lang}) - assert response.status_code == 200, f"Failed for lang={lang}" - assert response.json()["dateTo"] == "1920-12-31", f"Wrong dateTo for lang={lang}" + r = client.post("/parse", json={"query": query, "lang": lang}) + assert r.status_code == 200, f"Failed for lang={lang}" + assert r.json()["dateTo"] == "1920-12-31", f"Wrong dateTo for lang={lang}" diff --git a/nlp-service/test_sentences.md b/nlp-service/test_sentences.md new file mode 100644 index 00000000..66143c3f --- /dev/null +++ b/nlp-service/test_sentences.md @@ -0,0 +1,126 @@ +# NLP Service — Test Sentences + +Real data drawn from the Familienarchiv DB (2026-06-07). +Top persons: Clara Cram, Herbert Cram, Eugenie de Gruyter, Walter de Gruyter, Marie Cram, +Juan Cram, Albert de Gruyter, Hilde de Gruyter, Else Bohrmann, Anita Wöhler, Lili Duvenbeck. +Date range: ~1895–1945. Key tags: Krieg, Hochzeit, Reise, Geburtstag, Tod, Alltag, Briefwechsel. + +--- + +## German — full sentences + +```json +{"query": "Briefe von Clara Cram an Walter de Gruyter im Jahr 1920", "lang": "de"} +{"query": "Briefe von Herbert an Eugenie de Gruyter nach 1914", "lang": "de"} +{"query": "Schreiben von Albert de Gruyter an seine Kinder vor 1900", "lang": "de"} +{"query": "Briefe von Juan Cram an Marie zwischen 1915 und 1918", "lang": "de"} +{"query": "Telegramm von Walter de Gruyter an Clara im Jahr 1930", "lang": "de"} +{"query": "Briefe von Else Bohrmann an Herbert Cram nach 1939", "lang": "de"} +``` + +## German — medium (person + date, no strong role signal) + +```json +{"query": "Briefe von Clara Cram vor 1910", "lang": "de"} +{"query": "Dokumente über Walter de Gruyter aus den 1920er Jahren", "lang": "de"} +{"query": "Briefe an Herbert Cram nach dem Krieg", "lang": "de"} +{"query": "Schriften von Eugenie de Gruyter im Jahr 1905", "lang": "de"} +``` + +## German — short (person only) + +```json +{"query": "Briefe an Walter de Gruyter", "lang": "de"} +{"query": "Dokumente über Clara Cram", "lang": "de"} +{"query": "Herbert Cram", "lang": "de"} +{"query": "Anita Wöhler", "lang": "de"} +``` + +## German — topic only (keywords → tag resolution on Java side) + +```json +{"query": "Briefe aus dem Krieg", "lang": "de"} +{"query": "Kriegspost", "lang": "de"} +{"query": "Hochzeitsbriefe", "lang": "de"} +{"query": "Reisebriefe", "lang": "de"} +{"query": "Geburtstagsglückwünsche", "lang": "de"} +{"query": "Briefe über die Hochzeitsreise", "lang": "de"} +{"query": "Kinderbriefe", "lang": "de"} +{"query": "Familienbriefe aus dem Alltag", "lang": "de"} +{"query": "Brautbriefe", "lang": "de"} +{"query": "Kondolenzbriefe nach dem Tod von Eugenie", "lang": "de"} +``` + +## German — date range only + +```json +{"query": "Briefe aus dem Ersten Weltkrieg", "lang": "de"} +{"query": "Dokumente zwischen 1914 und 1918", "lang": "de"} +{"query": "Briefe vor 1900", "lang": "de"} +{"query": "Schriften nach 1920", "lang": "de"} +``` + +## German — combined (all fields) + +```json +{"query": "Briefe von Clara Cram an ihre Kinder über die Reise nach Mexiko im Jahr 1925", "lang": "de"} +{"query": "Kriegspost von Herbert Cram an Eugenie de Gruyter zwischen 1916 und 1918", "lang": "de"} +{"query": "Glückwünsche von Hilde de Gruyter zur Hochzeit im Jahr 1910", "lang": "de"} +{"query": "Kondolenzschreiben an Walter de Gruyter nach dem Tod von Eugenie", "lang": "de"} +``` + +## English + +```json +{"query": "Letters from Clara Cram to Walter de Gruyter in 1920", "lang": "en"} +{"query": "Letters about the war before 1918", "lang": "en"} +{"query": "Letters to Herbert Cram after 1939", "lang": "en"} +{"query": "Birthday greetings from Anita Wöhler", "lang": "en"} +{"query": "Letters between 1914 and 1918", "lang": "en"} +``` + +## Spanish + +```json +{"query": "Cartas de Clara Cram a Walter de Gruyter en 1920", "lang": "es"} +{"query": "Cartas antes de 1900", "lang": "es"} +{"query": "Cartas después de la guerra", "lang": "es"} +{"query": "Cartas de Juan Cram a sus hijos entre 1915 y 1920", "lang": "es"} +``` + +--- + +## Edge cases — lazy / missing words / typos + +```json +{"query": "Clara", "lang": "de"} +{"query": "Eugenie", "lang": "de"} +{"query": "Herbert", "lang": "de"} +{"query": "de Gruyter", "lang": "de"} +{"query": "Briefe von Klara Kram an Herbert", "lang": "de"} +{"query": "briefe von clara cram an herbert 1920", "lang": "de"} +{"query": "1918", "lang": "de"} +{"query": "1914 1918", "lang": "de"} +{"query": "Krieg", "lang": "de"} +{"query": "Briefe von Eugenie", "lang": "de"} +{"query": "Clara Cram Herbert Cram 1920", "lang": "de"} +{"query": "Wer hat an Herbert Cram 1918 geschrieben?", "lang": "de"} +{"query": "von Clara", "lang": "de"} +{"query": "an Walter", "lang": "de"} +{"query": "Clara 1920", "lang": "de"} +{"query": "Kriegsbriefe von Herbert", "lang": "de"} +{"query": "Briefe von Clara nach Herbert", "lang": "de"} +{"query": "Briefe von Herrbert Cram", "lang": "de"} +``` + +--- + +## Known spaCy failures now fixed by DB-backed matcher + +| Query | spaCy result | Expected | +|---|---|---| +| `Briefe von Eugenie` | persons=[] | persons=["Eugenie"] | +| `Kriegsbriefe von Herbert` | keywords=["herbert"] | persons=["Herbert"] | +| `Briefe von Herbert an Eugenie de Gruyter nach 1914` | persons=["Herbert an Eugenie de Gruyter"] (merged!) | persons=["Herbert", "Eugenie de Gruyter"] | +| `Letters from Clara Cram to Walter de Gruyter` | persons=[] (EN model doesn't know German names) | persons=["Clara Cram", "Walter de Gruyter"] | +| `Geburtstagsglückwünsche` | persons=["Geburtstagsglückwünsche"] (false positive!) | persons=[] | -- 2.49.1 From bda7855cade61597603ff2ef1a3942fa99e46dba Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 11:09:35 +0200 Subject: [PATCH 10/51] fix(nlp-service): eliminate false-positive person matches from dirty DB records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wire _EXTRA_SPAN_STOPS into _extract_persons_and_role so German function words (im, seine, ihre, dem, …) terminate name spans — fixes "Clara im" and "seine Kinder" leaking into personNames - Add _NON_NAME_TOKENS filter in PersonMatcher.load() to skip DB records whose first_name contains prepositions or possessives — filters 290 bad records (annotations like "an seine Eltern", "Eltern in", place references like "Enkel Cram aus Mexiko") that were causing exact Pass-2 matches - Remove spaCy model downloads from Dockerfile (no longer needed after the DB-backed matcher rewrite) Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/Dockerfile | 5 ----- nlp-service/extractor.py | 18 +++++++++++++++++- nlp-service/person_matcher.py | 20 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/nlp-service/Dockerfile b/nlp-service/Dockerfile index b47a71be..61c723b0 100644 --- a/nlp-service/Dockerfile +++ b/nlp-service/Dockerfile @@ -9,11 +9,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -# Bake models into the image — no volume needed, ~350 MB total -RUN python -m spacy download de_core_news_sm \ - && python -m spacy download en_core_web_sm \ - && python -m spacy download es_core_news_sm - COPY . . RUN useradd --no-create-home --shell /usr/sbin/nologin --uid 1001 nlp \ diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index b23a58b0..f4b16adf 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -66,6 +66,22 @@ _DATE_BETWEEN: dict[str, frozenset[str]] = { "es": frozenset({"entre"}), } +# ── Extra span-termination tokens (function words that cannot be in a name) ── + +_EXTRA_SPAN_STOPS: dict[str, frozenset[str]] = { + # German articles, possessives, and particles that end a name span + "de": frozenset({ + "im", "am", "beim", "zum", "zur", + "dem", "den", "des", + "sein", "seine", "seinen", "seiner", + "ihr", "ihre", "ihrem", "ihren", "ihrer", + "unser", "unsere", "unseren", + "über", "auch", "oder", "und", + }), + "en": frozenset(), + "es": frozenset({"el", "la", "los", "las", "su", "sus", "mi"}), +} + # ── Stopword lists ──────────────────────────────────────────────────────────── _STOPWORDS: dict[str, frozenset[str]] = { @@ -138,7 +154,7 @@ def _extract_persons_and_role( return [], "any" preps = _ALL_PERSON_PREPS[lang] - stops = preps | _DATE_BEFORE[lang] | _DATE_AFTER[lang] | _DATE_BETWEEN[lang] + stops = preps | _DATE_BEFORE[lang] | _DATE_AFTER[lang] | _DATE_BETWEEN[lang] | _EXTRA_SPAN_STOPS[lang] matches = m.find_in_query(query, preps, stop_tokens=stops) person_names = [text for text, _ in matches] diff --git a/nlp-service/person_matcher.py b/nlp-service/person_matcher.py index 5e6f69c7..2374208c 100644 --- a/nlp-service/person_matcher.py +++ b/nlp-service/person_matcher.py @@ -8,6 +8,22 @@ from rapidfuzz import fuzz, process _PUNCT_RE = re.compile(r"[^\w\s\-]", re.UNICODE) _YEAR_PAT = re.compile(r"^\d{4}$") +# Tokens that cannot appear in a real person's first name — used to filter DB +# records that are annotations or descriptions masquerading as persons. +_NON_NAME_TOKENS: frozenset[str] = frozenset({ + # German prepositions + "an", "in", "im", "am", "aus", "von", "vom", "nach", "zu", "zum", "zur", + "für", "bei", "beim", "mit", "über", "unter", "durch", "gegen", "ohne", + "bis", "seit", "des", "dem", "den", + # German possessives / pronouns + "sein", "seine", "seinen", "seiner", + "ihr", "ihre", "ihren", "ihrem", + # English prepositions + "for", "from", "by", "of", + # Spanish prepositions + "del", "por", "para", +}) + class PersonMatcher: """Match person name fragments from free-text queries against known persons. @@ -30,6 +46,10 @@ class PersonMatcher: for first, last in rows: first = (first or "").strip() last = (last or "").strip() + # Skip records whose first_name contains function words — these are + # annotations or descriptions in the DB, not real person names. + if any(w in _NON_NAME_TOKENS for w in first.lower().split()): + continue for variant in _name_variants(first, last): key = variant.lower() if key not in seen: -- 2.49.1 From d65879d27340c0b892675f0d31a313c98f5beadc Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:46:24 +0200 Subject: [PATCH 11/51] fix(nlp-service): return generic 500 detail to prevent credential leakage Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/main.py | 2 +- nlp-service/test_main.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/nlp-service/main.py b/nlp-service/main.py index 7163c8ac..3894cca0 100644 --- a/nlp-service/main.py +++ b/nlp-service/main.py @@ -50,4 +50,4 @@ def parse(request: ParseRequest) -> ParseResponse: try: return extract(request.query, request.lang) except Exception as exc: - raise HTTPException(status_code=500, detail=str(exc)) from exc + raise HTTPException(status_code=500, detail="internal error") from exc diff --git a/nlp-service/test_main.py b/nlp-service/test_main.py index 0b169c34..f02f3c03 100644 --- a/nlp-service/test_main.py +++ b/nlp-service/test_main.py @@ -79,3 +79,17 @@ def test_parse_all_languages(client): r = client.post("/parse", json={"query": query, "lang": lang}) assert r.status_code == 200, f"Failed for lang={lang}" assert r.json()["dateTo"] == "1920-12-31", f"Wrong dateTo for lang={lang}" + + +def test_parse_internal_exception_does_not_leak_detail(client, monkeypatch): + """500 errors must return generic message — never expose internal details.""" + import main as main_module + + def _boom(query, lang): + raise RuntimeError("postgresql://archive_user:s3cr3t@db:5432/family_archive_db") + + monkeypatch.setattr(main_module, "extract", _boom) + r = client.post("/parse", json={"query": "test", "lang": "de"}) + assert r.status_code == 500 + assert "s3cr3t" not in r.text + assert r.json()["detail"] == "internal error" -- 2.49.1 From 8d4f30019b388ca061b955aad845d1e678674af7 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:47:03 +0200 Subject: [PATCH 12/51] feat(nlp-service): log WARNING when DATABASE_URL absent, ERROR on DB failure Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/main.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nlp-service/main.py b/nlp-service/main.py index 3894cca0..52533188 100644 --- a/nlp-service/main.py +++ b/nlp-service/main.py @@ -1,11 +1,14 @@ """FastAPI app — /parse and /health endpoints.""" from __future__ import annotations +import logging import os from contextlib import asynccontextmanager from fastapi import FastAPI, HTTPException +logger = logging.getLogger(__name__) + from extractor import extract, get_person_matcher, set_person_matcher from models import ParseRequest, ParseResponse from person_matcher import PersonMatcher @@ -30,8 +33,14 @@ async def lifespan(app: FastAPI): m = PersonMatcher() db_url = os.environ.get("DATABASE_URL") if db_url: - rows = _load_persons_from_db(db_url) - m.load(rows) + try: + rows = _load_persons_from_db(db_url) + m.load(rows) + logger.info("PersonMatcher loaded %d name variants from DB", len(m)) + except Exception: + logger.error("Failed to load persons from DB — person matching disabled", exc_info=True) + else: + logger.warning("DATABASE_URL not set — person matching disabled") set_person_matcher(m) yield -- 2.49.1 From 778382cd610cebe22930354968c51b699fd6615f Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:47:40 +0200 Subject: [PATCH 13/51] feat(nlp-service): cap /parse query at 500 chars via Field(max_length=500) Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/models.py | 4 ++-- nlp-service/test_main.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nlp-service/models.py b/nlp-service/models.py index e36fb89e..7015d36a 100644 --- a/nlp-service/models.py +++ b/nlp-service/models.py @@ -1,10 +1,10 @@ from __future__ import annotations from typing import Literal -from pydantic import BaseModel +from pydantic import BaseModel, Field class ParseRequest(BaseModel): - query: str + query: str = Field(max_length=500) lang: Literal["de", "en", "es"] diff --git a/nlp-service/test_main.py b/nlp-service/test_main.py index f02f3c03..31ec4766 100644 --- a/nlp-service/test_main.py +++ b/nlp-service/test_main.py @@ -81,6 +81,11 @@ def test_parse_all_languages(client): assert r.json()["dateTo"] == "1920-12-31", f"Wrong dateTo for lang={lang}" +def test_parse_exceeds_max_length_returns_422(client): + r = client.post("/parse", json={"query": "a" * 501, "lang": "de"}) + assert r.status_code == 422 + + def test_parse_internal_exception_does_not_leak_detail(client, monkeypatch): """500 errors must return generic message — never expose internal details.""" import main as main_module -- 2.49.1 From 98ee6cf5871848e460c9ad2698c6d44e708ebbfa Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:48:57 +0200 Subject: [PATCH 14/51] feat(nlp-service): wire NLP_FUZZY_THRESHOLD env var with 0-100 validation Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/extractor.py | 10 ++++++++-- nlp-service/main.py | 19 ++++++++++++++++++- nlp-service/test_main.py | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index f4b16adf..de884910 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -13,9 +13,10 @@ from person_matcher import PersonMatcher if TYPE_CHECKING: pass -# ── Module-level PersonMatcher (set at startup) ─────────────────────────────── +# ── Module-level PersonMatcher and fuzzy threshold (set at startup) ────────── _matcher: PersonMatcher | None = None +_fuzzy_threshold: int = 80 def set_person_matcher(m: PersonMatcher) -> None: @@ -27,6 +28,11 @@ def get_person_matcher() -> PersonMatcher | None: return _matcher +def set_fuzzy_threshold(threshold: int) -> None: + global _fuzzy_threshold + _fuzzy_threshold = threshold + + # ── Preposition sets ────────────────────────────────────────────────────────── _SENDER_PREPS: dict[str, frozenset[str]] = { @@ -155,7 +161,7 @@ def _extract_persons_and_role( preps = _ALL_PERSON_PREPS[lang] stops = preps | _DATE_BEFORE[lang] | _DATE_AFTER[lang] | _DATE_BETWEEN[lang] | _EXTRA_SPAN_STOPS[lang] - matches = m.find_in_query(query, preps, stop_tokens=stops) + matches = m.find_in_query(query, preps, stop_tokens=stops, threshold=_fuzzy_threshold) person_names = [text for text, _ in matches] diff --git a/nlp-service/main.py b/nlp-service/main.py index 52533188..509ae7b1 100644 --- a/nlp-service/main.py +++ b/nlp-service/main.py @@ -9,10 +9,23 @@ from fastapi import FastAPI, HTTPException logger = logging.getLogger(__name__) -from extractor import extract, get_person_matcher, set_person_matcher +from extractor import extract, get_person_matcher, set_fuzzy_threshold, set_person_matcher from models import ParseRequest, ParseResponse from person_matcher import PersonMatcher +_DEFAULT_FUZZY_THRESHOLD = 80 + + +def _parse_fuzzy_threshold(val: str) -> int: + """Parse and validate NLP_FUZZY_THRESHOLD — must be integer in [0, 100].""" + try: + n = int(val) + except ValueError: + raise ValueError(f"NLP_FUZZY_THRESHOLD must be an integer, got: {val!r}") + if not (0 <= n <= 100): + raise ValueError(f"NLP_FUZZY_THRESHOLD must be between 0 and 100, got: {n}") + return n + def _load_persons_from_db(db_url: str) -> list[tuple[str | None, str | None]]: import psycopg2 # deferred — not available in test environments without a DB @@ -28,6 +41,10 @@ def _load_persons_from_db(db_url: str) -> list[tuple[str | None, str | None]]: @asynccontextmanager async def lifespan(app: FastAPI): + threshold_raw = os.environ.get("NLP_FUZZY_THRESHOLD", str(_DEFAULT_FUZZY_THRESHOLD)) + threshold = _parse_fuzzy_threshold(threshold_raw) + set_fuzzy_threshold(threshold) + # Only initialise the matcher when nothing was pre-seeded (e.g., by tests). if get_person_matcher() is None: m = PersonMatcher() diff --git a/nlp-service/test_main.py b/nlp-service/test_main.py index 31ec4766..5a81156a 100644 --- a/nlp-service/test_main.py +++ b/nlp-service/test_main.py @@ -81,6 +81,23 @@ def test_parse_all_languages(client): assert r.json()["dateTo"] == "1920-12-31", f"Wrong dateTo for lang={lang}" +def test_fuzzy_threshold_valid_range(): + from main import _parse_fuzzy_threshold + assert _parse_fuzzy_threshold("80") == 80 + assert _parse_fuzzy_threshold("0") == 0 + assert _parse_fuzzy_threshold("100") == 100 + + +def test_fuzzy_threshold_out_of_range_raises(): + from main import _parse_fuzzy_threshold + with pytest.raises(ValueError): + _parse_fuzzy_threshold("101") + with pytest.raises(ValueError): + _parse_fuzzy_threshold("-1") + with pytest.raises(ValueError): + _parse_fuzzy_threshold("abc") + + def test_parse_exceeds_max_length_returns_422(client): r = client.post("/parse", json={"query": "a" * 501, "lang": "de"}) assert r.status_code == 422 -- 2.49.1 From 829194f3457a833268a0f1f21c3fd7ee2723d4b0 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:49:37 +0200 Subject: [PATCH 15/51] chore(nlp-service): remove unused dateparser dependency Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/extractor.py | 2 -- nlp-service/requirements.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py index de884910..dace65cf 100644 --- a/nlp-service/extractor.py +++ b/nlp-service/extractor.py @@ -5,8 +5,6 @@ import re from datetime import date from typing import TYPE_CHECKING -import dateparser - from models import ParseResponse from person_matcher import PersonMatcher diff --git a/nlp-service/requirements.txt b/nlp-service/requirements.txt index 253b7ed0..42935fcc 100644 --- a/nlp-service/requirements.txt +++ b/nlp-service/requirements.txt @@ -1,6 +1,5 @@ fastapi[standard]==0.115.6 uvicorn[standard]==0.34.0 -dateparser>=1.2,<2.0 rapidfuzz>=3.0,<4.0 psycopg2-binary>=2.9,<3.0 pytest>=8.0,<9.0 -- 2.49.1 From f4e8632e0d59584e0688ba98e7e989a7d9f9445d Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:50:01 +0200 Subject: [PATCH 16/51] chore(nlp-service): add .dockerignore to exclude dev artifacts from image Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/.dockerignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 nlp-service/.dockerignore diff --git a/nlp-service/.dockerignore b/nlp-service/.dockerignore new file mode 100644 index 00000000..4a5db792 --- /dev/null +++ b/nlp-service/.dockerignore @@ -0,0 +1,6 @@ +venv/ +.env +__pycache__/ +.pytest_cache/ +test_*.py +*.md -- 2.49.1 From 324a76d6d291c273403b7b8eec8ff7b0cdd853f5 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:50:32 +0200 Subject: [PATCH 17/51] test(nlp-service): guard global matcher state in try/finally Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/test_extractor.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py index ac7490f1..1bf47a9c 100644 --- a/nlp-service/test_extractor.py +++ b/nlp-service/test_extractor.py @@ -324,14 +324,12 @@ class TestExtract: # Should still find persons despite lowercase assert len(r.personNames) >= 1 - def test_empty_matcher_returns_no_persons(self): - # Temporarily use an empty matcher - from extractor import set_person_matcher - empty = PersonMatcher() - set_person_matcher(empty) - r = extract("Briefe von Clara Cram", "de") - assert r.personNames == [] - # Restore seeded matcher - m = PersonMatcher() - m.load(_TEST_PERSONS) - set_person_matcher(m) + def test_empty_matcher_returns_no_persons(self, seeded_matcher): + from extractor import get_person_matcher, set_person_matcher + original = get_person_matcher() + try: + set_person_matcher(PersonMatcher()) + r = extract("Briefe von Clara Cram", "de") + assert r.personNames == [] + finally: + set_person_matcher(original) -- 2.49.1 From 4cbe1dc2d3f4dcb3a9ae9c7ea018c7a0377efcfd Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:51:26 +0200 Subject: [PATCH 18/51] feat(search): add NlpExtraction record, NlpClient and NlpHealthClient interfaces Co-Authored-By: Claude Sonnet 4.6 --- .../raddatz/familienarchiv/search/NlpClient.java | 5 +++++ .../familienarchiv/search/NlpExtraction.java | 14 ++++++++++++++ .../familienarchiv/search/NlpHealthClient.java | 5 +++++ 3 files changed, 24 insertions(+) create mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlpClient.java create mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlpExtraction.java create mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlpHealthClient.java diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpClient.java new file mode 100644 index 00000000..c7889c7e --- /dev/null +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpClient.java @@ -0,0 +1,5 @@ +package org.raddatz.familienarchiv.search; + +public interface NlpClient { + NlpExtraction parse(String query, String lang); +} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpExtraction.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpExtraction.java new file mode 100644 index 00000000..73c36027 --- /dev/null +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpExtraction.java @@ -0,0 +1,14 @@ +package org.raddatz.familienarchiv.search; + +import java.time.LocalDate; +import java.util.List; + +record NlpExtraction( + List personNames, + String personRole, + LocalDate dateFrom, + LocalDate dateTo, + List keywords, + String rawQuery +) { +} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpHealthClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpHealthClient.java new file mode 100644 index 00000000..a02475c2 --- /dev/null +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpHealthClient.java @@ -0,0 +1,5 @@ +package org.raddatz.familienarchiv.search; + +public interface NlpHealthClient { + boolean isHealthy(); +} -- 2.49.1 From c8543726ec2afd9e9a3c0c7879c488636260c5f7 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:52:12 +0200 Subject: [PATCH 19/51] feat(search): add NlpProperties config and @ConfigurationPropertiesScan Co-Authored-By: Claude Sonnet 4.6 --- .../FamilienarchivApplication.java | 2 ++ .../familienarchiv/search/NlpProperties.java | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlpProperties.java diff --git a/backend/src/main/java/org/raddatz/familienarchiv/FamilienarchivApplication.java b/backend/src/main/java/org/raddatz/familienarchiv/FamilienarchivApplication.java index 4fef338f..0b358e80 100644 --- a/backend/src/main/java/org/raddatz/familienarchiv/FamilienarchivApplication.java +++ b/backend/src/main/java/org/raddatz/familienarchiv/FamilienarchivApplication.java @@ -2,8 +2,10 @@ package org.raddatz.familienarchiv; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; @SpringBootApplication +@ConfigurationPropertiesScan public class FamilienarchivApplication { public static void main(String[] args) { diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpProperties.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpProperties.java new file mode 100644 index 00000000..8b939e1e --- /dev/null +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpProperties.java @@ -0,0 +1,16 @@ +package org.raddatz.familienarchiv.search; + +import jakarta.validation.constraints.NotBlank; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@ConfigurationProperties("app.nlp") +@Data +@Validated +public class NlpProperties { + @NotBlank + private String baseUrl; + private int timeoutSeconds = 5; + private int healthCheckTimeoutSeconds = 2; +} -- 2.49.1 From 381bd1d943a045f77a34881ca308aad75afc8f16 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:54:39 +0200 Subject: [PATCH 20/51] =?UTF-8?q?test(search):=20NlpPropertiesTest=20?= =?UTF-8?q?=E2=80=94=20validates=20baseUrl=20required=20and=20defaults?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .../search/NlpPropertiesTest.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/NlpPropertiesTest.java diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlpPropertiesTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlpPropertiesTest.java new file mode 100644 index 00000000..bb527066 --- /dev/null +++ b/backend/src/test/java/org/raddatz/familienarchiv/search/NlpPropertiesTest.java @@ -0,0 +1,33 @@ +package org.raddatz.familienarchiv.search; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +class NlpPropertiesTest { + + @EnableConfigurationProperties(NlpProperties.class) + static class TestConfig {} + + private final ApplicationContextRunner runner = new ApplicationContextRunner() + .withUserConfiguration(TestConfig.class); + + @Test + void failsWhenBaseUrlMissing() { + runner.run(context -> assertThat(context).hasFailed()); + } + + @Test + void bindsBaseUrlAndDefaults() { + runner.withPropertyValues("app.nlp.base-url=http://nlp:8001") + .run(context -> { + assertThat(context).hasNotFailed(); + NlpProperties props = context.getBean(NlpProperties.class); + assertThat(props.getBaseUrl()).isEqualTo("http://nlp:8001"); + assertThat(props.getTimeoutSeconds()).isEqualTo(5); + assertThat(props.getHealthCheckTimeoutSeconds()).isEqualTo(2); + }); + } +} -- 2.49.1 From 1fcadfcd8fbaf93f550caf29ba54bbe50defa588 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:55:50 +0200 Subject: [PATCH 21/51] =?UTF-8?q?feat(search):=20add=20RestClientNlpClient?= =?UTF-8?q?=20=E2=80=94=20POST=20/parse,=20GET=20/health=20with=20persons?= =?UTF-8?q?=5Floaded=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .../search/RestClientNlpClient.java | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/RestClientNlpClient.java diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientNlpClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientNlpClient.java new file mode 100644 index 00000000..d058e470 --- /dev/null +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientNlpClient.java @@ -0,0 +1,145 @@ +package org.raddatz.familienarchiv.search; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.extern.slf4j.Slf4j; +import org.raddatz.familienarchiv.exception.DomainException; +import org.raddatz.familienarchiv.exception.ErrorCode; +import org.springframework.http.MediaType; +import org.springframework.http.client.JdkClientHttpRequestFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClient; + +import java.net.http.HttpClient; +import java.time.Duration; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.Set; + +@Service +@Slf4j +public class RestClientNlpClient implements NlpClient, NlpHealthClient { + + private static final Set VALID_ROLES = Set.of("sender", "receiver", "any"); + private static final int MAX_NAME_LENGTH = 200; + private static final int MAX_KEYWORD_LENGTH = 100; + + private final RestClient parseClient; + private final RestClient healthRestClient; + + public RestClientNlpClient(NlpProperties props) { + HttpClient parseHttp = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .connectTimeout(Duration.ofSeconds(10)) + .build(); + JdkClientHttpRequestFactory parseFactory = new JdkClientHttpRequestFactory(parseHttp); + parseFactory.setReadTimeout(Duration.ofSeconds(props.getTimeoutSeconds())); + this.parseClient = RestClient.builder() + .baseUrl(props.getBaseUrl()) + .requestFactory(parseFactory) + .build(); + + HttpClient healthHttp = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .connectTimeout(Duration.ofSeconds(props.getHealthCheckTimeoutSeconds())) + .build(); + JdkClientHttpRequestFactory healthFactory = new JdkClientHttpRequestFactory(healthHttp); + healthFactory.setReadTimeout(Duration.ofSeconds(props.getHealthCheckTimeoutSeconds())); + this.healthRestClient = RestClient.builder() + .baseUrl(props.getBaseUrl()) + .requestFactory(healthFactory) + .build(); + } + + @Override + public NlpExtraction parse(String query, String lang) { + try { + NlpParseRequest request = new NlpParseRequest(query, lang); + NlpParseResponse response = parseClient.post() + .uri("/parse") + .contentType(MediaType.APPLICATION_JSON) + .body(request) + .retrieve() + .body(NlpParseResponse.class); + return toExtraction(response, query); + } catch (DomainException e) { + throw e; + } catch (Exception e) { + log.warn("NLP service inference failed: {}", e.getClass().getSimpleName()); + throw DomainException.serviceUnavailable(ErrorCode.SMART_SEARCH_UNAVAILABLE, + "NLP service unavailable: " + e.getClass().getSimpleName()); + } + } + + @Override + public boolean isHealthy() { + try { + NlpHealthResponse resp = healthRestClient.get() + .uri("/health") + .retrieve() + .body(NlpHealthResponse.class); + return resp != null && resp.personsLoaded() > 0; + } catch (Exception e) { + return false; + } + } + + private NlpExtraction toExtraction(NlpParseResponse response, String rawQuery) { + if (response == null) { + return fallbackExtraction(rawQuery); + } + List names = response.personNames() == null ? List.of() : response.personNames().stream() + .filter(n -> n != null && n.length() <= MAX_NAME_LENGTH) + .toList(); + List keywords = response.keywords() == null ? List.of() : response.keywords().stream() + .filter(k -> k != null && k.length() <= MAX_KEYWORD_LENGTH) + .toList(); + String role = sanitiseRole(response.personRole()); + LocalDate dateFrom = parseDate(response.dateFrom()); + LocalDate dateTo = parseDate(response.dateTo()); + return new NlpExtraction(names, role, dateFrom, dateTo, keywords, rawQuery); + } + + private NlpExtraction fallbackExtraction(String rawQuery) { + return new NlpExtraction(List.of(), "any", null, null, List.of(), rawQuery); + } + + private String sanitiseRole(String role) { + if (role != null && VALID_ROLES.contains(role)) { + return role; + } + log.warn("Unexpected personRole from NLP service: {}", role); + return "any"; + } + + private LocalDate parseDate(String raw) { + if (raw == null || raw.isBlank()) return null; + try { + return LocalDate.parse(raw, DateTimeFormatter.ISO_LOCAL_DATE); + } catch (DateTimeParseException ignored) { + } + return null; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + private record NlpParseRequest(String query, String lang) { + } + + @JsonIgnoreProperties(ignoreUnknown = true) + private record NlpParseResponse( + @JsonProperty("personNames") List personNames, + @JsonProperty("personRole") String personRole, + @JsonProperty("dateFrom") String dateFrom, + @JsonProperty("dateTo") String dateTo, + @JsonProperty("keywords") List keywords + ) { + } + + @JsonIgnoreProperties(ignoreUnknown = true) + private record NlpHealthResponse( + @JsonProperty("persons_loaded") int personsLoaded + ) { + } +} -- 2.49.1 From 9f78f25b0a77563bb1e17d6fa60466945afb1bf1 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:58:48 +0200 Subject: [PATCH 22/51] =?UTF-8?q?feat(search):=20thread=20lang=20through?= =?UTF-8?q?=20NlSearchRequest=20=E2=86=92=20controller=20=E2=86=92=20NlQue?= =?UTF-8?q?ryParserService=20=E2=86=92=20NlpClient?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - NlSearchRequest gains @NotBlank @Pattern(regexp="de|en|es") lang field - NlSearchController passes request.lang() to service - NlQueryParserService.search signature: (String, String, Pageable); renames ollamaClient→nlpClient; removes redundant length guard (Bean Validation is enforcement point) - application.yaml: replaces app.ollama.* with app.nlp.base-url; application-dev.yaml: points to localhost:8001 - frontend/documents/+page.svelte: sends lang: languageTag() in POST body Co-Authored-By: Claude Sonnet 4.6 --- .../familienarchiv/search/NlQueryParserService.java | 11 +++-------- .../familienarchiv/search/NlSearchController.java | 2 +- .../familienarchiv/search/NlSearchRequest.java | 6 +++++- backend/src/main/resources/application-dev.yaml | 4 ++-- backend/src/main/resources/application.yaml | 9 ++------- frontend/src/routes/documents/+page.svelte | 3 ++- 6 files changed, 15 insertions(+), 20 deletions(-) diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryParserService.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryParserService.java index 7f9d1edb..db8a9da9 100644 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryParserService.java +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryParserService.java @@ -34,18 +34,13 @@ public class NlQueryParserService { private static final int MIN_TAG_TERM = 3; private static final int MAX_RESOLVED_TAGS = 10; - private final OllamaClient ollamaClient; + private final NlpClient nlpClient; private final PersonService personService; private final DocumentService documentService; private final TagService tagService; - public NlSearchResponse search(String query, Pageable pageable) { - if (query == null || query.length() < MIN_QUERY || query.length() > MAX_QUERY) { - throw DomainException.badRequest(ErrorCode.VALIDATION_ERROR, - "Query must be between " + MIN_QUERY + " and " + MAX_QUERY + " characters"); - } - - OllamaExtraction ext = ollamaClient.parse(query); + public NlSearchResponse search(String query, String lang, Pageable pageable) { + NlpExtraction ext = nlpClient.parse(query, lang); List personNames = ext.personNames() != null ? ext.personNames() : List.of(); List keywords = ext.keywords() != null ? ext.keywords() : List.of(); diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchController.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchController.java index c58fff38..82391a72 100644 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchController.java +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchController.java @@ -23,6 +23,6 @@ public class NlSearchController { Pageable pageable, @AuthenticationPrincipal UserDetails principal) { rateLimiter.checkAndConsume(principal.getUsername()); - return nlQueryParserService.search(request.query(), pageable); + return nlQueryParserService.search(request.query(), request.lang(), pageable); } } diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRequest.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRequest.java index 0e9d3a9a..f23241d0 100644 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRequest.java +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRequest.java @@ -1,11 +1,15 @@ package org.raddatz.familienarchiv.search; import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Pattern; import jakarta.validation.constraints.Size; public record NlSearchRequest( @NotBlank @Size(min = 3, max = 500) - String query + String query, + @NotBlank + @Pattern(regexp = "de|en|es") + String lang ) { } diff --git a/backend/src/main/resources/application-dev.yaml b/backend/src/main/resources/application-dev.yaml index 954e430b..4f64aef9 100644 --- a/backend/src/main/resources/application-dev.yaml +++ b/backend/src/main/resources/application-dev.yaml @@ -13,5 +13,5 @@ springdoc: path: /swagger-ui.html app: - ollama: - base-url: http://localhost:11434 + nlp: + base-url: http://localhost:8001 diff --git a/backend/src/main/resources/application.yaml b/backend/src/main/resources/application.yaml index ce517f25..a0054de6 100644 --- a/backend/src/main/resources/application.yaml +++ b/backend/src/main/resources/application.yaml @@ -130,13 +130,8 @@ app: # The loader maps columns by header name — no positional indices (see ADR-025). dir: ${IMPORT_DIR:/import} - ollama: - base-url: http://ollama:11434 - model: qwen2.5:7b-instruct-q4_K_M - # CPU inference: ~18s warm. Higher ceiling absorbs the cold model load on the - # first query after an Ollama (re)start before OLLAMA_KEEP_ALIVE pins it. - timeout-seconds: 60 - health-check-timeout-seconds: 2 + nlp: + base-url: http://nlp-service:8001 nl-search: rate-limit: diff --git a/frontend/src/routes/documents/+page.svelte b/frontend/src/routes/documents/+page.svelte index 7d494b24..92556463 100644 --- a/frontend/src/routes/documents/+page.svelte +++ b/frontend/src/routes/documents/+page.svelte @@ -17,6 +17,7 @@ import { bulkSelectionStore } from '$lib/document/bulkSelection.svelte'; import { getErrorMessage, parseBackendError } from '$lib/shared/errors'; import { csrfFetch } from '$lib/shared/cookies'; import * as m from '$lib/paraglide/messages.js'; +import { languageTag } from '$lib/paraglide/runtime'; import type { components } from '$lib/generated/api'; type NlQueryInterpretation = components['schemas']['NlQueryInterpretation']; @@ -224,7 +225,7 @@ async function runSmartSearch() { const res = await csrfFetch('/api/search/nl', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ query }) + body: JSON.stringify({ query, lang: languageTag() }) }); if (!res.ok) { const backend = await parseBackendError(res); -- 2.49.1 From 28201e363a6a2f195458fabf342ca76b8bae9c74 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 15:59:20 +0200 Subject: [PATCH 23/51] refactor(search): delete Ollama* classes replaced by Nlp* equivalents Co-Authored-By: Claude Sonnet 4.6 --- .../familienarchiv/search/OllamaClient.java | 5 - .../search/OllamaExtraction.java | 18 -- .../search/OllamaHealthClient.java | 5 - .../search/OllamaProperties.java | 15 -- .../search/RestClientOllamaClient.java | 184 ------------------ 5 files changed, 227 deletions(-) delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/OllamaClient.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/OllamaExtraction.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/OllamaHealthClient.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/OllamaProperties.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/RestClientOllamaClient.java diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaClient.java deleted file mode 100644 index 8517d4df..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaClient.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.raddatz.familienarchiv.search; - -public interface OllamaClient { - OllamaExtraction parse(String query); -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaExtraction.java b/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaExtraction.java deleted file mode 100644 index cc3dce6a..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaExtraction.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import java.time.LocalDate; -import java.util.List; - -/** - * Raw structured output from Ollama after parsing and sanitising. - * personRole is always one of "sender", "receiver", "any" — defensive parsing ensures this. - */ -record OllamaExtraction( - List personNames, - String personRole, - LocalDate dateFrom, - LocalDate dateTo, - List keywords, - String rawQuery -) { -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaHealthClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaHealthClient.java deleted file mode 100644 index 9f1ad1d5..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaHealthClient.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.raddatz.familienarchiv.search; - -public interface OllamaHealthClient { - boolean isHealthy(); -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaProperties.java b/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaProperties.java deleted file mode 100644 index 673006e7..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/OllamaProperties.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Component -@ConfigurationProperties("app.ollama") -@Data -public class OllamaProperties { - private String baseUrl; - private String model; - private int timeoutSeconds = 30; - private int healthCheckTimeoutSeconds = 2; -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientOllamaClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientOllamaClient.java deleted file mode 100644 index 64f08554..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientOllamaClient.java +++ /dev/null @@ -1,184 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.extern.slf4j.Slf4j; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; -import org.springframework.http.client.JdkClientHttpRequestFactory; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestClient; -import org.springframework.web.client.RestClientException; - -import java.net.http.HttpClient; -import java.time.Duration; -import java.time.LocalDate; -import java.time.Year; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Service -@Slf4j -public class RestClientOllamaClient implements OllamaClient, OllamaHealthClient { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - private static final Set VALID_ROLES = Set.of("sender", "receiver", "any"); - private static final int MAX_NAME_LENGTH = 200; - private static final int MAX_KEYWORD_LENGTH = 100; - - private static final Map JSON_SCHEMA = Map.of( - "type", "object", - "required", List.of("personNames", "personRole", "keywords"), - "properties", Map.of( - "personNames", Map.of("type", "array", "items", Map.of("type", "string", "maxLength", MAX_NAME_LENGTH)), - "personRole", Map.of("type", "string", "enum", List.of("sender", "receiver", "any")), - "dateFrom", Map.of("type", List.of("string", "null"), "maxLength", 20), - "dateTo", Map.of("type", List.of("string", "null"), "maxLength", 20), - "keywords", Map.of("type", "array", "items", Map.of("type", "string", "maxLength", MAX_KEYWORD_LENGTH)) - ) - ); - - private final RestClient inferenceClient; - private final RestClient healthClient; - private final OllamaProperties props; - - public RestClientOllamaClient(OllamaProperties props) { - this.props = props; - - HttpClient inferenceHttp = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(Duration.ofSeconds(10)) - .build(); - JdkClientHttpRequestFactory inferenceFactory = new JdkClientHttpRequestFactory(inferenceHttp); - inferenceFactory.setReadTimeout(Duration.ofSeconds(props.getTimeoutSeconds())); - this.inferenceClient = RestClient.builder() - .baseUrl(props.getBaseUrl()) - .requestFactory(inferenceFactory) - .build(); - - HttpClient healthHttp = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(Duration.ofSeconds(props.getHealthCheckTimeoutSeconds())) - .build(); - JdkClientHttpRequestFactory healthFactory = new JdkClientHttpRequestFactory(healthHttp); - healthFactory.setReadTimeout(Duration.ofSeconds(props.getHealthCheckTimeoutSeconds())); - this.healthClient = RestClient.builder() - .baseUrl(props.getBaseUrl()) - .requestFactory(healthFactory) - .build(); - } - - @Override - public OllamaExtraction parse(String query) { - try { - OllamaGenerateRequest request = new OllamaGenerateRequest( - props.getModel(), query, JSON_SCHEMA, false); - String responseBody = inferenceClient.post() - .uri("/api/generate") - .contentType(org.springframework.http.MediaType.APPLICATION_JSON) - .body(request) - .retrieve() - .body(String.class); - return parseOllamaResponse(responseBody, query); - } catch (DomainException e) { - throw e; - } catch (Exception e) { - log.warn("Ollama inference failed: {}", e.getClass().getSimpleName()); - throw DomainException.serviceUnavailable(ErrorCode.SMART_SEARCH_UNAVAILABLE, - "Ollama unavailable: " + e.getClass().getSimpleName()); - } - } - - @Override - public boolean isHealthy() { - try { - healthClient.get().uri("/api/tags").retrieve().toBodilessEntity(); - return true; - } catch (Exception e) { - return false; - } - } - - private OllamaExtraction parseOllamaResponse(String responseBody, String rawQuery) { - try { - OllamaGenerateResponse response = MAPPER.readValue(responseBody, OllamaGenerateResponse.class); - String inner = response.response(); - if (inner == null || inner.isBlank()) { - return fallbackExtraction(rawQuery); - } - RawOllamaOutput raw = MAPPER.readValue(inner, RawOllamaOutput.class); - return toExtraction(raw, rawQuery); - } catch (Exception e) { - log.warn("Failed to parse Ollama response: {}", e.getClass().getSimpleName()); - throw DomainException.serviceUnavailable(ErrorCode.SMART_SEARCH_UNAVAILABLE, - "Failed to parse Ollama response: " + e.getClass().getSimpleName()); - } - } - - private OllamaExtraction toExtraction(RawOllamaOutput raw, String rawQuery) { - List names = raw.personNames() == null ? List.of() : raw.personNames().stream() - .filter(n -> n != null && n.length() <= MAX_NAME_LENGTH) - .toList(); - List keywords = raw.keywords() == null ? List.of() : raw.keywords().stream() - .filter(k -> k != null && k.length() <= MAX_KEYWORD_LENGTH) - .toList(); - String role = sanitiseRole(raw.personRole()); - LocalDate dateFrom = parseDate(raw.dateFrom(), true); - LocalDate dateTo = parseDate(raw.dateTo(), false); - return new OllamaExtraction(names, role, dateFrom, dateTo, keywords, rawQuery); - } - - private OllamaExtraction fallbackExtraction(String rawQuery) { - return new OllamaExtraction(List.of(), "any", null, null, List.of(), rawQuery); - } - - private String sanitiseRole(String role) { - if (role != null && VALID_ROLES.contains(role)) { - return role; - } - log.warn("Unexpected personRole from Ollama: {}", role); - return "any"; - } - - private LocalDate parseDate(String raw, boolean isFrom) { - if (raw == null || raw.isBlank()) return null; - try { - return LocalDate.parse(raw, DateTimeFormatter.ISO_LOCAL_DATE); - } catch (DateTimeParseException ignored) { - } - try { - int year = Integer.parseInt(raw.strip()); - if (year > 1000 && year < 3000) { - return isFrom ? Year.of(year).atDay(1) : Year.of(year).atMonth(12).atEndOfMonth(); - } - } catch (NumberFormatException ignored) { - } - return null; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - private record OllamaGenerateResponse(String response) { - } - - @JsonIgnoreProperties(ignoreUnknown = true) - private record RawOllamaOutput( - @JsonProperty("personNames") List personNames, - @JsonProperty("personRole") String personRole, - @JsonProperty("dateFrom") String dateFrom, - @JsonProperty("dateTo") String dateTo, - @JsonProperty("keywords") List keywords - ) { - } - - private record OllamaGenerateRequest( - String model, - String prompt, - Object format, - boolean stream - ) { - } -} -- 2.49.1 From 503ed6adef0c15fb71b0ca0e60f03551461e47f6 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:04:50 +0200 Subject: [PATCH 24/51] test(search): replace OllamaClient test suite with NlpClient equivalents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delete RestClientOllamaClientTest, add RestClientNlpClientTest: WireMock targets POST /parse; adds isHealthy_returnsFalse_whenPersonsLoadedIsZero - NlQueryParserServiceTest: @Mock NlpClient; all stubs updated to parse(String,String); NlpExtraction throughout; service.search(..., "de", PAGE); adds verify(nlpClient).parse(eq,eq) - NlSearchControllerTest: add lang:"de" to all request bodies; stubs use anyString×3; rename search_returns503_whenOllamaUnavailable → search_returns503_whenNlpServiceUnavailable Co-Authored-By: Claude Sonnet 4.6 --- .../search/NlQueryParserServiceTest.java | 244 ++++++++---------- .../search/NlSearchControllerTest.java | 32 +-- .../search/RestClientNlpClientTest.java | 124 +++++++++ 3 files changed, 248 insertions(+), 152 deletions(-) create mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/RestClientNlpClientTest.java diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlQueryParserServiceTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlQueryParserServiceTest.java index 61c00b6a..76f8e88c 100644 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/NlQueryParserServiceTest.java +++ b/backend/src/test/java/org/raddatz/familienarchiv/search/NlQueryParserServiceTest.java @@ -33,7 +33,7 @@ import static org.mockito.Mockito.*; class NlQueryParserServiceTest { - @Mock OllamaClient ollamaClient; + @Mock NlpClient nlpClient; @Mock PersonService personService; @Mock DocumentService documentService; @Mock TagService tagService; @@ -45,7 +45,7 @@ class NlQueryParserServiceTest { @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); - service = new NlQueryParserService(ollamaClient, personService, documentService, tagService); + service = new NlQueryParserService(nlpClient, personService, documentService, tagService); when(documentService.searchDocuments(any(), any(), any(), any())) .thenReturn(DocumentSearchResult.of(List.of())); when(documentService.searchDocumentsByPersonId(any(), any(), any(), any())) @@ -55,10 +55,10 @@ class NlQueryParserServiceTest { // --- Factory helpers --- - private OllamaExtraction extraction(List names, String role, LocalDate from, LocalDate to, - List keywords) { + private NlpExtraction extraction(List names, String role, LocalDate from, LocalDate to, + List keywords) { String raw = names.isEmpty() ? "test query" : String.join(" ", names); - return new OllamaExtraction(names, role, from, to, keywords, raw); + return new NlpExtraction(names, role, from, to, keywords, raw); } private Person person(UUID id, String firstName, String lastName) { @@ -86,12 +86,13 @@ class NlQueryParserServiceTest { @Test void search_resolvesSingleName_asSender() { Person walter = person(P1, "Walter", "Raddatz"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Walter"), "sender", null, null, List.of())); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - NlSearchResponse resp = service.search("Was hat Walter geschrieben?", PAGE); + NlSearchResponse resp = service.search("Was hat Walter geschrieben?", "de", PAGE); + verify(nlpClient).parse(eq("Was hat Walter geschrieben?"), eq("de")); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), eq(DocumentSort.DATE), eq("desc"), eq(PAGE)); assertThat(cap.getValue().sender()).isEqualTo(P1); @@ -107,11 +108,11 @@ class NlQueryParserServiceTest { void search_multiMatchName_populatesAmbiguous_andSkipsSearch() { Person a = person(UUID.randomUUID(), "Walter", "Braun"); Person b = person(UUID.randomUUID(), "Walter", "Schmidt"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Walter"), "sender", null, null, List.of())); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(a, b))); - NlSearchResponse resp = service.search("Briefe von Walter", PAGE); + NlSearchResponse resp = service.search("Briefe von Walter", "de", PAGE); verify(documentService, never()).searchDocuments(any(), any(), any(), any()); verify(documentService, never()).searchDocumentsByPersonId(any(), any(), any(), any()); @@ -125,11 +126,11 @@ class NlQueryParserServiceTest { void search_multiMatchName_withPersonRoleAny_stillSkipsSearch() { Person a = person(UUID.randomUUID(), "Emma", "Braun"); Person b = person(UUID.randomUUID(), "Emma", "Raddatz"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Emma"), "any", null, null, List.of())); when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(a, b))); - NlSearchResponse resp = service.search("Briefe an Emma", PAGE); + NlSearchResponse resp = service.search("Briefe an Emma", "de", PAGE); verify(documentService, never()).searchDocuments(any(), any(), any(), any()); verify(documentService, never()).searchDocumentsByPersonId(any(), any(), any(), any()); @@ -140,11 +141,11 @@ class NlQueryParserServiceTest { @Test void search_noMatchName_isFoldedIntoText() { - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Karl"), "any", null, null, List.of())); when(personService.resolveByName("Karl")).thenReturn(makeNameMatches()); - service.search("Briefe von Karl", PAGE); + service.search("Briefe von Karl", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -158,11 +159,11 @@ class NlQueryParserServiceTest { @Test void search_personRoleAny_singleMatch_callsSearchDocumentsByPersonId() { Person walter = person(P1, "Walter", "Raddatz"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Walter"), "any", null, null, List.of())); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - NlSearchResponse resp = service.search("Briefe von Walter", PAGE); + NlSearchResponse resp = service.search("Briefe von Walter", "de", PAGE); verify(documentService).searchDocumentsByPersonId(eq(P1), isNull(), isNull(), eq(PAGE)); verify(documentService, never()).searchDocuments(any(), any(), any(), any()); @@ -175,12 +176,12 @@ class NlQueryParserServiceTest { void search_twoNamesResolve_assignsSenderAndReceiver() { Person walter = person(P1, "Walter", "Raddatz"); Person emma = person(P2, "Emma", "Raddatz"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Walter", "Emma"), "any", null, null, List.of())); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma))); - NlSearchResponse resp = service.search("Briefe von Walter an Emma", PAGE); + NlSearchResponse resp = service.search("Briefe von Walter an Emma", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), eq(DocumentSort.DATE), eq("desc"), eq(PAGE)); @@ -197,12 +198,12 @@ class NlQueryParserServiceTest { Person walter = person(P1, "Walter", "Raddatz"); Person emma1 = person(P2, "Emma", "Braun"); Person emma2 = person(P3, "Emma", "Schmidt"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Walter", "Emma"), "sender", null, null, List.of())); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma1, emma2))); - NlSearchResponse resp = service.search("Briefe von Walter an Emma", PAGE); + NlSearchResponse resp = service.search("Briefe von Walter an Emma", "de", PAGE); verify(documentService, never()).searchDocuments(any(), any(), any(), any()); assertThat(resp.interpretation().ambiguousPersons()).hasSize(2); @@ -213,12 +214,12 @@ class NlQueryParserServiceTest { @Test void search_twoNames_firstNoMatch_secondResolved_foldFirstIntoText() { Person emma = person(P2, "Emma", "Raddatz"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Karl", "Emma"), "sender", null, null, List.of())); when(personService.resolveByName("Karl")).thenReturn(makeNameMatches()); when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma))); - service.search("Briefe von Karl an Emma", PAGE); + service.search("Briefe von Karl an Emma", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -233,13 +234,13 @@ class NlQueryParserServiceTest { Person walter = person(P1, "Walter", "Raddatz"); Person emma = person(P2, "Emma", "Raddatz"); Person heinrich = person(P3, "Heinrich", "Braun"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Walter", "Emma", "Heinrich"), "any", null, null, List.of())); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma))); when(personService.resolveByName("Heinrich")).thenReturn(makeNameMatches(List.of(heinrich))); - service.search("Briefe von Walter an Emma über Heinrich", PAGE); + service.search("Briefe von Walter an Emma über Heinrich", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -252,10 +253,10 @@ class NlQueryParserServiceTest { @Test void search_keywords_areJoinedIntoText() { - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Krieg", "Walter"))); - service.search("Dokumente über den Krieg Walter", PAGE); + service.search("Dokumente über den Krieg Walter", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -268,10 +269,10 @@ class NlQueryParserServiceTest { void search_dateRange_passedIntoSearchFilters() { LocalDate from = LocalDate.of(1914, 1, 1); LocalDate to = LocalDate.of(1914, 12, 31); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", from, to, List.of())); - service.search("Briefe aus dem Jahr 1914", PAGE); + service.search("Briefe aus dem Jahr 1914", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -283,10 +284,10 @@ class NlQueryParserServiceTest { @Test void search_nullDates_passedAsNullIntoFilters() { - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); - service.search("Hochzeitsbriefe", PAGE); + service.search("Hochzeitsbriefe", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -294,104 +295,75 @@ class NlQueryParserServiceTest { assertThat(cap.getValue().to()).isNull(); } - // --- 13. Query under 3 chars → VALIDATION_ERROR before Ollama call --- + // --- 13. NLP service returns empty names/keywords → raw query used as keyword fallback --- @Test - void search_queryTooShort_throwsValidationError() { - assertThatThrownBy(() -> service.search("ab", PAGE)) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.VALIDATION_ERROR); - - verify(ollamaClient, never()).parse(anyString()); - } - - // --- 14. Query over 500 chars → VALIDATION_ERROR --- - - @Test - void search_queryTooLong_throwsValidationError() { - String longQuery = "a".repeat(501); - assertThatThrownBy(() -> service.search(longQuery, PAGE)) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.VALIDATION_ERROR); - - verify(ollamaClient, never()).parse(anyString()); - } - - // --- 15. Ollama returns empty names/keywords → raw query used as keyword fallback --- - - @Test - void search_ollamaReturnsEmpty_usesRawQueryAsTextFallback() { + void search_nlpReturnsEmpty_usesRawQueryAsTextFallback() { String raw = "Briefe aus dem Krieg"; - when(ollamaClient.parse(anyString())) - .thenReturn(new OllamaExtraction(List.of(), "any", null, null, List.of(), raw)); + when(nlpClient.parse(anyString(), anyString())) + .thenReturn(new NlpExtraction(List.of(), "any", null, null, List.of(), raw)); - service.search(raw, PAGE); + service.search(raw, "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); assertThat(cap.getValue().text()).isEqualTo(raw); } - // --- 16. Null personNames/keywords from Ollama → no NPE --- + // --- 14. Null personNames/keywords → no NPE --- @Test void search_nullPersonNamesAndKeywords_handledWithoutNpe() { - OllamaExtraction ext = new OllamaExtraction(null, "any", null, null, null, "test query"); - when(ollamaClient.parse(anyString())).thenReturn(ext); + NlpExtraction ext = new NlpExtraction(null, "any", null, null, null, "test query"); + when(nlpClient.parse(anyString(), anyString())).thenReturn(ext); - NlSearchResponse resp = service.search("test query", PAGE); + NlSearchResponse resp = service.search("test query", "de", PAGE); assertThat(resp).isNotNull(); verify(documentService).searchDocuments(any(), any(), any(), any()); } - // --- 17. Unrecognized personRole → defaults to any-like behavior (no crash) --- + // --- 15. Unrecognized personRole → defaults to any-like behavior (no crash) --- @Test void search_unrecognizedPersonRole_treatedLikeAny_withSingleResolvedPerson() { Person walter = person(P1, "Walter", "Raddatz"); - // OllamaClient defensive parsing returns "any" for unknown roles, - // but NlQueryParserService must also be safe if something unexpected arrives. - when(ollamaClient.parse(anyString())) - .thenReturn(new OllamaExtraction(List.of("Walter"), "unknown_role", null, null, List.of(), "query")); + when(nlpClient.parse(anyString(), anyString())) + .thenReturn(new NlpExtraction(List.of("Walter"), "unknown_role", null, null, List.of(), "query")); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - NlSearchResponse resp = service.search("Briefe von Walter", PAGE); + NlSearchResponse resp = service.search("Briefe von Walter", "de", PAGE); - // Should not crash; "unknown_role" treated as fallback (neither sender nor receiver → any) assertThat(resp).isNotNull(); } - // --- 18. Ollama throws SMART_SEARCH_UNAVAILABLE → propagates to caller --- + // --- 16. NLP service throws SMART_SEARCH_UNAVAILABLE → propagates to caller --- @Test - void search_ollamaThrowsUnavailable_propagates() { - when(ollamaClient.parse(anyString())) + void search_nlpThrowsUnavailable_propagates() { + when(nlpClient.parse(anyString(), anyString())) .thenThrow(DomainException.tooManyRequests(ErrorCode.SMART_SEARCH_UNAVAILABLE, "offline")); - assertThatThrownBy(() -> service.search("Was hat Walter geschrieben?", PAGE)) + assertThatThrownBy(() -> service.search("Was hat Walter geschrieben?", "de", PAGE)) .isInstanceOf(DomainException.class) .extracting(e -> ((DomainException) e).getCode()) .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); } - // --- 19. LLM-extracted name > 200 chars → skipped, PersonService never called --- + // --- 17. LLM-extracted name > 200 chars → skipped, PersonService never called --- @Test void search_nameLongerThan200Chars_isSkippedBeforePersonServiceCall() { String longName = "A".repeat(201); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(longName), "sender", null, null, List.of())); - service.search("Briefe von sehr langem Namen", PAGE); + service.search("Briefe von sehr langem Namen", "de", PAGE); verify(personService, never()).resolveByName(anyString()); } - // --- 20. Cap lives in resolveByName (after classification): a pre-capped 10-direct result - // maps straight to ambiguousPersons; the search layer adds no second cap. --- + // --- 18. Cap: 10 direct matches → all shown as ambiguous --- @Test void search_tenDirectMatches_allShownAsAmbiguous() { @@ -399,24 +371,24 @@ class NlQueryParserServiceTest { for (int i = 0; i < 10; i++) { ten.add(person(UUID.randomUUID(), "Walter", "Person" + i)); } - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Walter"), "sender", null, null, List.of())); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(ten)); - NlSearchResponse resp = service.search("Briefe von Walter", PAGE); + NlSearchResponse resp = service.search("Briefe von Walter", "de", PAGE); assertThat(resp.interpretation().ambiguousPersons()).hasSize(10); verify(documentService, never()).searchDocuments(any(), any(), any(), any()); } - // --- 21. SearchFilters defaults: tagOperator=AND, status=null, undated=false, tags=empty --- + // --- 19. SearchFilters defaults: tagOperator=AND, status=null, undated=false, tags=empty --- @Test void search_searchFiltersDefaults_areCorrect() { - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Krieg"))); - service.search("Dokumente über den Krieg", PAGE); + service.search("Dokumente über den Krieg", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), eq(DocumentSort.DATE), eq("desc"), eq(PAGE)); @@ -428,16 +400,16 @@ class NlQueryParserServiceTest { assertThat(f.tagQ()).isNull(); } - // --- 22. personRole=receiver + 1 resolved → receiver UUID set --- + // --- 20. personRole=receiver + 1 resolved → receiver UUID set --- @Test void search_personRoleReceiver_singleMatch_setsReceiver() { Person emma = person(P2, "Emma", "Raddatz"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Emma"), "receiver", null, null, List.of())); when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma))); - service.search("Briefe an Emma", PAGE); + service.search("Briefe an Emma", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -445,59 +417,59 @@ class NlQueryParserServiceTest { assertThat(cap.getValue().sender()).isNull(); } - // --- 23. keywordsApplied=true when text is non-blank --- + // --- 21. keywordsApplied=true when text is non-blank --- @Test void search_keywordsApplied_trueWhenTextNonBlank() { - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Feldpost"))); - NlSearchResponse resp = service.search("Feldpost aus dem Krieg", PAGE); + NlSearchResponse resp = service.search("Feldpost aus dem Krieg", "de", PAGE); assertThat(resp.interpretation().keywordsApplied()).isTrue(); } - // --- 23a. Partial-only, one candidate → ambiguous (1-item picker), search skipped --- + // --- 22. Partial-only, one candidate → ambiguous (1-item picker), search skipped --- @Test void search_partialOnly_oneCandidate_populatesAmbiguous() { Person cramer = person(P1, "Clara", "Cramer"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Clara Cram"), "any", null, null, List.of())); when(personService.resolveByName("Clara Cram")).thenReturn(makeNameMatches(List.of(), List.of(cramer))); - NlSearchResponse resp = service.search("Briefe von Clara Cram", PAGE); + NlSearchResponse resp = service.search("Briefe von Clara Cram", "de", PAGE); assertThat(resp.interpretation().ambiguousPersons()).hasSize(1); verify(documentService, never()).searchDocuments(any(), any(), any(), any()); } - // --- 23b. Partial-only, two candidates → ambiguous (multi-item picker) --- + // --- 23. Partial-only, two candidates → ambiguous (multi-item picker) --- @Test void search_partialOnly_twoCandidates_populatesAmbiguous() { Person cramer = person(P1, "Clara", "Cramer"); Person crammond = person(P2, "Clara", "Crammond"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Clara Cram"), "any", null, null, List.of())); when(personService.resolveByName("Clara Cram")) .thenReturn(makeNameMatches(List.of(), List.of(cramer, crammond))); - NlSearchResponse resp = service.search("Briefe von Clara Cram", PAGE); + NlSearchResponse resp = service.search("Briefe von Clara Cram", "de", PAGE); assertThat(resp.interpretation().ambiguousPersons()).hasSize(2); } - // --- 23c. Exactly one direct match → search executes, no picker --- + // --- 24. Exactly one direct match → search executes, no picker --- @Test void search_oneDirect_executesSearch() { Person clara = person(P1, "Clara", "Cram"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Clara Cram"), "any", null, null, List.of())); when(personService.resolveByName("Clara Cram")).thenReturn(makeNameMatches(List.of(clara))); - NlSearchResponse resp = service.search("Briefe von Clara Cram", PAGE); + NlSearchResponse resp = service.search("Briefe von Clara Cram", "de", PAGE); verify(documentService).searchDocumentsByPersonId(eq(P1), isNull(), isNull(), eq(PAGE)); assertThat(resp.interpretation().ambiguousPersons()).isEmpty(); @@ -519,16 +491,16 @@ class NlQueryParserServiceTest { private static final UUID T1 = UUID.fromString("00000000-0000-0000-0001-000000000001"); - // --- 24. Single keyword resolves to one tag → tag filter applied --- + // --- 25. Single keyword resolves to one tag → tag filter applied --- @Test void search_singleKeywordResolvesToTag_appliesTagFilter() { Tag hochzeit = tag(T1, "Hochzeit"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - NlSearchResponse resp = service.search("Briefe über Hochzeit", PAGE); + NlSearchResponse resp = service.search("Briefe über Hochzeit", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -542,17 +514,17 @@ class NlQueryParserServiceTest { private static final UUID T2 = UUID.fromString("00000000-0000-0000-0001-000000000002"); - // --- 25. Keyword matches multiple tags → all in resolvedTags, OR-union --- + // --- 26. Keyword matches multiple tags → all in resolvedTags, OR-union --- @Test void search_keywordMatchesMultipleTags_allIncluded() { Tag hochzeit1 = tag(T1, "Hochzeit Raddatz"); Tag hochzeit2 = tag(T2, "Hochzeit Braun"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit1, hochzeit2)); - NlSearchResponse resp = service.search("Briefe über Hochzeit", PAGE); + NlSearchResponse resp = service.search("Briefe über Hochzeit", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -561,14 +533,14 @@ class NlQueryParserServiceTest { assertThat(resp.interpretation().resolvedTags()).hasSize(2); } - // --- 26. Keyword no tag match → stays as FTS text, resolvedTags empty --- + // --- 27. Keyword no tag match → stays as FTS text, resolvedTags empty --- @Test void search_keywordNoTagMatch_staysAsFtsText() { - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Feldpost"))); - NlSearchResponse resp = service.search("Feldpost Briefe", PAGE); + NlSearchResponse resp = service.search("Feldpost Briefe", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -578,16 +550,16 @@ class NlQueryParserServiceTest { assertThat(resp.interpretation().tagsApplied()).isFalse(); } - // --- 27. Mixed: one keyword resolves, one doesn't → tag filter + FTS text --- + // --- 28. Mixed: one keyword resolves, one doesn't → tag filter + FTS text --- @Test void search_mixedKeywords_oneResolves_oneStaysAsText() { Tag hochzeit = tag(T1, "Hochzeit"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit", "Feldpost"))); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - NlSearchResponse resp = service.search("Hochzeit und Feldpost", PAGE); + NlSearchResponse resp = service.search("Hochzeit und Feldpost", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -598,18 +570,18 @@ class NlQueryParserServiceTest { assertThat(resp.interpretation().tagsApplied()).isTrue(); } - // --- 28. personRole=any + 1 person + resolvable keyword → personId search, tagsApplied=false --- + // --- 29. personRole=any + 1 person + resolvable keyword → personId search, tagsApplied=false --- @Test void search_personRoleAny_singlePerson_resolvableKeyword_tagsAppliedFalse() { Person walter = person(P1, "Walter", "Raddatz"); Tag hochzeit = tag(T1, "Hochzeit"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of("Walter"), "any", null, null, List.of("Hochzeit"))); when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - NlSearchResponse resp = service.search("Briefe von Walter über Hochzeit", PAGE); + NlSearchResponse resp = service.search("Briefe von Walter über Hochzeit", "de", PAGE); verify(documentService).searchDocumentsByPersonId(eq(P1), isNull(), isNull(), eq(PAGE)); verify(documentService, never()).searchDocuments(any(), any(), any(), any()); @@ -618,7 +590,7 @@ class NlQueryParserServiceTest { assertThat(resp.interpretation().resolvedTags().get(0).name()).isEqualTo("Hochzeit"); } - // --- 29. Cap: keyword matches > 10 tags → capped at 10 --- + // --- 30. Cap: keyword matches > 10 tags → capped at 10 --- @Test void search_keywordMatchesMoreThanMaxTags_cappedAtTen() { @@ -626,11 +598,11 @@ class NlQueryParserServiceTest { for (int i = 0; i < 11; i++) { eleven.add(tag(UUID.randomUUID(), "Thema " + i)); } - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Thema"))); when(tagService.findByNameContaining("Thema")).thenReturn(eleven); - NlSearchResponse resp = service.search("Dokumente zum Thema", PAGE); + NlSearchResponse resp = service.search("Dokumente zum Thema", "de", PAGE); assertThat(resp.interpretation().resolvedTags()).hasSize(10); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); @@ -638,14 +610,14 @@ class NlQueryParserServiceTest { assertThat(cap.getValue().tags()).hasSize(10); } - // --- 30. Short keyword (< 3 chars) → skipped, not passed to TagService --- + // --- 31. Short keyword (< 3 chars) → skipped, not passed to TagService --- @Test void search_shortKeyword_skippedByTagResolution() { - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("ab", "Krieg"))); - service.search("ab Krieg", PAGE); + service.search("ab Krieg", "de", PAGE); verify(tagService, never()).findByNameContaining("ab"); verify(tagService).findByNameContaining("Krieg"); @@ -654,17 +626,17 @@ class NlQueryParserServiceTest { assertThat(cap.getValue().text()).contains("ab"); } - // --- 31. Dedup: same tag matched by two keywords → appears once --- + // --- 32. Dedup: same tag matched by two keywords → appears once --- @Test void search_sameTagMatchedByTwoKeywords_deduplicatedToOne() { Tag hochzeit = tag(T1, "Hochzeit"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit", "hoch"))); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); when(tagService.findByNameContaining("hoch")).thenReturn(List.of(hochzeit)); - NlSearchResponse resp = service.search("Hochzeit hoch", PAGE); + NlSearchResponse resp = service.search("Hochzeit hoch", "de", PAGE); assertThat(resp.interpretation().resolvedTags()).hasSize(1); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); @@ -672,16 +644,16 @@ class NlQueryParserServiceTest { assertThat(cap.getValue().tags()).hasSize(1); } - // --- 32. All keywords resolve → rawQuery fallback suppressed, text=null --- + // --- 33. All keywords resolve → rawQuery fallback suppressed, text=null --- @Test void search_allKeywordsResolveToTags_rawQueryFallbackSuppressed() { Tag hochzeit = tag(T1, "Hochzeit"); - when(ollamaClient.parse(anyString())) - .thenReturn(new OllamaExtraction(List.of(), "any", null, null, List.of("Hochzeit"), "raw query text")); + when(nlpClient.parse(anyString(), anyString())) + .thenReturn(new NlpExtraction(List.of(), "any", null, null, List.of("Hochzeit"), "raw query text")); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - NlSearchResponse resp = service.search("Hochzeit", PAGE); + NlSearchResponse resp = service.search("Hochzeit", "de", PAGE); ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); @@ -689,22 +661,22 @@ class NlQueryParserServiceTest { assertThat(cap.getValue().tags()).containsExactly("Hochzeit"); } - // --- 33. Flag independence: keywordsApplied=false AND tagsApplied=true --- + // --- 34. Flag independence: keywordsApplied=false AND tagsApplied=true --- @Test void search_allKeywordsResolveToTags_keywordsAppliedFalse_tagsAppliedTrue() { Tag hochzeit = tag(T1, "Hochzeit"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - NlSearchResponse resp = service.search("Hochzeit Briefe", PAGE); + NlSearchResponse resp = service.search("Hochzeit Briefe", "de", PAGE); assertThat(resp.interpretation().keywordsApplied()).isFalse(); assertThat(resp.interpretation().tagsApplied()).isTrue(); } - // --- 34. Color carried through from resolveEffectiveColors --- + // --- 35. Color carried through from resolveEffectiveColors --- @Test void search_tagHint_carriesColorSetByResolveEffectiveColors() { @@ -714,25 +686,25 @@ class NlQueryParserServiceTest { tags.forEach(t -> t.setColor("sage")); return null; }).when(tagService).resolveEffectiveColors(any()); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - NlSearchResponse resp = service.search("Hochzeit", PAGE); + NlSearchResponse resp = service.search("Hochzeit", "de", PAGE); assertThat(resp.interpretation().resolvedTags().get(0).color()).isEqualTo("sage"); } - // --- 35. Color stays null when resolveEffectiveColors leaves it unset --- + // --- 36. Color stays null when resolveEffectiveColors leaves it unset --- @Test void search_tagHint_colorIsNull_whenNoColorResolved() { Tag hochzeit = tag(T1, "Hochzeit"); - when(ollamaClient.parse(anyString())) + when(nlpClient.parse(anyString(), anyString())) .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - NlSearchResponse resp = service.search("Hochzeit", PAGE); + NlSearchResponse resp = service.search("Hochzeit", "de", PAGE); assertThat(resp.interpretation().resolvedTags().get(0).color()).isNull(); } diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java index c0d30f40..51600aa2 100644 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java +++ b/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java @@ -57,11 +57,11 @@ class NlSearchControllerTest { @Test @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) void search_returns200_withNlSearchResponse() throws Exception { - when(nlQueryParserService.search(anyString(), any())).thenReturn(makeResponse()); + when(nlQueryParserService.search(anyString(), anyString(), any())).thenReturn(makeResponse()); mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter im Krieg\"}")) + .content("{\"query\":\"Briefe von Walter im Krieg\",\"lang\":\"de\"}")) .andExpect(status().isOk()) .andExpect(jsonPath("$.interpretation.rawQuery").value("Briefe von Walter im Krieg")) .andExpect(jsonPath("$.interpretation.resolvedPersons[0].displayName").value("Walter Raddatz")) @@ -79,11 +79,11 @@ class NlSearchControllerTest { List.of(), List.of(a, b), null, null, List.of(), List.of(), "Briefe von Walter", false, false); NlSearchResponse resp = new NlSearchResponse(DocumentSearchResult.of(List.of()), interp); - when(nlQueryParserService.search(anyString(), any())).thenReturn(resp); + when(nlQueryParserService.search(anyString(), anyString(), any())).thenReturn(resp); mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\"}")) + .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) .andExpect(status().isOk()) .andExpect(jsonPath("$.interpretation.ambiguousPersons").isArray()) .andExpect(jsonPath("$.interpretation.ambiguousPersons[0].displayName").value("Walter Braun")) @@ -96,7 +96,7 @@ class NlSearchControllerTest { void search_returns401_whenUnauthenticated() throws Exception { mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\"}")) + .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) .andExpect(status().isUnauthorized()); } @@ -107,7 +107,7 @@ class NlSearchControllerTest { void search_returns400_whenQueryTooShort() throws Exception { mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"ab\"}")) + .content("{\"query\":\"ab\",\"lang\":\"de\"}")) .andExpect(status().isBadRequest()); } @@ -119,42 +119,42 @@ class NlSearchControllerTest { String longQuery = "a".repeat(501); mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"" + longQuery + "\"}")) + .content("{\"query\":\"" + longQuery + "\",\"lang\":\"de\"}")) .andExpect(status().isBadRequest()); } - // --- 6. Ollama unavailable → 503 --- + // --- 6. NLP service unavailable → 503 --- @Test @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) - void search_returns503_whenOllamaUnavailable() throws Exception { - when(nlQueryParserService.search(anyString(), any())) - .thenThrow(DomainException.serviceUnavailable(ErrorCode.SMART_SEARCH_UNAVAILABLE, "Ollama offline")); + void search_returns503_whenNlpServiceUnavailable() throws Exception { + when(nlQueryParserService.search(anyString(), anyString(), any())) + .thenThrow(DomainException.serviceUnavailable(ErrorCode.SMART_SEARCH_UNAVAILABLE, "NLP service offline")); mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\"}")) + .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) .andExpect(status().isServiceUnavailable()) .andExpect(jsonPath("$.code").value("SMART_SEARCH_UNAVAILABLE")); } - // --- 7. 6th request in 1 minute → 429 --- + // --- 7. 6th request in 1 minute → 429 (rate limit = 5/min default) --- @Test @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) void search_returns429_onSixthRequestWithinRateLimit() throws Exception { - when(nlQueryParserService.search(anyString(), any())).thenReturn(makeResponse()); + when(nlQueryParserService.search(anyString(), anyString(), any())).thenReturn(makeResponse()); for (int i = 0; i < 5; i++) { mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\"}")) + .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) .andExpect(status().isOk()); } mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\"}")) + .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) .andExpect(status().isTooManyRequests()) .andExpect(jsonPath("$.code").value("SMART_SEARCH_RATE_LIMITED")); } diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientNlpClientTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientNlpClientTest.java new file mode 100644 index 00000000..0198f6c0 --- /dev/null +++ b/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientNlpClientTest.java @@ -0,0 +1,124 @@ +package org.raddatz.familienarchiv.search; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.raddatz.familienarchiv.exception.DomainException; +import org.raddatz.familienarchiv.exception.ErrorCode; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class RestClientNlpClientTest { + + private WireMockServer wireMock; + private RestClientNlpClient client; + + @BeforeEach + void setUp() { + wireMock = new WireMockServer(WireMockConfiguration.wireMockConfig().dynamicPort()); + wireMock.start(); + + NlpProperties props = new NlpProperties(); + props.setBaseUrl("http://localhost:" + wireMock.port()); + props.setTimeoutSeconds(5); + props.setHealthCheckTimeoutSeconds(2); + + client = new RestClientNlpClient(props); + } + + @AfterEach + void tearDown() { + wireMock.stop(); + } + + private String makeParseResponseJson(String personNamesJson, String personRole, + String dateFrom, String dateTo, String keywordsJson, + String rawQuery) { + return String.format( + "{\"personNames\":%s,\"personRole\":\"%s\",\"dateFrom\":%s,\"dateTo\":%s,\"keywords\":%s,\"rawQuery\":\"%s\"}", + personNamesJson, personRole, + dateFrom == null ? "null" : "\"" + dateFrom + "\"", + dateTo == null ? "null" : "\"" + dateTo + "\"", + keywordsJson, rawQuery + ); + } + + @Test + void parse_returnsExtraction_whenNlpServiceReturnsValidJson() { + String body = makeParseResponseJson("[\"Walter\"]", "sender", "1914-01-01", "1914-12-31", + "[\"Krieg\"]", "Briefe von Walter im Krieg"); + wireMock.stubFor(post(urlEqualTo("/parse")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(body))); + + NlpExtraction result = client.parse("Briefe von Walter im Krieg", "de"); + + assertThat(result.personNames()).containsExactly("Walter"); + assertThat(result.personRole()).isEqualTo("sender"); + assertThat(result.keywords()).containsExactly("Krieg"); + assertThat(result.dateFrom()).isNotNull(); + assertThat(result.dateTo()).isNotNull(); + } + + @Test + void parse_throwsSmartSearchUnavailable_whenNlpServiceReturns500() { + wireMock.stubFor(post(urlEqualTo("/parse")) + .willReturn(aResponse().withStatus(500))); + + assertThatThrownBy(() -> client.parse("some query", "de")) + .isInstanceOf(DomainException.class) + .extracting(e -> ((DomainException) e).getCode()) + .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); + } + + @Test + void parse_throwsSmartSearchUnavailable_whenNlpServiceExceedsTimeout() { + wireMock.stubFor(post(urlEqualTo("/parse")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withFixedDelay(6000) + .withBody("{\"personNames\":[],\"personRole\":\"any\",\"dateFrom\":null,\"dateTo\":null,\"keywords\":[],\"rawQuery\":\"q\"}"))); + + assertThatThrownBy(() -> client.parse("some query", "de")) + .isInstanceOf(DomainException.class) + .extracting(e -> ((DomainException) e).getCode()) + .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); + } + + @Test + void isHealthy_returnsTrue_whenPersonsLoadedIsPositive() { + wireMock.stubFor(get(urlEqualTo("/health")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{\"status\":\"ok\",\"persons_loaded\":42}"))); + + assertThat(client.isHealthy()).isTrue(); + } + + @Test + void isHealthy_returnsFalse_whenPersonsLoadedIsZero() { + wireMock.stubFor(get(urlEqualTo("/health")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{\"status\":\"ok\",\"persons_loaded\":0}"))); + + assertThat(client.isHealthy()).isFalse(); + } + + @Test + void isHealthy_returnsFalse_whenNlpServiceIsDown() { + wireMock.stubFor(get(urlEqualTo("/health")) + .willReturn(aResponse().withStatus(503))); + + assertThat(client.isHealthy()).isFalse(); + } +} -- 2.49.1 From 08c11e567cdc73dc9e53272ee1c2c7e5f2cc6911 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:06:04 +0200 Subject: [PATCH 25/51] feat(search): raise NL search rate limit from 5 to 20 req/min The rule-based NLP service is <100ms vs Ollama's ~15s, making the old limit too restrictive for normal interactive use. Co-Authored-By: Claude Sonnet 4.6 --- .../familienarchiv/search/NlSearchRateLimitProperties.java | 2 +- backend/src/main/resources/application.yaml | 2 +- .../familienarchiv/search/NlSearchControllerTest.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimitProperties.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimitProperties.java index e71f8a36..3be4b3b0 100644 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimitProperties.java +++ b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimitProperties.java @@ -8,5 +8,5 @@ import org.springframework.stereotype.Component; @ConfigurationProperties("app.nl-search.rate-limit") @Data public class NlSearchRateLimitProperties { - private int maxRequestsPerMinute = 5; + private int maxRequestsPerMinute = 20; } diff --git a/backend/src/main/resources/application.yaml b/backend/src/main/resources/application.yaml index a0054de6..c4756780 100644 --- a/backend/src/main/resources/application.yaml +++ b/backend/src/main/resources/application.yaml @@ -135,7 +135,7 @@ app: nl-search: rate-limit: - max-requests-per-minute: 5 + max-requests-per-minute: 20 ocr: sender-model: diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java index 51600aa2..8630a2cb 100644 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java +++ b/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java @@ -138,14 +138,14 @@ class NlSearchControllerTest { .andExpect(jsonPath("$.code").value("SMART_SEARCH_UNAVAILABLE")); } - // --- 7. 6th request in 1 minute → 429 (rate limit = 5/min default) --- + // --- 7. 21st request in 1 minute → 429 (rate limit = 20/min default) --- @Test @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) - void search_returns429_onSixthRequestWithinRateLimit() throws Exception { + void search_returns429_on21stRequestWithinRateLimit() throws Exception { when(nlQueryParserService.search(anyString(), anyString(), any())).thenReturn(makeResponse()); - for (int i = 0; i < 5; i++) { + for (int i = 0; i < 20; i++) { mockMvc.perform(post("/api/search/nl").with(csrf()) .contentType(MediaType.APPLICATION_JSON) .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) -- 2.49.1 From 4a23dfff8e09251cc6575d8a76bbe6181a433069 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:08:57 +0200 Subject: [PATCH 26/51] test(search): assert lang field sent in E2E NL search request Co-Authored-By: Claude Sonnet 4.6 --- frontend/e2e/nl-search.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/e2e/nl-search.spec.ts b/frontend/e2e/nl-search.spec.ts index 210ad173..2bebecee 100644 --- a/frontend/e2e/nl-search.spec.ts +++ b/frontend/e2e/nl-search.spec.ts @@ -37,6 +37,8 @@ test.describe('NL (smart) search — happy path', () => { }) => { // Deliberate delay so the loading state is assertable before the response arrives. await page.route('**/api/search/nl', async (route) => { + const body = route.request().postDataJSON(); + expect(body.lang).toBeTruthy(); await new Promise((resolve) => setTimeout(resolve, 150)); await route.fulfill({ status: 200, -- 2.49.1 From ab10daf325159ef9cd29839b83f0eb439bcb1a2b Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:10:32 +0200 Subject: [PATCH 27/51] chore(i18n): remove AI/KI/IA and timing refs from smart search strings Co-Authored-By: Claude Sonnet 4.6 --- frontend/messages/de.json | 6 +++--- frontend/messages/en.json | 6 +++--- frontend/messages/es.json | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/messages/de.json b/frontend/messages/de.json index e53f9583..d8afe368 100644 --- a/frontend/messages/de.json +++ b/frontend/messages/de.json @@ -25,14 +25,14 @@ "error_smart_search_unavailable": "Die intelligente Suche ist momentan nicht verfügbar. Bitte nutzen Sie die normale Suche.", "error_smart_search_rate_limited": "Sie haben die Suchfunktion zu häufig genutzt. Bitte warten Sie eine Minute.", "smart_search_keywords_not_applied": "Schlüsselwörter konnten bei dieser Suche nicht berücksichtigt werden.", - "search_toggle_smart_label": "KI", + "search_toggle_smart_label": "Smart", "search_toggle_smart_label_suffix": "-Suche", "search_toggle_keyword_label": "Text", "search_toggle_keyword_label_suffix": "suche", "search_loading_nl": "Archiv wird befragt…", - "search_loading_nl_sub": "Die KI analysiert Ihre Anfrage. Das kann bis zu 15 Sekunden dauern.", + "search_loading_nl_sub": "Die Anfrage wird analysiert…", "search_error_unavailable": "Intelligente Suche nicht verfügbar", - "search_error_unavailable_body": "Die KI-Suche ist momentan nicht erreichbar. Sie können Ihre Anfrage als einfache Volltextsuche wiederholen.", + "search_error_unavailable_body": "Die intelligente Suche ist momentan nicht erreichbar. Sie können Ihre Anfrage als einfache Volltextsuche wiederholen.", "search_switch_to_keyword": "Zur Volltextsuche wechseln", "search_error_rate_limited": "Zu viele Anfragen", "search_error_rate_limited_body": "Sie haben die intelligente Suche zu häufig genutzt. Bitte warten Sie eine Minute und versuchen Sie es erneut.", diff --git a/frontend/messages/en.json b/frontend/messages/en.json index 84be1557..e473d770 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -25,14 +25,14 @@ "error_smart_search_unavailable": "The smart search is currently unavailable. Please use the regular search.", "error_smart_search_rate_limited": "You have used the search function too frequently. Please wait a minute.", "smart_search_keywords_not_applied": "Keywords could not be applied to this search.", - "search_toggle_smart_label": "AI", + "search_toggle_smart_label": "Smart", "search_toggle_smart_label_suffix": " search", "search_toggle_keyword_label": "Text", "search_toggle_keyword_label_suffix": " search", "search_loading_nl": "Querying the archive…", - "search_loading_nl_sub": "The AI is analysing your request. This can take up to 15 seconds.", + "search_loading_nl_sub": "Your request is being analysed…", "search_error_unavailable": "Smart search unavailable", - "search_error_unavailable_body": "The AI search is currently unreachable. You can repeat your request as a plain full-text search.", + "search_error_unavailable_body": "The smart search is currently unreachable. You can repeat your request as a plain full-text search.", "search_switch_to_keyword": "Switch to full-text search", "search_error_rate_limited": "Too many requests", "search_error_rate_limited_body": "You have used the smart search too frequently. Please wait a minute and try again.", diff --git a/frontend/messages/es.json b/frontend/messages/es.json index 7a40b82c..5337f39c 100644 --- a/frontend/messages/es.json +++ b/frontend/messages/es.json @@ -25,14 +25,14 @@ "error_smart_search_unavailable": "La búsqueda inteligente no está disponible en este momento. Por favor, usa la búsqueda normal.", "error_smart_search_rate_limited": "Has utilizado la función de búsqueda demasiadas veces. Por favor, espera un minuto.", "smart_search_keywords_not_applied": "Las palabras clave no pudieron aplicarse a esta búsqueda.", - "search_toggle_smart_label": "IA", - "search_toggle_smart_label_suffix": " búsqueda", + "search_toggle_smart_label": "búsqueda inteligente", + "search_toggle_smart_label_suffix": "", "search_toggle_keyword_label": "Texto", "search_toggle_keyword_label_suffix": " búsqueda", "search_loading_nl": "Consultando el archivo…", - "search_loading_nl_sub": "La IA está analizando su solicitud. Esto puede tardar hasta 15 segundos.", + "search_loading_nl_sub": "Su solicitud está siendo analizada…", "search_error_unavailable": "Búsqueda inteligente no disponible", - "search_error_unavailable_body": "La búsqueda con IA no está disponible en este momento. Puede repetir su solicitud como una búsqueda de texto completo.", + "search_error_unavailable_body": "La búsqueda inteligente no está disponible en este momento. Puede repetir su solicitud como una búsqueda de texto completo.", "search_switch_to_keyword": "Cambiar a búsqueda de texto completo", "search_error_rate_limited": "Demasiadas solicitudes", "search_error_rate_limited_body": "Ha utilizado la búsqueda inteligente con demasiada frecuencia. Espere un minuto e inténtelo de nuevo.", -- 2.49.1 From 34ff3dbdfd805ad98b8d8694110ce988b0c734e4 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:12:13 +0200 Subject: [PATCH 28/51] feat(infra): replace Ollama with nlp-service in docker-compose Co-Authored-By: Claude Sonnet 4.6 --- docker-compose.yml | 77 ++++++++++++------------------------------ nlp-service/Dockerfile | 2 +- 2 files changed, 23 insertions(+), 56 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f9e618ea..eb0a75ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,74 +141,41 @@ services: security_opt: - no-new-privileges:true - # --- Ollama: Model init (one-shot pull) --- - # Pulls qwen2.5:7b-instruct-q4_K_M (~4.7 GB) into the ollama_models volume on first start. - # On subsequent starts (model already in volume), exits quickly without re-downloading. + # --- NLP service: rule-based NL query parser --- + # FastAPI Python service; replaces Ollama for smart search query parsing. # Not started in CI — CI uses explicit service selection # (docker-compose.ci.yml: db minio create-buckets) - ollama-model-init: - image: ollama/ollama:0.30.6 - restart: "no" - networks: - - archiv-net - volumes: - - ollama_models:/root/.ollama - mem_limit: 2g - read_only: true - tmpfs: - - /tmp:size=512m - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - # The image ENTRYPOINT is `ollama`, so override it to a shell; the image has - # no curl, so readiness is probed with `ollama list` instead of a curl loop. - # The pull is guarded by a `grep` on the cached model list so an already-cached - # model exits clean without a registry round-trip (offline-safe re-up). - entrypoint: ["/bin/sh", "-c"] - command: - - "ollama serve & until ollama list >/dev/null 2>&1; do sleep 1; done && (ollama list | grep -q 'qwen2.5:7b-instruct-q4_K_M' || ollama pull qwen2.5:7b-instruct-q4_K_M)" - - # --- Ollama: LLM inference server --- - # Serves the pre-pulled model for NL search inference. - # Not started in CI — CI uses explicit service selection - # (docker-compose.ci.yml: db minio create-buckets) - ollama: - image: ollama/ollama:0.30.6 - container_name: archive-ollama + nlp-service: + build: + context: ./nlp-service + dockerfile: Dockerfile + container_name: archive-nlp restart: unless-stopped expose: - - "11434" + - "8001" networks: - archiv-net - volumes: - - ollama_models:/root/.ollama environment: - OLLAMA_API_KEY: "${OLLAMA_API_KEY}" - # Pin the model in memory (no idle unload) so queries never pay a cold-load - # penalty that exceeds the backend read timeout → NL search 503 after idle. - OLLAMA_KEEP_ALIVE: "-1" - cpus: "${OLLAMA_CPU_LIMIT:-4.0}" - mem_limit: "${OLLAMA_MEM_LIMIT:-8g}" - memswap_limit: "${OLLAMA_MEM_LIMIT:-8g}" + DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}" + NLP_FUZZY_THRESHOLD: "${NLP_FUZZY_THRESHOLD:-80}" + mem_limit: 256m + memswap_limit: 256m read_only: true tmpfs: - - /tmp:size=512m + - /tmp:size=32m cap_drop: - ALL security_opt: - no-new-privileges:true healthcheck: - # `ollama list` hits the local API and exits non-zero if the server is - # down — used instead of curl, which the image does not ship. - test: ["CMD", "ollama", "list"] - interval: 30s - timeout: 10s + test: ["CMD", "curl", "-f", "http://localhost:8001/health"] + interval: 10s + timeout: 5s retries: 5 - start_period: 60s # model weights are pre-loaded by ollama-model-init; service only needs to bind port + start_period: 15s depends_on: - ollama-model-init: - condition: service_completed_successfully + db: + condition: service_healthy # --- Backend: Spring Boot --- backend: @@ -228,6 +195,8 @@ services: condition: service_started ocr-service: condition: service_started + nlp-service: + condition: service_started environment: SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${POSTGRES_DB} SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER} @@ -253,8 +222,7 @@ services: SPRING_MAIL_PROPERTIES_MAIL_SMTP_STARTTLS_ENABLE: ${MAIL_STARTTLS_ENABLE:-false} APP_OCR_BASE_URL: http://ocr-service:8000 APP_OCR_TRAINING_TOKEN: "${OCR_TRAINING_TOKEN:-}" - APP_OLLAMA_BASE_URL: "${APP_OLLAMA_BASE_URL:-http://ollama:11434}" - APP_OLLAMA_API_KEY: "${OLLAMA_API_KEY}" + APP_NLP_BASE_URL: "http://nlp-service:8001" SENTRY_DSN: ${SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${SENTRY_TRACES_SAMPLE_RATE:-1.0} # Observability: send traces to Tempo inside archiv-net (OTLP gRPC port 4317) @@ -318,4 +286,3 @@ volumes: frontend_node_modules: ocr_models: ocr_cache: - ollama_models: diff --git a/nlp-service/Dockerfile b/nlp-service/Dockerfile index 61c723b0..ccb9e7e6 100644 --- a/nlp-service/Dockerfile +++ b/nlp-service/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.11.12-slim WORKDIR /app -- 2.49.1 From cc7132d11d7e324d7232d14b4f1456b5ed16cd6f Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:12:59 +0200 Subject: [PATCH 29/51] docs(c4): replace Ollama with nlp-service in L2 container diagram Co-Authored-By: Claude Sonnet 4.6 --- docs/architecture/c4/l2-containers.puml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/architecture/c4/l2-containers.puml b/docs/architecture/c4/l2-containers.puml index b8630001..9592b649 100644 --- a/docs/architecture/c4/l2-containers.puml +++ b/docs/architecture/c4/l2-containers.puml @@ -12,15 +12,14 @@ System_Boundary(archiv, "Familienarchiv (Docker Compose)") { Container(frontend, "Web Frontend", "SvelteKit / Node adapter / port 3000", "Server-side rendered UI. Handles auth session cookies, document search and viewer, transcription editor, annotation layer, family tree (Stammbaum), stories (Geschichten), activity feed (Chronik), enrichment workflow, and admin panel.") Container(backend, "API Backend", "Spring Boot 4 / Java 21 / Jetty / port 8080", "REST API. Implements document management, search, user auth, file upload/download, transcription, OCR orchestration, and SSE notifications. Trusts X-Forwarded-* headers from Caddy.") Container(ocr, "OCR Service", "Python FastAPI / port 8000", "Handwritten text recognition (HTR) and OCR microservice. Single-node by design — see ADR-001. Reachable only on the internal Docker network; no external port exposed.") - Container(ollama, "Ollama LLM Service", "ollama/ollama:0.30.6 / port 11434 (internal only)", "Local LLM inference server for NL search. Runs qwen2.5:7b-instruct-q4_K_M on CPU. Reachable only on the internal Docker network; no external port exposed. Disabled when APP_OLLAMA_BASE_URL is unset or blank.") - ' Named volume: ollama_models — model weights, fully reproducible, no backup needed + Container(nlp, "NLP Service", "Python FastAPI / port 8001 (internal only)", "Rule-based NL search query parser. Extracts person names (fuzzy DB lookup), date ranges (regex), person role, and keywords. Reachable only on the internal Docker network; no external port exposed.") ContainerDb(db, "Relational Database", "PostgreSQL 16", "Stores document metadata, persons, users, permission groups, tags, transcription blocks, audit log, and Spring Session data.") ContainerDb(storage, "Object Storage", "MinIO (S3-compatible)", "Stores the actual document files (PDFs, scans). Backend uses a bucket-scoped service account (archiv-app), not MinIO root.") Container(mc, "Bucket / Service-Account Init", "MinIO Client (mc)", "One-shot container on startup. Idempotent: creates the archive bucket, the archiv-app service account, and attaches the readwrite policy.") } System_Boundary(observability, "Observability Stack (/opt/familienarchiv/docker-compose.observability.yml)") { - Container(prometheus, "Prometheus", "prom/prometheus:v3.4.0", "Scrapes metrics from backend (8081 /actuator/prometheus), OCR service (8000 /metrics), Ollama (11434 /metrics), node-exporter, and cAdvisor. Retention: 30 days.") + Container(prometheus, "Prometheus", "prom/prometheus:v3.4.0", "Scrapes metrics from backend (8081 /actuator/prometheus), OCR service (8000 /metrics), node-exporter, and cAdvisor. Retention: 30 days.") Container(node_exporter, "Node Exporter", "prom/node-exporter:v1.9.0", "Host-level CPU, memory, disk, and network metrics.") Container(cadvisor, "cAdvisor", "gcr.io/cadvisor/cadvisor:v0.52.1", "Per-container resource metrics.") Container(loki, "Loki", "grafana/loki:3.4.2", "Stores log streams from all containers.") @@ -43,12 +42,12 @@ Rel(backend, ocr, "OCR job requests with presigned MinIO URL", "HTTP / REST / JS Rel(backend, mail, "Sends notification and password-reset emails (optional)", "SMTP") Rel(ocr, storage, "Fetches PDF via presigned URL", "HTTP / S3 presigned") Rel(mc, storage, "Bootstraps bucket + service account on startup", "MinIO Client CLI") -Rel(backend, ollama, "NL query parsing (POST /api/generate)", "HTTP / REST / JSON") +Rel(backend, nlp, "NL query parsing (POST /parse)", "HTTP / REST / JSON") Rel(promtail, loki, "Pushes log streams", "HTTP/Loki push API") Rel(backend, tempo, "Sends distributed traces via OTLP", "HTTP / OTLP / port 4318 (archiv-net)") Rel(prometheus, backend, "Scrapes JVM + HTTP metrics", "HTTP 8081 /actuator/prometheus") Rel(prometheus, ocr, "Scrapes OCR + http_* metrics", "HTTP 8000 /metrics") -Rel(prometheus, ollama, "Scrapes LLM request metrics", "HTTP 11434 /metrics") + Rel(grafana, prometheus, "Queries metrics", "HTTP 9090") Rel(grafana, loki, "Queries logs", "HTTP 3100") Rel(grafana, tempo, "Queries traces", "HTTP 3200") -- 2.49.1 From 3d2b907fb4328b6b135ca0c30a47c7fe867c8ff7 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:13:58 +0200 Subject: [PATCH 30/51] =?UTF-8?q?docs(adr):=20ADR-035=20=E2=80=94=20replac?= =?UTF-8?q?e=20Ollama=20with=20rule-based=20nlp-service?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- docs/adr/035-rule-based-nlp-service.md | 105 +++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 docs/adr/035-rule-based-nlp-service.md diff --git a/docs/adr/035-rule-based-nlp-service.md b/docs/adr/035-rule-based-nlp-service.md new file mode 100644 index 00000000..3955e423 --- /dev/null +++ b/docs/adr/035-rule-based-nlp-service.md @@ -0,0 +1,105 @@ +# ADR-035: Replace Ollama with a rule-based NLP service for smart search + +**Date:** 2026-06-07 +**Status:** Accepted +**Deciders:** Marcel Raddatz +**Supersedes:** ADR-028 (Ollama for NL search), ADR-034 (Ollama production deployment) +**Relates to:** #771 (implementation) + +--- + +## Context + +ADR-028 introduced Ollama + qwen2.5-7B to parse free-text search queries into structured +extractions (person names, date ranges, person role, keywords). After deploying to +staging (ADR-034) the approach showed three problems: + +1. **Cold-start latency:** even with `OLLAMA_KEEP_ALIVE=-1` a Qwen inference on CPU takes + ~18 s. This blows the UX budget for a search feature and requires a 60 s timeout. +2. **Resource cost:** 8 GB resident RAM + 4 vCPU cap for an LLM whose only job is regex- + level entity extraction from short (< 500 char) German family-history queries. +3. **Fragility:** model-weight downloads, version pinning, and init-container orchestration + add operational surface area with no quality benefit over a deterministic parser. + +The query set is narrow and well-understood: person names are all in the PostgreSQL +`persons` table; date patterns are a fixed repertoire of German/English/Spanish formats; +person role (sender vs. receiver) is reliably signalled by a handful of prepositions +("von", "an", "von … an"); keywords are nouns/proper nouns not consumed by the other +extractors. + +--- + +## Decision + +Replace Ollama with a lightweight, rule-based Python FastAPI service (`nlp-service`). + +### Architecture + +``` +POST /api/search/nl (NlSearchController) + → NlQueryParserService + → RestClientNlpClient.parse(query, lang) + → POST http://nlp-service:8001/parse + ← { personNames, personRole, dateFrom, dateTo, keywords, rawQuery } +``` + +The response contract is identical to the old `OllamaExtraction`; only the transport +and implementation change. Java callers see `NlpExtraction` (renamed, same shape). + +### Implementation + +- **`nlp-service/`** — standalone FastAPI app (Python 3.11.12-slim image, ~256 MB RAM) + - `extractor.py` — pipeline: person extraction → role detection → date parsing → keywords + - `person_matcher.py` — two-pass fuzzy lookup (rapidfuzz 3.x) against the `persons` DB table; + loaded at startup, no live DB queries during extraction + - `models.py` — Pydantic `ParseRequest` (max 500 chars), `ParseResponse` + - `main.py` — lifespan loads persons from `DATABASE_URL`; `/health` reports `persons_loaded` + +- **`backend/search/`** — `OllamaClient` / `OllamaExtraction` renamed to `NlpClient` / + `NlpExtraction`; `NlpProperties` (`@ConfigurationProperties("app.nlp")`) replaces + `OllamaProperties`; `lang` parameter added to `/parse` and threaded through the stack. + +### Tunable parameters + +| Env var | Default | Effect | +|---|---|---| +| `DATABASE_URL` | — | PostgreSQL DSN; unset → person matching disabled | +| `NLP_FUZZY_THRESHOLD` | `80` | rapidfuzz similarity floor (0–100) | + +### Graceful degradation + +The backend's `RestClientNlpClient` wraps all HTTP errors and timeouts in +`DomainException.serviceUnavailable(SMART_SEARCH_UNAVAILABLE)`, returning HTTP 503 to +the client — identical behaviour to the Ollama path. The rate limiter is relaxed from +5 to 20 requests/min (rule-based extraction completes in < 50 ms vs. ~18 s for LLM). + +--- + +## Consequences + +### Positive + +- **Latency:** < 50 ms per extraction vs. ~18 s — smart search is now interactive. +- **Memory:** ~256 MB vs. 8 GB — frees 7.75 GB on the production host. +- **No model downloads:** the image ships no weights; startup is a single DB query. +- **Deterministic:** same query always produces the same result; no temperature/sampling. +- **Testable without infrastructure:** pytest with a seeded `PersonMatcher` fixture; no + WireMock stubs needed for most unit tests. + +### Trade-offs + +- **No semantic generalisation.** The LLM could handle novel phrasing; the rule-based + parser only handles the preposition patterns it was written for. Edge cases that fall + outside the pattern produce an empty extraction rather than a best-effort result. +- **Person matching depends on DB content.** A person not yet in the archive will never + match, even if the user types their exact name. The LLM could surface the name as a + raw string; this service surfaces nothing. This is acceptable for the current archive + size and query patterns. +- **Language support is fixed at de/en/es** (Paraglide locales). Adding a fourth locale + requires adding its stopword list and preposition table to `extractor.py`. + +### Superseded ADRs + +ADR-028 and ADR-034 documented the Ollama topology, init recipe, keep-alive pin, and +memory budget. All of that is now moot. The `ollama`, `ollama-model-init`, and +`ollama_models` volume are removed from `docker-compose.yml`. -- 2.49.1 From 644b7c26835d1f6d604f28df43c8b67caed3520e Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:39:25 +0200 Subject: [PATCH 31/51] fix(infra): wait for nlp-service healthy before starting backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes condition: service_started → service_healthy so the backend container does not start until FastAPI has bound its port and loaded person names from the database. Eliminates the startup race where a first NL search would return 503 during nlp-service bootstrap. Co-Authored-By: Claude Sonnet 4.6 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index eb0a75ce..4de4245d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -196,7 +196,7 @@ services: ocr-service: condition: service_started nlp-service: - condition: service_started + condition: service_healthy environment: SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${POSTGRES_DB} SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER} -- 2.49.1 From 960f1c171a98d53025be28c9c54070c27e5d231d Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:40:01 +0200 Subject: [PATCH 32/51] =?UTF-8?q?docs(nlp-service):=20update=20CLAUDE.md?= =?UTF-8?q?=20=E2=80=94=20remove=20stale=20dateparser=20entry=20and=20prot?= =?UTF-8?q?otype=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes 'dateparser 1.2' from the stack section (dependency was dropped in favour of the rule-based date regex pipeline). Rewrites the Notes section to reflect that docker-compose integration and Java-side wiring were both delivered in this PR. Co-Authored-By: Claude Sonnet 4.6 --- nlp-service/CLAUDE.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nlp-service/CLAUDE.md b/nlp-service/CLAUDE.md index f7579b7d..821dc996 100644 --- a/nlp-service/CLAUDE.md +++ b/nlp-service/CLAUDE.md @@ -5,13 +5,13 @@ replacing Ollama for the Familienarchiv NL search feature. ## Stack -- Python 3.11, FastAPI 0.115, rapidfuzz 3.x, dateparser 1.2, psycopg2-binary +- Python 3.11, FastAPI 0.115, rapidfuzz 3.x, psycopg2-binary No ML models — persons are matched against the live DB via fuzzy lookup. ## Endpoints -- `POST /parse` — parse a free-text query, return extraction matching `OllamaExtraction` contract +- `POST /parse` — parse a free-text query, return extraction matching `NlpExtraction` contract - `GET /health` — returns `{"status": "ok", "persons_loaded": N}` ## Running locally @@ -51,8 +51,9 @@ See `docs/superpowers/specs/2026-06-07-spacy-nlp-service-design.md`. ## Notes -This is a **prototype** for extraction quality evaluation. No docker-compose integration or -Java-side changes in this iteration. The extraction contract matches `OllamaExtraction` in +This service is fully wired into `docker-compose.yml` (container `archive-nlp`, port 8001 +internal-only) and the Java search path (`RestClientNlpClient` → `NlQueryParserService` → +`NlSearchController`). The extraction contract matches `NlpExtraction` in `backend/src/main/java/org/raddatz/familienarchiv/search/`. Test sentences for manual evaluation are in `test_sentences.md`. -- 2.49.1 From 884c1156bdc32e05e5425939ce70cc9b90fdf7be Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:41:46 +0200 Subject: [PATCH 33/51] docs(deployment): replace Ollama with nlp-service in DEPLOYMENT.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - §1: update memory table (nlp-service ~256 MB vs Ollama ~8 GB); update memory budget note; add nlp-service to topology diagram - §2: replace 'Ollama (NL search) service' env var table with 'NLP service' table (APP_NLP_BASE_URL, NLP_FUZZY_THRESHOLD); add credential-rotation restart note - §3.4: replace Ollama model-pull first-deploy warning with nlp-service startup note (no download, --wait safe) - §6: replace Ollama operational section (model pull, ollama list, upgrade guide) with nlp-service health check and tuning guide Co-Authored-By: Claude Sonnet 4.6 --- docs/DEPLOYMENT.md | 90 ++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 52 deletions(-) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index f8523515..e8bc6b67 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -33,6 +33,7 @@ graph TD Backend -->|JDBC :5432| DB[(PostgreSQL 16)] Backend -->|S3 API :9000| MinIO[(MinIO)] Backend -->|HTTP :8000 internal| OCR["OCR Service\nPython FastAPI"] + Backend -->|HTTP :8001 internal| NLP["NLP Service\nPython FastAPI"] OCR -->|presigned URL| MinIO Caddy -->|SSE proxy_pass| Backend ``` @@ -40,7 +41,7 @@ graph TD **Key facts:** - Caddy terminates TLS and reverse-proxies to frontend (`:3000`) and backend (`:8080`). The Caddyfile is committed at [`infra/caddy/Caddyfile`](../infra/caddy/Caddyfile) and is installed on the host as `/etc/caddy/Caddyfile` (symlink). - The host binds all docker-published ports to `127.0.0.1` only; Caddy is the sole external entry point. -- The OCR service has **no published port** — reachable only on the internal Docker network from the backend. +- The OCR service and NLP service have **no published ports** — reachable only on the internal Docker network from the backend. - SSE notifications transit Caddy (browser → Caddy → backend); the backend is never reachable directly from the public internet. The SvelteKit SSR layer is bypassed for SSE, but Caddy is not. - The Caddyfile responds `404` on `/actuator/*` (defense in depth). Internal monitoring scrapes the backend on the docker network, not through Caddy. - Production and staging cohabit on the same host via docker compose project names: `archiv-production` (ports 8080/3000) and `archiv-staging` (ports 8081/3001). @@ -52,14 +53,14 @@ The OCR service requires significant RAM for model loading. The dev compose sets | Production target | RAM | Recommended OCR limit | NL Search | Notes | |---|---|---|---|---| -| Current server (Hetzner Serverbörse, i7-6700) | 64 GB | 12 GB | Supported | Default `mem_limit: 12g` works comfortably; plenty of headroom for Ollama | +| Current server (Hetzner Serverbörse, i7-6700) | 64 GB | 12 GB | Supported | Default `mem_limit: 12g` works comfortably; nlp-service adds only ~256 MB | | ≥ 16 GB RAM | 16+ GB | 12 GB | Supported | Default works | -| 8 GB RAM | 8 GB | 6 GB | Disabled — set `APP_OLLAMA_BASE_URL=` (empty) | Set `OCR_MEM_LIMIT=6g`; accept reduced batch sizes | -| 4 GB RAM | 4 GB | — | Unsupported | Disable OCR service (`profiles: [ocr]`); run OCR on demand only | +| 8 GB RAM | 8 GB | 6 GB | Supported | Set `OCR_MEM_LIMIT=6g`; accept reduced batch sizes; nlp-service is lightweight | +| 4 GB RAM | 4 GB | — | Supported | Disable OCR service (`profiles: [ocr]`); run OCR on demand only; nlp-service still runs | On servers with less than 16 GB RAM the default `mem_limit: 12g` cannot be honoured — set the `OCR_MEM_LIMIT` env var (in `.env.production` / `.env.staging`, or as a Gitea secret consumed by the workflow). The prod compose interpolates this var with a 12g default. -> **Memory budget:** OCR (~6 GB active) + Ollama (~8 GB) = ~14 GB. On servers with less than 16 GB RAM, do not run `docker-compose.observability.yml` continuously alongside both OCR and Ollama. +> **Memory budget:** OCR (~6 GB active) + nlp-service (~256 MB) = ~6.25 GB. The previous Ollama LLM (~8 GB) has been replaced by the rule-based nlp-service — significant memory headroom freed on all server tiers. ### Dev vs production differences @@ -147,15 +148,18 @@ All vars are set in `.env` at the repo root (copy from `.env.example`). The back | `XDG_CACHE_HOME` | XDG cache base dir — redirects Matplotlib and other XDG-aware libraries away from the read-only `HOME` (`/home/ocr`) to the writable cache volume | `/app/cache` | — | — | | `TORCH_HOME` | PyTorch model cache — redirects `~/.cache/torch` to the writable models volume | `/app/models/torch` | — | — | -### Ollama (NL search) service +### NLP service (NL search) | Variable | Purpose | Default | Required? | Sensitive? | |---|---|---|---|---| -| `APP_OLLAMA_BASE_URL` | Base URL for the Ollama service. Leave empty to disable NL search. | `http://ollama:11434` | — | — | -| `APP_OLLAMA_API_KEY` | API key passed as `Authorization: Bearer` to Ollama. Leave empty for unauthenticated access. Note: `OLLAMA_API_KEY` is not enforced in Ollama 0.6.5 or 0.30.6 (see ADR-028). | — | — | YES | -| `OLLAMA_CPU_LIMIT` | Docker CPU quota for the Ollama container. On CX42 (8 vCPUs) can be raised to `7.5`. | `4.0` | — | — | -| `OLLAMA_MEM_LIMIT` | Memory limit for the Ollama container. Requires CX42 (16 GB RAM). | `8g` | — | — | -| `OLLAMA_API_KEY` | API key set on the Ollama service itself. Same value as `APP_OLLAMA_API_KEY`. Leave empty for unauthenticated. | — | — | YES | +| `APP_NLP_BASE_URL` | Internal URL of the nlp-service container. Wired automatically in compose via `http://nlp-service:8001`. | `http://nlp-service:8001` | YES | — | +| `NLP_FUZZY_THRESHOLD` | Rapidfuzz similarity floor for person-name matching (0–100). Lower values match more aggressively; raise if false positives appear. | `80` | — | — | + +The nlp-service reads `DATABASE_URL` at startup (composed from `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB`). Any credential rotation that touches those three vars must be followed by a restart of **both** `backend` and `nlp-service`: + +```bash +docker compose restart nlp-service backend +``` ### Observability stack (`docker-compose.observability.yml`) @@ -277,17 +281,13 @@ git.raddatz.cloud A ### 3.4 First deploy -> **First start — Ollama model pull:** On first `docker compose up -d`, the `ollama-model-init` container pulls `qwen2.5:7b-instruct-q4_K_M` (~4.7 GB). At 10 Mbps this takes approximately 60–90 minutes; at 100 Mbps approximately 6–10 minutes. The pull is a one-time operation — subsequent restarts skip it (model already on the `ollama_models` volume). Monitor progress with `docker logs -f $(docker ps -q --filter name=ollama-model-init)`. +> **NL search startup:** `nlp-service` loads person names from the database at startup (single query, ~1–2 s). No model weights to download. The backend waits for `nlp-service` to pass its healthcheck (`/health` returns `{"status":"ok","persons_loaded":N}`) before starting, so `docker compose up -d --wait` is safe to use on first deploy. > -> **Do not use `--wait` on first deploy** — `docker compose up -d --wait` waits for all services to reach their health/completion target, including `ollama-model-init`. On first pull this blocks for 60–90 minutes and will time out any CI/deploy script that uses `--wait`. -> -> **Re-deploy idempotency:** on subsequent `docker compose up -d` runs (including `--force-recreate`), `ollama-model-init` re-executes but exits in seconds — Ollama's CLI skips the download when the model digest already matches what is on the volume. -> -> **Verify NL search is active** after enabling Ollama (`APP_OLLAMA_BASE_URL=http://ollama:11434`): +> **Verify NL search is active:** > ```bash -> curl -s http://localhost:8080/api/nl-search?q=brief+von+grossmutter -> # Returns 200 with results → NL search is active -> # Returns 503 NL_SEARCH_UNAVAILABLE → Ollama is not reachable or APP_OLLAMA_BASE_URL is unset +> curl -s http://localhost:8001/health +> # Returns {"status":"ok","persons_loaded":N} with N > 0 → person matching enabled +> # Returns {"status":"ok","persons_loaded":0} → DB not reachable or persons table empty > ``` ```bash @@ -328,7 +328,7 @@ docker compose logs --follow # Single snapshot docker compose logs --tail=200 -# services: frontend, backend, db, minio, ocr-service +# services: frontend, backend, db, minio, ocr-service, nlp-service ``` ### Log locations @@ -585,54 +585,40 @@ bash scripts/download-kraken-models.sh > Downloads the Kurrent/Sütterlin HTR models. Run once after a fresh clone or when models are updated. -### Ollama — natural-language search (NL Search) +### NLP service — natural-language search (NL Search) -NL search uses a local Ollama instance for query parsing. The `ollama` service is defined in `docker-compose.yml` alongside the main stack. +NL search uses the rule-based `nlp-service` FastAPI container for query parsing. It has no model weights — it loads person names from the database at startup and applies regex + fuzzy matching. See ADR-035. -**First-time model pull** (required before the feature works): +**Health check:** ```bash -docker compose exec ollama ollama pull qwen2.5:7b-instruct-q4_K_M +curl -s http://localhost:8001/health +# {"status":"ok","persons_loaded":1247} ``` -This downloads ~4.4 GB. The model is stored in the `ollama_data` Docker volume and persists across container restarts. +`persons_loaded: 0` means the service started but could not reach the database (check `DATABASE_URL` and that `db` is healthy). -**Verify the model is available:** +If `POST /api/search/nl` returns HTTP 503 `SMART_SEARCH_UNAVAILABLE`, the backend cannot reach `nlp-service`. Check with: ```bash -docker compose exec ollama ollama list +docker compose logs nlp-service --tail=50 +docker compose ps nlp-service ``` -Expected output includes `qwen2.5:7b-instruct-q4_K_M`. - -**Health check** — the backend polls `GET /api/tags` on Ollama at startup and before inference. If Ollama is absent, `POST /api/search/nl` returns HTTP 503 with `SMART_SEARCH_UNAVAILABLE`. - -**Configuration** (see `application.yaml` under `app.ollama`): +**Configuration** (see `application.yaml` under `app.nlp`): | Property | Default | Description | |---|---|---| -| `app.ollama.base-url` | `http://ollama:11434` | Ollama service URL (dev: `http://localhost:11434`) | -| `app.ollama.model` | `qwen2.5:7b-instruct-q4_K_M` | Model to use for inference | -| `app.ollama.timeout-seconds` | `60` | Read timeout for inference calls (absorbs cold model load on the first query after an Ollama restart) | -| `app.nl-search.rate-limit.max-requests-per-minute` | `5` | Per-user rate limit | +| `app.nlp.base-url` | `http://nlp-service:8001` | nlp-service URL; set via `APP_NLP_BASE_URL` env var | +| `app.nl-search.rate-limit.max-requests-per-minute` | `20` | Per-user rate limit | -### Upgrade the Ollama model +**Tuning person matching:** -To switch to a newer model version (e.g. a future release of `qwen2.5`): +Set `NLP_FUZZY_THRESHOLD` in `.env` (default: `80`, range: `0–100`). Lower values match more aggressively at the cost of false positives. Restart nlp-service after changing: -1. Update the model name in the `ollama-model-init` `command:` in `docker-compose.yml`. -2. Remove the existing model volume to free the old weights: - ```bash - docker volume rm familienarchiv_ollama_models - ``` - (In production the volume name is prefixed with the compose project: `archiv-production_ollama-models`.) -3. Restart the stack: - ```bash - docker compose up -d - ``` - The `ollama-model-init` container pulls the new model weights on first start (~4–8 GB download depending on the model). The `ollama` inference server will not start until the pull completes (`condition: service_completed_successfully`). - -> **`ollama_models` volume:** holds model weights only — fully reproducible by re-pull, no backup needed. +```bash +docker compose restart nlp-service +``` ### Trigger a canonical import -- 2.49.1 From 53fdc8e3f9e010ade931fb86de69d87545fd76e4 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 16:42:20 +0200 Subject: [PATCH 34/51] refactor(search): delete orphaned RestClientOllamaClientTest The source class RestClientOllamaClient was removed in 864f44a4 but the corresponding test file was not staged at the time. Removes the leftover file; coverage is provided by RestClientNlpClientTest. Co-Authored-By: Claude Sonnet 4.6 --- .../search/RestClientOllamaClientTest.java | 113 ------------------ 1 file changed, 113 deletions(-) delete mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/RestClientOllamaClientTest.java diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientOllamaClientTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientOllamaClientTest.java deleted file mode 100644 index 058ec095..00000000 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientOllamaClientTest.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import com.github.tomakehurst.wiremock.WireMockServer; -import com.github.tomakehurst.wiremock.core.WireMockConfiguration; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; - -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class RestClientOllamaClientTest { - - private WireMockServer wireMock; - private RestClientOllamaClient client; - - @BeforeEach - void setUp() { - wireMock = new WireMockServer(WireMockConfiguration.wireMockConfig().dynamicPort()); - wireMock.start(); - - OllamaProperties props = new OllamaProperties(); - props.setBaseUrl("http://localhost:" + wireMock.port()); - props.setModel("qwen2.5:7b-instruct-q4_K_M"); - props.setTimeoutSeconds(5); - props.setHealthCheckTimeoutSeconds(2); - - client = new RestClientOllamaClient(props); - } - - @AfterEach - void tearDown() { - wireMock.stop(); - } - - // --- Factory helpers --- - - private String makeOllamaResponseJson(String personNamesJson, String personRole, - String dateFrom, String dateTo, String keywordsJson) { - String inner = String.format( - "{\"personNames\":%s,\"personRole\":\"%s\",\"dateFrom\":%s,\"dateTo\":%s,\"keywords\":%s}", - personNamesJson, personRole, - dateFrom == null ? "null" : "\"" + dateFrom + "\"", - dateTo == null ? "null" : "\"" + dateTo + "\"", - keywordsJson - ); - return String.format("{\"model\":\"qwen2.5:7b-instruct-q4_K_M\",\"response\":\"%s\",\"done\":true}", - inner.replace("\"", "\\\"")); - } - - // --- Test cases --- - - @Test - void parse_returnsExtraction_whenOllamaReturnsValidJson() { - String body = makeOllamaResponseJson("[\"Walter\"]", "sender", "1914-01-01", "1914-12-31", "[\"Krieg\"]"); - wireMock.stubFor(post(urlEqualTo("/api/generate")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(body))); - - OllamaExtraction result = client.parse("Was hat Walter im Krieg geschrieben?"); - - assertThat(result.personNames()).containsExactly("Walter"); - assertThat(result.personRole()).isEqualTo("sender"); - assertThat(result.keywords()).containsExactly("Krieg"); - assertThat(result.dateFrom()).isNotNull(); - assertThat(result.dateTo()).isNotNull(); - } - - @Test - void parse_throwsSmartSearchUnavailable_whenOllamaReturns500() { - wireMock.stubFor(post(urlEqualTo("/api/generate")) - .willReturn(aResponse().withStatus(500))); - - assertThatThrownBy(() -> client.parse("some query")) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); - } - - @Test - void parse_throwsSmartSearchUnavailable_whenOllamaExceedsTimeout() { - wireMock.stubFor(post(urlEqualTo("/api/generate")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withFixedDelay(6000) - .withBody("{\"response\":\"{}\",\"done\":true}"))); - - assertThatThrownBy(() -> client.parse("some query")) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); - } - - @Test - void parse_throwsSmartSearchUnavailable_whenOllamaReturnsMalformedJson() { - wireMock.stubFor(post(urlEqualTo("/api/generate")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody("{\"response\":\"not-json-at-all\",\"done\":true}"))); - - assertThatThrownBy(() -> client.parse("some query")) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); - } -} -- 2.49.1 From 345b3eca87f1b83ebeb1637ae66ffcf4027f30f9 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 17:44:38 +0200 Subject: [PATCH 35/51] =?UTF-8?q?fix(search):=20replace=20languageTag()=20?= =?UTF-8?q?with=20getLocale();=20sync=20KI=E2=86=92Smart=20in=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paraglide 2.5 runtime exports getLocale(), not languageTag(). The 8bed0cc6 commit introduced the wrong import when threading lang through the NL search path. Also updates two test assertions that still expected the old 'KI' button label after 0b31a51e renamed it to 'Smart-Suche'. Co-Authored-By: Claude Sonnet 4.6 --- frontend/src/routes/SearchFilterBar.svelte.spec.ts | 2 +- frontend/src/routes/documents/+page.svelte | 4 ++-- frontend/src/routes/search/SmartModeToggle.svelte.spec.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/routes/SearchFilterBar.svelte.spec.ts b/frontend/src/routes/SearchFilterBar.svelte.spec.ts index fa389d8c..1de166f2 100644 --- a/frontend/src/routes/SearchFilterBar.svelte.spec.ts +++ b/frontend/src/routes/SearchFilterBar.svelte.spec.ts @@ -209,7 +209,7 @@ describe('SearchFilterBar – smart-mode chip lifecycle hooks', () => { smartMode: true, onModeToggle }); - await page.getByRole('button', { name: /KI/ }).click(); + await page.getByRole('button', { name: /Smart/ }).click(); expect(onModeToggle).toHaveBeenCalledOnce(); }); diff --git a/frontend/src/routes/documents/+page.svelte b/frontend/src/routes/documents/+page.svelte index 92556463..ad3744f5 100644 --- a/frontend/src/routes/documents/+page.svelte +++ b/frontend/src/routes/documents/+page.svelte @@ -17,7 +17,7 @@ import { bulkSelectionStore } from '$lib/document/bulkSelection.svelte'; import { getErrorMessage, parseBackendError } from '$lib/shared/errors'; import { csrfFetch } from '$lib/shared/cookies'; import * as m from '$lib/paraglide/messages.js'; -import { languageTag } from '$lib/paraglide/runtime'; +import { getLocale } from '$lib/paraglide/runtime'; import type { components } from '$lib/generated/api'; type NlQueryInterpretation = components['schemas']['NlQueryInterpretation']; @@ -225,7 +225,7 @@ async function runSmartSearch() { const res = await csrfFetch('/api/search/nl', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ query, lang: languageTag() }) + body: JSON.stringify({ query, lang: getLocale() }) }); if (!res.ok) { const backend = await parseBackendError(res); diff --git a/frontend/src/routes/search/SmartModeToggle.svelte.spec.ts b/frontend/src/routes/search/SmartModeToggle.svelte.spec.ts index 01347271..1661a50e 100644 --- a/frontend/src/routes/search/SmartModeToggle.svelte.spec.ts +++ b/frontend/src/routes/search/SmartModeToggle.svelte.spec.ts @@ -22,7 +22,7 @@ describe('SmartModeToggle', () => { it('shows the smart label when smartMode is true', async () => { render(SmartModeToggle, { smartMode: true }); const btn = page.getByRole('button'); - await expect.element(btn).toHaveTextContent('KI'); + await expect.element(btn).toHaveTextContent('Smart'); }); it('shows the keyword label when smartMode is false', async () => { -- 2.49.1 From e485626471fcb18532326e4a10ce7b0df51c0942 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 18:16:45 +0200 Subject: [PATCH 36/51] fix(infra): replace Ollama with nlp-service in docker-compose.prod.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the ollama and ollama-model-init services (and ollama-models volume) from the production/staging compose file. Adds the nlp-service in their place — mirroring the dev compose — and wires the backend dependency and APP_NLP_BASE_URL env var so staging can reach the new service. Co-Authored-By: Claude Sonnet 4.6 --- docker-compose.prod.yml | 74 ++++++++++++----------------------------- 1 file changed, 21 insertions(+), 53 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9c60b3bf..9a328b36 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -50,7 +50,6 @@ volumes: minio-data: ocr-models: ocr-cache: - ollama-models: services: db: @@ -201,72 +200,38 @@ services: security_opt: - no-new-privileges:true - # --- Ollama: Model init (one-shot pull) --- - # Pulls qwen2.5:7b-instruct-q4_K_M (~4.7 GB) into the ollama-models volume on - # first start; exits quickly on subsequent starts (model already cached). - # The ollama/ollama image's ENTRYPOINT is `ollama` and the image ships WITHOUT - # curl, so the entrypoint is overridden to a shell and readiness is probed with - # `ollama list` (not curl). The pull is guarded by a `grep` on the cached model - # list so a model already on the volume exits clean WITHOUT a registry round-trip - # — a host reboot during a registry/network blip can no longer fail init (which - # would block the ollama service via service_completed_successfully). - # Backend degrades gracefully (503) if Ollama is absent. - ollama-model-init: - image: ollama/ollama:0.30.6 - restart: "no" - entrypoint: ["/bin/sh", "-c"] - command: - - "ollama serve & until ollama list >/dev/null 2>&1; do sleep 1; done && (ollama list | grep -q 'qwen2.5:7b-instruct-q4_K_M' || ollama pull qwen2.5:7b-instruct-q4_K_M)" - networks: - - archiv-net - volumes: - - ollama-models:/root/.ollama - mem_limit: 2g - read_only: true - tmpfs: - - /tmp:size=512m - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - - # --- Ollama: LLM inference server --- - # Serves the pre-pulled model for NL search inference. Backend reaches it at - # http://ollama:11434 (application.yaml default; no env override required). - # Healthcheck uses `ollama list` because the image has no curl. - ollama: - image: ollama/ollama:0.30.6 + # --- NLP service: rule-based NL query parser --- + # Lightweight FastAPI service; replaces Ollama for smart search query parsing. + # Connects to the DB at startup to build person/tag lookup tables. + nlp-service: + build: + context: ./nlp-service restart: unless-stopped expose: - - "11434" + - "8001" networks: - archiv-net - volumes: - - ollama-models:/root/.ollama environment: - # Pin the model in memory (no idle unload). Without this, Ollama evicts - # the model after ~5 min idle and the next query pays a cold-load penalty - # that exceeds the backend read timeout → NL search 503 after idle. - OLLAMA_KEEP_ALIVE: "-1" - cpus: "${OLLAMA_CPU_LIMIT:-4.0}" - mem_limit: "${OLLAMA_MEM_LIMIT:-8g}" - memswap_limit: "${OLLAMA_MEM_LIMIT:-8g}" + DATABASE_URL: "postgresql://archiv:${POSTGRES_PASSWORD}@db:5432/archiv" + NLP_FUZZY_THRESHOLD: "${NLP_FUZZY_THRESHOLD:-80}" + mem_limit: 256m + memswap_limit: 256m read_only: true tmpfs: - - /tmp:size=512m + - /tmp:size=32m cap_drop: - ALL security_opt: - no-new-privileges:true healthcheck: - test: ["CMD", "ollama", "list"] - interval: 30s - timeout: 10s + test: ["CMD", "curl", "-f", "http://localhost:8001/health"] + interval: 10s + timeout: 5s retries: 5 - start_period: 60s + start_period: 15s depends_on: - ollama-model-init: - condition: service_completed_successfully + db: + condition: service_healthy backend: image: familienarchiv/backend:${TAG:-nightly} @@ -286,6 +251,8 @@ services: # is a one-shot that must complete successfully. See #510. create-buckets: condition: service_completed_successfully + nlp-service: + condition: service_healthy # Bound to localhost only — Caddy fronts external traffic. ports: - "127.0.0.1:${PORT_BACKEND}:8080" @@ -320,6 +287,7 @@ services: APP_ADMIN_PASSWORD: ${APP_ADMIN_PASSWORD} APP_OCR_BASE_URL: http://ocr-service:8000 APP_OCR_TRAINING_TOKEN: ${OCR_TRAINING_TOKEN} + APP_NLP_BASE_URL: http://nlp-service:8001 MAIL_HOST: ${MAIL_HOST} MAIL_PORT: ${MAIL_PORT:-587} MAIL_USERNAME: ${MAIL_USERNAME:-} -- 2.49.1 From 63926f440f0e0729827e8056022d06104263a385 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 18:41:36 +0200 Subject: [PATCH 37/51] refactor(search): delete backend NLP search package Remove entire backend search domain including: - NlSearchController, NlQueryParserService, NlpClient implementations - Rate limiting, properties, DTOs (NlSearchRequest/Response/NlQueryInterpretation) - All domain logic and tests (5 test files deleted) Backend compiles successfully post-deletion. Co-Authored-By: Claude Sonnet 4.6 --- .../search/NlQueryInterpretation.java | 26 - .../search/NlQueryParserService.java | 211 ----- .../search/NlSearchController.java | 28 - .../search/NlSearchRateLimitProperties.java | 12 - .../search/NlSearchRateLimiter.java | 46 -- .../search/NlSearchRequest.java | 15 - .../search/NlSearchResponse.java | 12 - .../familienarchiv/search/NlpClient.java | 5 - .../familienarchiv/search/NlpExtraction.java | 14 - .../search/NlpHealthClient.java | 5 - .../familienarchiv/search/NlpProperties.java | 16 - .../familienarchiv/search/PersonHint.java | 13 - .../search/RestClientNlpClient.java | 145 ---- .../familienarchiv/search/TagHint.java | 14 - .../search/NlQueryParserServiceTest.java | 711 ---------------- .../search/NlSearchControllerTest.java | 161 ---- .../search/NlSearchRateLimiterTest.java | 62 -- .../NlSearchTagResolutionIntegrationTest.java | 56 -- .../search/NlpPropertiesTest.java | 33 - .../search/RestClientNlpClientTest.java | 124 --- .../plans/2026-06-07-remove-nlp-search.md | 768 ++++++++++++++++++ 21 files changed, 768 insertions(+), 1709 deletions(-) delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryInterpretation.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryParserService.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchController.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimitProperties.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimiter.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRequest.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchResponse.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlpClient.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlpExtraction.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlpHealthClient.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/NlpProperties.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/PersonHint.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/RestClientNlpClient.java delete mode 100644 backend/src/main/java/org/raddatz/familienarchiv/search/TagHint.java delete mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/NlQueryParserServiceTest.java delete mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java delete mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchRateLimiterTest.java delete mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchTagResolutionIntegrationTest.java delete mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/NlpPropertiesTest.java delete mode 100644 backend/src/test/java/org/raddatz/familienarchiv/search/RestClientNlpClientTest.java create mode 100644 docs/superpowers/plans/2026-06-07-remove-nlp-search.md diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryInterpretation.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryInterpretation.java deleted file mode 100644 index 37611488..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryInterpretation.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import io.swagger.v3.oas.annotations.media.Schema; - -import java.time.LocalDate; -import java.util.List; - -public record NlQueryInterpretation( - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - List resolvedPersons, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - List ambiguousPersons, - LocalDate dateFrom, - LocalDate dateTo, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - List keywords, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - List resolvedTags, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - String rawQuery, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - boolean keywordsApplied, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - boolean tagsApplied -) { -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryParserService.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryParserService.java deleted file mode 100644 index db8a9da9..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlQueryParserService.java +++ /dev/null @@ -1,211 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.raddatz.familienarchiv.document.DocumentSearchResult; -import org.raddatz.familienarchiv.document.DocumentService; -import org.raddatz.familienarchiv.document.DocumentSort; -import org.raddatz.familienarchiv.document.SearchFilters; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; -import org.raddatz.familienarchiv.person.NameMatches; -import org.raddatz.familienarchiv.person.Person; -import org.raddatz.familienarchiv.person.PersonService; -import org.raddatz.familienarchiv.tag.Tag; -import org.raddatz.familienarchiv.tag.TagOperator; -import org.raddatz.familienarchiv.tag.TagService; -import org.springframework.data.domain.Pageable; -import org.springframework.stereotype.Service; - -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.UUID; - -@Service -@RequiredArgsConstructor -@Slf4j -public class NlQueryParserService { - - private static final int MIN_QUERY = 3; - private static final int MAX_QUERY = 500; - private static final int MAX_NAME_LENGTH = 200; - private static final int MIN_TAG_TERM = 3; - private static final int MAX_RESOLVED_TAGS = 10; - - private final NlpClient nlpClient; - private final PersonService personService; - private final DocumentService documentService; - private final TagService tagService; - - public NlSearchResponse search(String query, String lang, Pageable pageable) { - NlpExtraction ext = nlpClient.parse(query, lang); - - List personNames = ext.personNames() != null ? ext.personNames() : List.of(); - List keywords = ext.keywords() != null ? ext.keywords() : List.of(); - - TagResolution tagResolution = resolveTags(keywords); - List resolvedTagHints = tagResolution.hints(); - List resolvedTagNames = tagResolution.tagNames(); - List remainingKeywords = tagResolution.remaining(); - - NameResolution resolution = resolveNames(personNames); - - if (!resolution.ambiguous().isEmpty()) { - NlQueryInterpretation interpretation = new NlQueryInterpretation( - List.of(), resolution.ambiguous(), - ext.dateFrom(), ext.dateTo(), - keywords, List.of(), ext.rawQuery(), false, false); - return new NlSearchResponse(DocumentSearchResult.of(List.of()), interpretation); - } - - List resolved = resolution.resolved(); - List noMatchFragments = resolution.noMatchFragments(); - List extraFragments = resolution.extraFragments(); - - boolean hadStructuredMatch = !resolvedTagHints.isEmpty() || !resolved.isEmpty(); - String text = buildText(remainingKeywords, noMatchFragments, extraFragments, ext.rawQuery(), hadStructuredMatch); - - if (resolved.size() == 1 && isAnyRole(ext.personRole())) { - UUID personId = resolved.get(0).id(); - DocumentSearchResult docs = documentService.searchDocumentsByPersonId( - personId, ext.dateFrom(), ext.dateTo(), pageable); - NlQueryInterpretation interpretation = new NlQueryInterpretation( - resolved, List.of(), ext.dateFrom(), ext.dateTo(), keywords, resolvedTagHints, ext.rawQuery(), false, false); - return new NlSearchResponse(docs, interpretation); - } - - UUID sender = buildSender(resolved, ext.personRole()); - UUID receiver = buildReceiver(resolved, ext.personRole()); - - boolean tagsApplied = !resolvedTagHints.isEmpty(); - TagOperator tagOperator = tagsApplied ? TagOperator.OR : TagOperator.AND; - - SearchFilters filters = new SearchFilters( - text.isBlank() ? null : text, - ext.dateFrom(), ext.dateTo(), - sender, receiver, - resolvedTagNames, null, - null, tagOperator, false); - - DocumentSearchResult docs = documentService.searchDocuments(filters, DocumentSort.DATE, "desc", pageable); - boolean keywordsApplied = !text.isBlank(); - NlQueryInterpretation interpretation = new NlQueryInterpretation( - resolved, List.of(), ext.dateFrom(), ext.dateTo(), keywords, resolvedTagHints, ext.rawQuery(), keywordsApplied, tagsApplied); - return new NlSearchResponse(docs, interpretation); - } - - private NameResolution resolveNames(List personNames) { - List resolved = new ArrayList<>(); - List ambiguous = new ArrayList<>(); - List noMatchFragments = new ArrayList<>(); - List extraFragments = new ArrayList<>(); - - int resolvedIndex = 0; - for (String name : personNames) { - if (name == null || name.length() > MAX_NAME_LENGTH) { - log.debug("Skipping name fragment (too long or null): length={}", name == null ? 0 : name.length()); - continue; - } - NameMatches matches = personService.resolveByName(name); - List direct = matches.direct(); - List partial = matches.partial(); - - if (direct.size() == 1) { - Person p = direct.get(0); - resolvedIndex++; - if (resolvedIndex <= 2) { - resolved.add(new PersonHint(p.getId(), p.getDisplayName())); - } else { - extraFragments.add(name); - } - } else if (direct.size() >= 2) { - direct.forEach(p -> ambiguous.add(new PersonHint(p.getId(), p.getDisplayName()))); - } else if (!partial.isEmpty()) { - partial.forEach(p -> ambiguous.add(new PersonHint(p.getId(), p.getDisplayName()))); - } else { - noMatchFragments.add(name); - } - } - - return new NameResolution(resolved, ambiguous, noMatchFragments, extraFragments); - } - - private TagResolution resolveTags(List keywords) { - LinkedHashSet seen = new LinkedHashSet<>(); - List remaining = new ArrayList<>(); - - for (String kw : keywords) { - if (kw == null || kw.length() < MIN_TAG_TERM) { - remaining.add(kw); - continue; - } - List matches = tagService.findByNameContaining(kw); - if (matches.isEmpty()) { - remaining.add(kw); - } else { - seen.addAll(matches); - } - } - - if (seen.size() > MAX_RESOLVED_TAGS) { - log.debug("Keyword matched {} tags; capping at {}", seen.size(), MAX_RESOLVED_TAGS); - } - List capped = seen.size() > MAX_RESOLVED_TAGS - ? new ArrayList<>(seen).subList(0, MAX_RESOLVED_TAGS) - : new ArrayList<>(seen); - - // safe: entities are detached here; mutation is for DTO projection only, no dirty-check fires - tagService.resolveEffectiveColors(capped); - - List hints = capped.stream() - .map(t -> new TagHint(t.getId(), t.getName(), t.getColor())) - .toList(); - List tagNames = capped.stream().map(Tag::getName).toList(); - - return new TagResolution(hints, tagNames, remaining); - } - - private String buildText(List keywords, List noMatchFragments, - List extraFragments, String rawQuery, boolean hadStructuredMatch) { - List parts = new ArrayList<>(); - parts.addAll(keywords); - parts.addAll(noMatchFragments); - parts.addAll(extraFragments); - String text = String.join(" ", parts).strip(); - if (text.isBlank() && !hadStructuredMatch && rawQuery != null && !rawQuery.isBlank()) { - return rawQuery; - } - return text; - } - - private boolean isAnyRole(String role) { - return role == null || "any".equals(role) || (!"sender".equals(role) && !"receiver".equals(role)); - } - - private UUID buildSender(List resolved, String role) { - if (resolved.size() >= 2) return resolved.get(0).id(); - if (resolved.size() == 1 && "sender".equals(role)) return resolved.get(0).id(); - return null; - } - - private UUID buildReceiver(List resolved, String role) { - if (resolved.size() >= 2) return resolved.get(1).id(); - if (resolved.size() == 1 && "receiver".equals(role)) return resolved.get(0).id(); - return null; - } - - private record NameResolution( - List resolved, - List ambiguous, - List noMatchFragments, - List extraFragments - ) {} - - private record TagResolution( - List hints, - List tagNames, - List remaining - ) {} -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchController.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchController.java deleted file mode 100644 index 82391a72..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchController.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import jakarta.validation.Valid; -import lombok.RequiredArgsConstructor; -import org.raddatz.familienarchiv.security.Permission; -import org.raddatz.familienarchiv.security.RequirePermission; -import org.springframework.data.domain.Pageable; -import org.springframework.security.core.annotation.AuthenticationPrincipal; -import org.springframework.security.core.userdetails.UserDetails; -import org.springframework.web.bind.annotation.*; - -@RestController -@RequestMapping("/api/search/nl") -@RequiredArgsConstructor -public class NlSearchController { - - private final NlQueryParserService nlQueryParserService; - private final NlSearchRateLimiter rateLimiter; - - @PostMapping - @RequirePermission(Permission.READ_ALL) - public NlSearchResponse search(@Valid @RequestBody NlSearchRequest request, - Pageable pageable, - @AuthenticationPrincipal UserDetails principal) { - rateLimiter.checkAndConsume(principal.getUsername()); - return nlQueryParserService.search(request.query(), request.lang(), pageable); - } -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimitProperties.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimitProperties.java deleted file mode 100644 index 3be4b3b0..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimitProperties.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -@Component -@ConfigurationProperties("app.nl-search.rate-limit") -@Data -public class NlSearchRateLimitProperties { - private int maxRequestsPerMinute = 20; -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimiter.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimiter.java deleted file mode 100644 index 100296fa..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRateLimiter.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.LoadingCache; -import io.github.bucket4j.Bandwidth; -import io.github.bucket4j.Bucket; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; -import org.springframework.stereotype.Service; - -import java.time.Duration; -import java.util.concurrent.TimeUnit; - -@Service -public class NlSearchRateLimiter { - - private final LoadingCache byUser; - private final int maxRequestsPerMinute; - - public NlSearchRateLimiter(NlSearchRateLimitProperties props) { - this.maxRequestsPerMinute = props.getMaxRequestsPerMinute(); - this.byUser = Caffeine.newBuilder() - .expireAfterAccess(1, TimeUnit.MINUTES) - .build(key -> newBucket(maxRequestsPerMinute)); - } - - public void checkAndConsume(String userKey) { - if (!byUser.get(userKey).tryConsume(1)) { - throw DomainException.tooManyRequests(ErrorCode.SMART_SEARCH_RATE_LIMITED, - "NL search rate limit exceeded for user: " + userKey, 60L); - } - } - - void resetForTest() { - byUser.invalidateAll(); - } - - private static Bucket newBucket(int limit) { - return Bucket.builder() - .addLimit(Bandwidth.builder() - .capacity(limit) - .refillGreedy(limit, Duration.ofMinutes(1)) - .build()) - .build(); - } -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRequest.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRequest.java deleted file mode 100644 index f23241d0..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchRequest.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.Pattern; -import jakarta.validation.constraints.Size; - -public record NlSearchRequest( - @NotBlank - @Size(min = 3, max = 500) - String query, - @NotBlank - @Pattern(regexp = "de|en|es") - String lang -) { -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchResponse.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchResponse.java deleted file mode 100644 index 04e51bff..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlSearchResponse.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import io.swagger.v3.oas.annotations.media.Schema; -import org.raddatz.familienarchiv.document.DocumentSearchResult; - -public record NlSearchResponse( - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - DocumentSearchResult result, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - NlQueryInterpretation interpretation -) { -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpClient.java deleted file mode 100644 index c7889c7e..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpClient.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.raddatz.familienarchiv.search; - -public interface NlpClient { - NlpExtraction parse(String query, String lang); -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpExtraction.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpExtraction.java deleted file mode 100644 index 73c36027..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpExtraction.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import java.time.LocalDate; -import java.util.List; - -record NlpExtraction( - List personNames, - String personRole, - LocalDate dateFrom, - LocalDate dateTo, - List keywords, - String rawQuery -) { -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpHealthClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpHealthClient.java deleted file mode 100644 index a02475c2..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpHealthClient.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.raddatz.familienarchiv.search; - -public interface NlpHealthClient { - boolean isHealthy(); -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpProperties.java b/backend/src/main/java/org/raddatz/familienarchiv/search/NlpProperties.java deleted file mode 100644 index 8b939e1e..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/NlpProperties.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import jakarta.validation.constraints.NotBlank; -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -@ConfigurationProperties("app.nlp") -@Data -@Validated -public class NlpProperties { - @NotBlank - private String baseUrl; - private int timeoutSeconds = 5; - private int healthCheckTimeoutSeconds = 2; -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/PersonHint.java b/backend/src/main/java/org/raddatz/familienarchiv/search/PersonHint.java deleted file mode 100644 index 61a0e0b9..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/PersonHint.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import io.swagger.v3.oas.annotations.media.Schema; - -import java.util.UUID; - -public record PersonHint( - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - UUID id, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - String displayName -) { -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientNlpClient.java b/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientNlpClient.java deleted file mode 100644 index d058e470..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/RestClientNlpClient.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.extern.slf4j.Slf4j; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; -import org.springframework.http.MediaType; -import org.springframework.http.client.JdkClientHttpRequestFactory; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestClient; - -import java.net.http.HttpClient; -import java.time.Duration; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.List; -import java.util.Set; - -@Service -@Slf4j -public class RestClientNlpClient implements NlpClient, NlpHealthClient { - - private static final Set VALID_ROLES = Set.of("sender", "receiver", "any"); - private static final int MAX_NAME_LENGTH = 200; - private static final int MAX_KEYWORD_LENGTH = 100; - - private final RestClient parseClient; - private final RestClient healthRestClient; - - public RestClientNlpClient(NlpProperties props) { - HttpClient parseHttp = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(Duration.ofSeconds(10)) - .build(); - JdkClientHttpRequestFactory parseFactory = new JdkClientHttpRequestFactory(parseHttp); - parseFactory.setReadTimeout(Duration.ofSeconds(props.getTimeoutSeconds())); - this.parseClient = RestClient.builder() - .baseUrl(props.getBaseUrl()) - .requestFactory(parseFactory) - .build(); - - HttpClient healthHttp = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(Duration.ofSeconds(props.getHealthCheckTimeoutSeconds())) - .build(); - JdkClientHttpRequestFactory healthFactory = new JdkClientHttpRequestFactory(healthHttp); - healthFactory.setReadTimeout(Duration.ofSeconds(props.getHealthCheckTimeoutSeconds())); - this.healthRestClient = RestClient.builder() - .baseUrl(props.getBaseUrl()) - .requestFactory(healthFactory) - .build(); - } - - @Override - public NlpExtraction parse(String query, String lang) { - try { - NlpParseRequest request = new NlpParseRequest(query, lang); - NlpParseResponse response = parseClient.post() - .uri("/parse") - .contentType(MediaType.APPLICATION_JSON) - .body(request) - .retrieve() - .body(NlpParseResponse.class); - return toExtraction(response, query); - } catch (DomainException e) { - throw e; - } catch (Exception e) { - log.warn("NLP service inference failed: {}", e.getClass().getSimpleName()); - throw DomainException.serviceUnavailable(ErrorCode.SMART_SEARCH_UNAVAILABLE, - "NLP service unavailable: " + e.getClass().getSimpleName()); - } - } - - @Override - public boolean isHealthy() { - try { - NlpHealthResponse resp = healthRestClient.get() - .uri("/health") - .retrieve() - .body(NlpHealthResponse.class); - return resp != null && resp.personsLoaded() > 0; - } catch (Exception e) { - return false; - } - } - - private NlpExtraction toExtraction(NlpParseResponse response, String rawQuery) { - if (response == null) { - return fallbackExtraction(rawQuery); - } - List names = response.personNames() == null ? List.of() : response.personNames().stream() - .filter(n -> n != null && n.length() <= MAX_NAME_LENGTH) - .toList(); - List keywords = response.keywords() == null ? List.of() : response.keywords().stream() - .filter(k -> k != null && k.length() <= MAX_KEYWORD_LENGTH) - .toList(); - String role = sanitiseRole(response.personRole()); - LocalDate dateFrom = parseDate(response.dateFrom()); - LocalDate dateTo = parseDate(response.dateTo()); - return new NlpExtraction(names, role, dateFrom, dateTo, keywords, rawQuery); - } - - private NlpExtraction fallbackExtraction(String rawQuery) { - return new NlpExtraction(List.of(), "any", null, null, List.of(), rawQuery); - } - - private String sanitiseRole(String role) { - if (role != null && VALID_ROLES.contains(role)) { - return role; - } - log.warn("Unexpected personRole from NLP service: {}", role); - return "any"; - } - - private LocalDate parseDate(String raw) { - if (raw == null || raw.isBlank()) return null; - try { - return LocalDate.parse(raw, DateTimeFormatter.ISO_LOCAL_DATE); - } catch (DateTimeParseException ignored) { - } - return null; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - private record NlpParseRequest(String query, String lang) { - } - - @JsonIgnoreProperties(ignoreUnknown = true) - private record NlpParseResponse( - @JsonProperty("personNames") List personNames, - @JsonProperty("personRole") String personRole, - @JsonProperty("dateFrom") String dateFrom, - @JsonProperty("dateTo") String dateTo, - @JsonProperty("keywords") List keywords - ) { - } - - @JsonIgnoreProperties(ignoreUnknown = true) - private record NlpHealthResponse( - @JsonProperty("persons_loaded") int personsLoaded - ) { - } -} diff --git a/backend/src/main/java/org/raddatz/familienarchiv/search/TagHint.java b/backend/src/main/java/org/raddatz/familienarchiv/search/TagHint.java deleted file mode 100644 index c488796f..00000000 --- a/backend/src/main/java/org/raddatz/familienarchiv/search/TagHint.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import io.swagger.v3.oas.annotations.media.Schema; - -import java.util.UUID; - -public record TagHint( - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - UUID id, - @Schema(requiredMode = Schema.RequiredMode.REQUIRED) - String name, - String color -) { -} diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlQueryParserServiceTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlQueryParserServiceTest.java deleted file mode 100644 index 76f8e88c..00000000 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/NlQueryParserServiceTest.java +++ /dev/null @@ -1,711 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.raddatz.familienarchiv.document.DocumentSearchResult; -import org.raddatz.familienarchiv.document.DocumentService; -import org.raddatz.familienarchiv.document.DocumentSort; -import org.raddatz.familienarchiv.document.SearchFilters; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; -import org.raddatz.familienarchiv.person.NameMatches; -import org.raddatz.familienarchiv.person.Person; -import org.raddatz.familienarchiv.person.PersonService; -import org.raddatz.familienarchiv.tag.Tag; -import org.raddatz.familienarchiv.tag.TagOperator; -import org.raddatz.familienarchiv.tag.TagService; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; - -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -class NlQueryParserServiceTest { - - @Mock NlpClient nlpClient; - @Mock PersonService personService; - @Mock DocumentService documentService; - @Mock TagService tagService; - - NlQueryParserService service; - - static final Pageable PAGE = PageRequest.of(0, 20); - - @BeforeEach - void setUp() { - MockitoAnnotations.openMocks(this); - service = new NlQueryParserService(nlpClient, personService, documentService, tagService); - when(documentService.searchDocuments(any(), any(), any(), any())) - .thenReturn(DocumentSearchResult.of(List.of())); - when(documentService.searchDocumentsByPersonId(any(), any(), any(), any())) - .thenReturn(DocumentSearchResult.of(List.of())); - when(tagService.findByNameContaining(anyString())).thenReturn(List.of()); - } - - // --- Factory helpers --- - - private NlpExtraction extraction(List names, String role, LocalDate from, LocalDate to, - List keywords) { - String raw = names.isEmpty() ? "test query" : String.join(" ", names); - return new NlpExtraction(names, role, from, to, keywords, raw); - } - - private Person person(UUID id, String firstName, String lastName) { - return Person.builder().id(id).firstName(firstName).lastName(lastName).build(); - } - - private NameMatches makeNameMatches() { - return new NameMatches(List.of(), List.of()); - } - - private NameMatches makeNameMatches(List direct) { - return new NameMatches(direct, List.of()); - } - - private NameMatches makeNameMatches(List direct, List partial) { - return new NameMatches(direct, partial); - } - - private static final UUID P1 = UUID.fromString("00000000-0000-0000-0000-000000000001"); - private static final UUID P2 = UUID.fromString("00000000-0000-0000-0000-000000000002"); - private static final UUID P3 = UUID.fromString("00000000-0000-0000-0000-000000000003"); - - // --- 1. Single resolved name + personRole=sender --- - - @Test - void search_resolvesSingleName_asSender() { - Person walter = person(P1, "Walter", "Raddatz"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Walter"), "sender", null, null, List.of())); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - - NlSearchResponse resp = service.search("Was hat Walter geschrieben?", "de", PAGE); - - verify(nlpClient).parse(eq("Was hat Walter geschrieben?"), eq("de")); - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), eq(DocumentSort.DATE), eq("desc"), eq(PAGE)); - assertThat(cap.getValue().sender()).isEqualTo(P1); - assertThat(cap.getValue().receiver()).isNull(); - assertThat(resp.interpretation().resolvedPersons()).hasSize(1); - assertThat(resp.interpretation().resolvedPersons().get(0).id()).isEqualTo(P1); - assertThat(resp.interpretation().ambiguousPersons()).isEmpty(); - } - - // --- 2. Multi-match name → ambiguous, search NOT executed --- - - @Test - void search_multiMatchName_populatesAmbiguous_andSkipsSearch() { - Person a = person(UUID.randomUUID(), "Walter", "Braun"); - Person b = person(UUID.randomUUID(), "Walter", "Schmidt"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Walter"), "sender", null, null, List.of())); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(a, b))); - - NlSearchResponse resp = service.search("Briefe von Walter", "de", PAGE); - - verify(documentService, never()).searchDocuments(any(), any(), any(), any()); - verify(documentService, never()).searchDocumentsByPersonId(any(), any(), any(), any()); - assertThat(resp.interpretation().ambiguousPersons()).hasSize(2); - assertThat(resp.interpretation().resolvedPersons()).isEmpty(); - } - - // --- 3. Multi-match + personRole=any → still ambiguous, search NOT executed --- - - @Test - void search_multiMatchName_withPersonRoleAny_stillSkipsSearch() { - Person a = person(UUID.randomUUID(), "Emma", "Braun"); - Person b = person(UUID.randomUUID(), "Emma", "Raddatz"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Emma"), "any", null, null, List.of())); - when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(a, b))); - - NlSearchResponse resp = service.search("Briefe an Emma", "de", PAGE); - - verify(documentService, never()).searchDocuments(any(), any(), any(), any()); - verify(documentService, never()).searchDocumentsByPersonId(any(), any(), any(), any()); - assertThat(resp.interpretation().ambiguousPersons()).hasSize(2); - } - - // --- 4. No-match name → folded into text --- - - @Test - void search_noMatchName_isFoldedIntoText() { - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Karl"), "any", null, null, List.of())); - when(personService.resolveByName("Karl")).thenReturn(makeNameMatches()); - - service.search("Briefe von Karl", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().text()).contains("Karl"); - assertThat(cap.getValue().sender()).isNull(); - assertThat(cap.getValue().receiver()).isNull(); - } - - // --- 5. personRole=any + 1 resolved → searchDocumentsByPersonId called --- - - @Test - void search_personRoleAny_singleMatch_callsSearchDocumentsByPersonId() { - Person walter = person(P1, "Walter", "Raddatz"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Walter"), "any", null, null, List.of())); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - - NlSearchResponse resp = service.search("Briefe von Walter", "de", PAGE); - - verify(documentService).searchDocumentsByPersonId(eq(P1), isNull(), isNull(), eq(PAGE)); - verify(documentService, never()).searchDocuments(any(), any(), any(), any()); - assertThat(resp.interpretation().keywordsApplied()).isFalse(); - } - - // --- 6. 2 names both resolve → sender=person1, receiver=person2 --- - - @Test - void search_twoNamesResolve_assignsSenderAndReceiver() { - Person walter = person(P1, "Walter", "Raddatz"); - Person emma = person(P2, "Emma", "Raddatz"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Walter", "Emma"), "any", null, null, List.of())); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma))); - - NlSearchResponse resp = service.search("Briefe von Walter an Emma", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), eq(DocumentSort.DATE), eq("desc"), eq(PAGE)); - assertThat(cap.getValue().sender()).isEqualTo(P1); - assertThat(cap.getValue().receiver()).isEqualTo(P2); - assertThat(resp.interpretation().resolvedPersons().get(0).id()).isEqualTo(P1); - assertThat(resp.interpretation().resolvedPersons().get(1).id()).isEqualTo(P2); - } - - // --- 7. 2 names, first resolves, second ambiguous → search NOT executed --- - - @Test - void search_twoNames_secondAmbiguous_skipsSearch() { - Person walter = person(P1, "Walter", "Raddatz"); - Person emma1 = person(P2, "Emma", "Braun"); - Person emma2 = person(P3, "Emma", "Schmidt"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Walter", "Emma"), "sender", null, null, List.of())); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma1, emma2))); - - NlSearchResponse resp = service.search("Briefe von Walter an Emma", "de", PAGE); - - verify(documentService, never()).searchDocuments(any(), any(), any(), any()); - assertThat(resp.interpretation().ambiguousPersons()).hasSize(2); - } - - // --- 8. 2 names, first no match → folded into text, second used as single person --- - - @Test - void search_twoNames_firstNoMatch_secondResolved_foldFirstIntoText() { - Person emma = person(P2, "Emma", "Raddatz"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Karl", "Emma"), "sender", null, null, List.of())); - when(personService.resolveByName("Karl")).thenReturn(makeNameMatches()); - when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma))); - - service.search("Briefe von Karl an Emma", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().text()).contains("Karl"); - assertThat(cap.getValue().sender()).isEqualTo(P2); - } - - // --- 9. 3+ names all resolve → first two as sender/receiver, third folded into text --- - - @Test - void search_threeNamesResolve_extraFoldedIntoText() { - Person walter = person(P1, "Walter", "Raddatz"); - Person emma = person(P2, "Emma", "Raddatz"); - Person heinrich = person(P3, "Heinrich", "Braun"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Walter", "Emma", "Heinrich"), "any", null, null, List.of())); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma))); - when(personService.resolveByName("Heinrich")).thenReturn(makeNameMatches(List.of(heinrich))); - - service.search("Briefe von Walter an Emma über Heinrich", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().sender()).isEqualTo(P1); - assertThat(cap.getValue().receiver()).isEqualTo(P2); - assertThat(cap.getValue().text()).contains("Heinrich"); - } - - // --- 10. Keywords space-joined into text --- - - @Test - void search_keywords_areJoinedIntoText() { - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Krieg", "Walter"))); - - service.search("Dokumente über den Krieg Walter", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().text()).isEqualTo("Krieg Walter"); - } - - // --- 11. Date range passed through --- - - @Test - void search_dateRange_passedIntoSearchFilters() { - LocalDate from = LocalDate.of(1914, 1, 1); - LocalDate to = LocalDate.of(1914, 12, 31); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", from, to, List.of())); - - service.search("Briefe aus dem Jahr 1914", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().from()).isEqualTo(from); - assertThat(cap.getValue().to()).isEqualTo(to); - } - - // --- 12. Null dates → null in SearchFilters (not an error) --- - - @Test - void search_nullDates_passedAsNullIntoFilters() { - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); - - service.search("Hochzeitsbriefe", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().from()).isNull(); - assertThat(cap.getValue().to()).isNull(); - } - - // --- 13. NLP service returns empty names/keywords → raw query used as keyword fallback --- - - @Test - void search_nlpReturnsEmpty_usesRawQueryAsTextFallback() { - String raw = "Briefe aus dem Krieg"; - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(new NlpExtraction(List.of(), "any", null, null, List.of(), raw)); - - service.search(raw, "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().text()).isEqualTo(raw); - } - - // --- 14. Null personNames/keywords → no NPE --- - - @Test - void search_nullPersonNamesAndKeywords_handledWithoutNpe() { - NlpExtraction ext = new NlpExtraction(null, "any", null, null, null, "test query"); - when(nlpClient.parse(anyString(), anyString())).thenReturn(ext); - - NlSearchResponse resp = service.search("test query", "de", PAGE); - - assertThat(resp).isNotNull(); - verify(documentService).searchDocuments(any(), any(), any(), any()); - } - - // --- 15. Unrecognized personRole → defaults to any-like behavior (no crash) --- - - @Test - void search_unrecognizedPersonRole_treatedLikeAny_withSingleResolvedPerson() { - Person walter = person(P1, "Walter", "Raddatz"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(new NlpExtraction(List.of("Walter"), "unknown_role", null, null, List.of(), "query")); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - - NlSearchResponse resp = service.search("Briefe von Walter", "de", PAGE); - - assertThat(resp).isNotNull(); - } - - // --- 16. NLP service throws SMART_SEARCH_UNAVAILABLE → propagates to caller --- - - @Test - void search_nlpThrowsUnavailable_propagates() { - when(nlpClient.parse(anyString(), anyString())) - .thenThrow(DomainException.tooManyRequests(ErrorCode.SMART_SEARCH_UNAVAILABLE, "offline")); - - assertThatThrownBy(() -> service.search("Was hat Walter geschrieben?", "de", PAGE)) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); - } - - // --- 17. LLM-extracted name > 200 chars → skipped, PersonService never called --- - - @Test - void search_nameLongerThan200Chars_isSkippedBeforePersonServiceCall() { - String longName = "A".repeat(201); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(longName), "sender", null, null, List.of())); - - service.search("Briefe von sehr langem Namen", "de", PAGE); - - verify(personService, never()).resolveByName(anyString()); - } - - // --- 18. Cap: 10 direct matches → all shown as ambiguous --- - - @Test - void search_tenDirectMatches_allShownAsAmbiguous() { - List ten = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - ten.add(person(UUID.randomUUID(), "Walter", "Person" + i)); - } - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Walter"), "sender", null, null, List.of())); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(ten)); - - NlSearchResponse resp = service.search("Briefe von Walter", "de", PAGE); - - assertThat(resp.interpretation().ambiguousPersons()).hasSize(10); - verify(documentService, never()).searchDocuments(any(), any(), any(), any()); - } - - // --- 19. SearchFilters defaults: tagOperator=AND, status=null, undated=false, tags=empty --- - - @Test - void search_searchFiltersDefaults_areCorrect() { - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Krieg"))); - - service.search("Dokumente über den Krieg", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), eq(DocumentSort.DATE), eq("desc"), eq(PAGE)); - SearchFilters f = cap.getValue(); - assertThat(f.tagOperator()).isEqualTo(TagOperator.AND); - assertThat(f.status()).isNull(); - assertThat(f.undated()).isFalse(); - assertThat(f.tags()).isEmpty(); - assertThat(f.tagQ()).isNull(); - } - - // --- 20. personRole=receiver + 1 resolved → receiver UUID set --- - - @Test - void search_personRoleReceiver_singleMatch_setsReceiver() { - Person emma = person(P2, "Emma", "Raddatz"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Emma"), "receiver", null, null, List.of())); - when(personService.resolveByName("Emma")).thenReturn(makeNameMatches(List.of(emma))); - - service.search("Briefe an Emma", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().receiver()).isEqualTo(P2); - assertThat(cap.getValue().sender()).isNull(); - } - - // --- 21. keywordsApplied=true when text is non-blank --- - - @Test - void search_keywordsApplied_trueWhenTextNonBlank() { - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Feldpost"))); - - NlSearchResponse resp = service.search("Feldpost aus dem Krieg", "de", PAGE); - - assertThat(resp.interpretation().keywordsApplied()).isTrue(); - } - - // --- 22. Partial-only, one candidate → ambiguous (1-item picker), search skipped --- - - @Test - void search_partialOnly_oneCandidate_populatesAmbiguous() { - Person cramer = person(P1, "Clara", "Cramer"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Clara Cram"), "any", null, null, List.of())); - when(personService.resolveByName("Clara Cram")).thenReturn(makeNameMatches(List.of(), List.of(cramer))); - - NlSearchResponse resp = service.search("Briefe von Clara Cram", "de", PAGE); - - assertThat(resp.interpretation().ambiguousPersons()).hasSize(1); - verify(documentService, never()).searchDocuments(any(), any(), any(), any()); - } - - // --- 23. Partial-only, two candidates → ambiguous (multi-item picker) --- - - @Test - void search_partialOnly_twoCandidates_populatesAmbiguous() { - Person cramer = person(P1, "Clara", "Cramer"); - Person crammond = person(P2, "Clara", "Crammond"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Clara Cram"), "any", null, null, List.of())); - when(personService.resolveByName("Clara Cram")) - .thenReturn(makeNameMatches(List.of(), List.of(cramer, crammond))); - - NlSearchResponse resp = service.search("Briefe von Clara Cram", "de", PAGE); - - assertThat(resp.interpretation().ambiguousPersons()).hasSize(2); - } - - // --- 24. Exactly one direct match → search executes, no picker --- - - @Test - void search_oneDirect_executesSearch() { - Person clara = person(P1, "Clara", "Cram"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Clara Cram"), "any", null, null, List.of())); - when(personService.resolveByName("Clara Cram")).thenReturn(makeNameMatches(List.of(clara))); - - NlSearchResponse resp = service.search("Briefe von Clara Cram", "de", PAGE); - - verify(documentService).searchDocumentsByPersonId(eq(P1), isNull(), isNull(), eq(PAGE)); - assertThat(resp.interpretation().ambiguousPersons()).isEmpty(); - } - - // --- Tag resolution helpers --- - - private Tag tag(UUID id, String name) { - return Tag.builder().id(id).name(name).build(); - } - - private Tag tag(UUID id, String name, String color) { - return Tag.builder().id(id).name(name).color(color).build(); - } - - private TagHint tagHint(UUID id, String name, String color) { - return new TagHint(id, name, color); - } - - private static final UUID T1 = UUID.fromString("00000000-0000-0000-0001-000000000001"); - - // --- 25. Single keyword resolves to one tag → tag filter applied --- - - @Test - void search_singleKeywordResolvesToTag_appliesTagFilter() { - Tag hochzeit = tag(T1, "Hochzeit"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - - NlSearchResponse resp = service.search("Briefe über Hochzeit", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().tags()).containsExactly("Hochzeit"); - assertThat(cap.getValue().tagOperator()).isEqualTo(TagOperator.OR); - assertThat(resp.interpretation().resolvedTags()).hasSize(1); - assertThat(resp.interpretation().resolvedTags().get(0).name()).isEqualTo("Hochzeit"); - assertThat(resp.interpretation().tagsApplied()).isTrue(); - assertThat(cap.getValue().text()).isNull(); - } - - private static final UUID T2 = UUID.fromString("00000000-0000-0000-0001-000000000002"); - - // --- 26. Keyword matches multiple tags → all in resolvedTags, OR-union --- - - @Test - void search_keywordMatchesMultipleTags_allIncluded() { - Tag hochzeit1 = tag(T1, "Hochzeit Raddatz"); - Tag hochzeit2 = tag(T2, "Hochzeit Braun"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit1, hochzeit2)); - - NlSearchResponse resp = service.search("Briefe über Hochzeit", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().tags()).containsExactlyInAnyOrder("Hochzeit Raddatz", "Hochzeit Braun"); - assertThat(cap.getValue().tagOperator()).isEqualTo(TagOperator.OR); - assertThat(resp.interpretation().resolvedTags()).hasSize(2); - } - - // --- 27. Keyword no tag match → stays as FTS text, resolvedTags empty --- - - @Test - void search_keywordNoTagMatch_staysAsFtsText() { - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Feldpost"))); - - NlSearchResponse resp = service.search("Feldpost Briefe", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().text()).contains("Feldpost"); - assertThat(cap.getValue().tags()).isEmpty(); - assertThat(resp.interpretation().resolvedTags()).isEmpty(); - assertThat(resp.interpretation().tagsApplied()).isFalse(); - } - - // --- 28. Mixed: one keyword resolves, one doesn't → tag filter + FTS text --- - - @Test - void search_mixedKeywords_oneResolves_oneStaysAsText() { - Tag hochzeit = tag(T1, "Hochzeit"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit", "Feldpost"))); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - - NlSearchResponse resp = service.search("Hochzeit und Feldpost", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().tags()).containsExactly("Hochzeit"); - assertThat(cap.getValue().tagOperator()).isEqualTo(TagOperator.OR); - assertThat(cap.getValue().text()).contains("Feldpost"); - assertThat(resp.interpretation().resolvedTags()).hasSize(1); - assertThat(resp.interpretation().tagsApplied()).isTrue(); - } - - // --- 29. personRole=any + 1 person + resolvable keyword → personId search, tagsApplied=false --- - - @Test - void search_personRoleAny_singlePerson_resolvableKeyword_tagsAppliedFalse() { - Person walter = person(P1, "Walter", "Raddatz"); - Tag hochzeit = tag(T1, "Hochzeit"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of("Walter"), "any", null, null, List.of("Hochzeit"))); - when(personService.resolveByName("Walter")).thenReturn(makeNameMatches(List.of(walter))); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - - NlSearchResponse resp = service.search("Briefe von Walter über Hochzeit", "de", PAGE); - - verify(documentService).searchDocumentsByPersonId(eq(P1), isNull(), isNull(), eq(PAGE)); - verify(documentService, never()).searchDocuments(any(), any(), any(), any()); - assertThat(resp.interpretation().tagsApplied()).isFalse(); - assertThat(resp.interpretation().resolvedTags()).hasSize(1); - assertThat(resp.interpretation().resolvedTags().get(0).name()).isEqualTo("Hochzeit"); - } - - // --- 30. Cap: keyword matches > 10 tags → capped at 10 --- - - @Test - void search_keywordMatchesMoreThanMaxTags_cappedAtTen() { - List eleven = new ArrayList<>(); - for (int i = 0; i < 11; i++) { - eleven.add(tag(UUID.randomUUID(), "Thema " + i)); - } - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Thema"))); - when(tagService.findByNameContaining("Thema")).thenReturn(eleven); - - NlSearchResponse resp = service.search("Dokumente zum Thema", "de", PAGE); - - assertThat(resp.interpretation().resolvedTags()).hasSize(10); - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().tags()).hasSize(10); - } - - // --- 31. Short keyword (< 3 chars) → skipped, not passed to TagService --- - - @Test - void search_shortKeyword_skippedByTagResolution() { - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("ab", "Krieg"))); - - service.search("ab Krieg", "de", PAGE); - - verify(tagService, never()).findByNameContaining("ab"); - verify(tagService).findByNameContaining("Krieg"); - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().text()).contains("ab"); - } - - // --- 32. Dedup: same tag matched by two keywords → appears once --- - - @Test - void search_sameTagMatchedByTwoKeywords_deduplicatedToOne() { - Tag hochzeit = tag(T1, "Hochzeit"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit", "hoch"))); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - when(tagService.findByNameContaining("hoch")).thenReturn(List.of(hochzeit)); - - NlSearchResponse resp = service.search("Hochzeit hoch", "de", PAGE); - - assertThat(resp.interpretation().resolvedTags()).hasSize(1); - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().tags()).hasSize(1); - } - - // --- 33. All keywords resolve → rawQuery fallback suppressed, text=null --- - - @Test - void search_allKeywordsResolveToTags_rawQueryFallbackSuppressed() { - Tag hochzeit = tag(T1, "Hochzeit"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(new NlpExtraction(List.of(), "any", null, null, List.of("Hochzeit"), "raw query text")); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - - NlSearchResponse resp = service.search("Hochzeit", "de", PAGE); - - ArgumentCaptor cap = ArgumentCaptor.forClass(SearchFilters.class); - verify(documentService).searchDocuments(cap.capture(), any(), any(), any()); - assertThat(cap.getValue().text()).isNull(); - assertThat(cap.getValue().tags()).containsExactly("Hochzeit"); - } - - // --- 34. Flag independence: keywordsApplied=false AND tagsApplied=true --- - - @Test - void search_allKeywordsResolveToTags_keywordsAppliedFalse_tagsAppliedTrue() { - Tag hochzeit = tag(T1, "Hochzeit"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - - NlSearchResponse resp = service.search("Hochzeit Briefe", "de", PAGE); - - assertThat(resp.interpretation().keywordsApplied()).isFalse(); - assertThat(resp.interpretation().tagsApplied()).isTrue(); - } - - // --- 35. Color carried through from resolveEffectiveColors --- - - @Test - void search_tagHint_carriesColorSetByResolveEffectiveColors() { - Tag hochzeit = tag(T1, "Hochzeit"); - doAnswer(invocation -> { - Collection tags = invocation.getArgument(0); - tags.forEach(t -> t.setColor("sage")); - return null; - }).when(tagService).resolveEffectiveColors(any()); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - - NlSearchResponse resp = service.search("Hochzeit", "de", PAGE); - - assertThat(resp.interpretation().resolvedTags().get(0).color()).isEqualTo("sage"); - } - - // --- 36. Color stays null when resolveEffectiveColors leaves it unset --- - - @Test - void search_tagHint_colorIsNull_whenNoColorResolved() { - Tag hochzeit = tag(T1, "Hochzeit"); - when(nlpClient.parse(anyString(), anyString())) - .thenReturn(extraction(List.of(), "any", null, null, List.of("Hochzeit"))); - when(tagService.findByNameContaining("Hochzeit")).thenReturn(List.of(hochzeit)); - - NlSearchResponse resp = service.search("Hochzeit", "de", PAGE); - - assertThat(resp.interpretation().resolvedTags().get(0).color()).isNull(); - } -} diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java deleted file mode 100644 index 8630a2cb..00000000 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchControllerTest.java +++ /dev/null @@ -1,161 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import tools.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.raddatz.familienarchiv.document.DocumentSearchResult; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; -import org.raddatz.familienarchiv.security.SecurityConfig; -import org.raddatz.familienarchiv.security.PermissionAspect; -import org.raddatz.familienarchiv.user.CustomUserDetailsService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.aop.AopAutoConfiguration; -import org.springframework.boot.webmvc.test.autoconfigure.WebMvcTest; -import org.springframework.context.annotation.Import; -import org.springframework.http.MediaType; -import org.springframework.security.test.context.support.WithMockUser; -import org.springframework.test.context.bean.override.mockito.MockitoBean; -import org.springframework.test.web.servlet.MockMvc; - -import java.util.List; -import java.util.UUID; - -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.when; -import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; - -@WebMvcTest(NlSearchController.class) -@Import({SecurityConfig.class, PermissionAspect.class, AopAutoConfiguration.class, - NlSearchRateLimiter.class, NlSearchRateLimitProperties.class}) -class NlSearchControllerTest { - - @Autowired MockMvc mockMvc; - private final ObjectMapper objectMapper = new ObjectMapper(); - - @MockitoBean NlQueryParserService nlQueryParserService; - @MockitoBean CustomUserDetailsService customUserDetailsService; - @Autowired NlSearchRateLimiter rateLimiter; - - @BeforeEach - void resetRateLimiter() { - rateLimiter.resetForTest(); - } - - private NlSearchResponse makeResponse() { - PersonHint hint = new PersonHint(UUID.randomUUID(), "Walter Raddatz"); - NlQueryInterpretation interp = new NlQueryInterpretation( - List.of(hint), List.of(), null, null, - List.of("Krieg"), List.of(), "Briefe von Walter im Krieg", true, false); - return new NlSearchResponse(DocumentSearchResult.of(List.of()), interp); - } - - // --- 1. Happy path --- - - @Test - @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) - void search_returns200_withNlSearchResponse() throws Exception { - when(nlQueryParserService.search(anyString(), anyString(), any())).thenReturn(makeResponse()); - - mockMvc.perform(post("/api/search/nl").with(csrf()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter im Krieg\",\"lang\":\"de\"}")) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.interpretation.rawQuery").value("Briefe von Walter im Krieg")) - .andExpect(jsonPath("$.interpretation.resolvedPersons[0].displayName").value("Walter Raddatz")) - .andExpect(jsonPath("$.interpretation.keywordsApplied").value(true)); - } - - // --- 2. ambiguousPersons in response shape --- - - @Test - @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) - void search_returns200_withAmbiguousPersons() throws Exception { - PersonHint a = new PersonHint(UUID.randomUUID(), "Walter Braun"); - PersonHint b = new PersonHint(UUID.randomUUID(), "Walter Schmidt"); - NlQueryInterpretation interp = new NlQueryInterpretation( - List.of(), List.of(a, b), null, null, - List.of(), List.of(), "Briefe von Walter", false, false); - NlSearchResponse resp = new NlSearchResponse(DocumentSearchResult.of(List.of()), interp); - when(nlQueryParserService.search(anyString(), anyString(), any())).thenReturn(resp); - - mockMvc.perform(post("/api/search/nl").with(csrf()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.interpretation.ambiguousPersons").isArray()) - .andExpect(jsonPath("$.interpretation.ambiguousPersons[0].displayName").value("Walter Braun")) - .andExpect(jsonPath("$.interpretation.ambiguousPersons[1].id").isNotEmpty()); - } - - // --- 3. Unauthenticated → 401 --- - - @Test - void search_returns401_whenUnauthenticated() throws Exception { - mockMvc.perform(post("/api/search/nl").with(csrf()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) - .andExpect(status().isUnauthorized()); - } - - // --- 4. Query < 3 chars → 400 --- - - @Test - @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) - void search_returns400_whenQueryTooShort() throws Exception { - mockMvc.perform(post("/api/search/nl").with(csrf()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"ab\",\"lang\":\"de\"}")) - .andExpect(status().isBadRequest()); - } - - // --- 5. Query > 500 chars → 400 --- - - @Test - @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) - void search_returns400_whenQueryTooLong() throws Exception { - String longQuery = "a".repeat(501); - mockMvc.perform(post("/api/search/nl").with(csrf()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"" + longQuery + "\",\"lang\":\"de\"}")) - .andExpect(status().isBadRequest()); - } - - // --- 6. NLP service unavailable → 503 --- - - @Test - @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) - void search_returns503_whenNlpServiceUnavailable() throws Exception { - when(nlQueryParserService.search(anyString(), anyString(), any())) - .thenThrow(DomainException.serviceUnavailable(ErrorCode.SMART_SEARCH_UNAVAILABLE, "NLP service offline")); - - mockMvc.perform(post("/api/search/nl").with(csrf()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) - .andExpect(status().isServiceUnavailable()) - .andExpect(jsonPath("$.code").value("SMART_SEARCH_UNAVAILABLE")); - } - - // --- 7. 21st request in 1 minute → 429 (rate limit = 20/min default) --- - - @Test - @WithMockUser(username = "user@test.com", authorities = {"READ_ALL"}) - void search_returns429_on21stRequestWithinRateLimit() throws Exception { - when(nlQueryParserService.search(anyString(), anyString(), any())).thenReturn(makeResponse()); - - for (int i = 0; i < 20; i++) { - mockMvc.perform(post("/api/search/nl").with(csrf()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) - .andExpect(status().isOk()); - } - - mockMvc.perform(post("/api/search/nl").with(csrf()) - .contentType(MediaType.APPLICATION_JSON) - .content("{\"query\":\"Briefe von Walter\",\"lang\":\"de\"}")) - .andExpect(status().isTooManyRequests()) - .andExpect(jsonPath("$.code").value("SMART_SEARCH_RATE_LIMITED")); - } -} diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchRateLimiterTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchRateLimiterTest.java deleted file mode 100644 index 43a2bbe0..00000000 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchRateLimiterTest.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; - -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class NlSearchRateLimiterTest { - - private NlSearchRateLimiter rateLimiter; - - @BeforeEach - void setUp() { - NlSearchRateLimitProperties props = new NlSearchRateLimitProperties(); - props.setMaxRequestsPerMinute(5); - rateLimiter = new NlSearchRateLimiter(props); - } - - @Test - void checkAndConsume_allowsRequestsWithinLimit() { - for (int i = 0; i < 5; i++) { - assertThatCode(() -> rateLimiter.checkAndConsume("user@example.com")) - .doesNotThrowAnyException(); - } - } - - @Test - void checkAndConsume_throwsRateLimited_onSixthRequest() { - for (int i = 0; i < 5; i++) { - rateLimiter.checkAndConsume("user@example.com"); - } - - assertThatThrownBy(() -> rateLimiter.checkAndConsume("user@example.com")) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.SMART_SEARCH_RATE_LIMITED); - } - - @Test - void checkAndConsume_limitsAreIndependentPerUser() { - for (int i = 0; i < 5; i++) { - rateLimiter.checkAndConsume("alice@example.com"); - } - assertThatCode(() -> rateLimiter.checkAndConsume("bob@example.com")) - .doesNotThrowAnyException(); - } - - @Test - void resetForTest_clearsAllBuckets() { - for (int i = 0; i < 5; i++) { - rateLimiter.checkAndConsume("user@example.com"); - } - - rateLimiter.resetForTest(); - - assertThatCode(() -> rateLimiter.checkAndConsume("user@example.com")) - .doesNotThrowAnyException(); - } -} diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchTagResolutionIntegrationTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchTagResolutionIntegrationTest.java deleted file mode 100644 index bbe0a88b..00000000 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/NlSearchTagResolutionIntegrationTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import org.junit.jupiter.api.Test; -import org.raddatz.familienarchiv.PostgresContainerConfig; -import org.raddatz.familienarchiv.config.FlywayConfig; -import org.raddatz.familienarchiv.tag.Tag; -import org.raddatz.familienarchiv.tag.TagRepository; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.jdbc.test.autoconfigure.AutoConfigureTestDatabase; -import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest; -import org.springframework.context.annotation.Import; - -import java.util.List; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; - -@DataJpaTest -@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) -@Import({PostgresContainerConfig.class, FlywayConfig.class}) -class NlSearchTagResolutionIntegrationTest { - - @Autowired - private TagRepository tagRepository; - - @Test - void findDescendantIdsByName_parentName_includesChildId() { - Tag parent = tagRepository.save(Tag.builder().name("Krieg").build()); - Tag child = tagRepository.save(Tag.builder().name("Weltkrieg").parentId(parent.getId()).build()); - - List ids = tagRepository.findDescendantIdsByName("Krieg"); - - assertThat(ids).containsExactlyInAnyOrder(parent.getId(), child.getId()); - } - - @Test - void findDescendantIdsByName_childName_returnsOnlyChild() { - Tag parent = tagRepository.save(Tag.builder().name("Krieg").build()); - Tag child = tagRepository.save(Tag.builder().name("Weltkrieg").parentId(parent.getId()).build()); - - List ids = tagRepository.findDescendantIdsByName("Weltkrieg"); - - assertThat(ids).containsExactly(child.getId()); - assertThat(ids).doesNotContain(parent.getId()); - } - - @Test - void findByNameContainingIgnoreCase_parentSubstring_matchesParentOnly() { - Tag parent = tagRepository.save(Tag.builder().name("Krieg").build()); - tagRepository.save(Tag.builder().name("Weltkrieg").parentId(parent.getId()).build()); - - List found = tagRepository.findByNameContainingIgnoreCase("Krieg"); - - assertThat(found).extracting(Tag::getName).containsExactlyInAnyOrder("Krieg", "Weltkrieg"); - } -} diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/NlpPropertiesTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/NlpPropertiesTest.java deleted file mode 100644 index bb527066..00000000 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/NlpPropertiesTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; - -import static org.assertj.core.api.Assertions.assertThat; - -class NlpPropertiesTest { - - @EnableConfigurationProperties(NlpProperties.class) - static class TestConfig {} - - private final ApplicationContextRunner runner = new ApplicationContextRunner() - .withUserConfiguration(TestConfig.class); - - @Test - void failsWhenBaseUrlMissing() { - runner.run(context -> assertThat(context).hasFailed()); - } - - @Test - void bindsBaseUrlAndDefaults() { - runner.withPropertyValues("app.nlp.base-url=http://nlp:8001") - .run(context -> { - assertThat(context).hasNotFailed(); - NlpProperties props = context.getBean(NlpProperties.class); - assertThat(props.getBaseUrl()).isEqualTo("http://nlp:8001"); - assertThat(props.getTimeoutSeconds()).isEqualTo(5); - assertThat(props.getHealthCheckTimeoutSeconds()).isEqualTo(2); - }); - } -} diff --git a/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientNlpClientTest.java b/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientNlpClientTest.java deleted file mode 100644 index 0198f6c0..00000000 --- a/backend/src/test/java/org/raddatz/familienarchiv/search/RestClientNlpClientTest.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.raddatz.familienarchiv.search; - -import com.github.tomakehurst.wiremock.WireMockServer; -import com.github.tomakehurst.wiremock.core.WireMockConfiguration; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.raddatz.familienarchiv.exception.DomainException; -import org.raddatz.familienarchiv.exception.ErrorCode; - -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class RestClientNlpClientTest { - - private WireMockServer wireMock; - private RestClientNlpClient client; - - @BeforeEach - void setUp() { - wireMock = new WireMockServer(WireMockConfiguration.wireMockConfig().dynamicPort()); - wireMock.start(); - - NlpProperties props = new NlpProperties(); - props.setBaseUrl("http://localhost:" + wireMock.port()); - props.setTimeoutSeconds(5); - props.setHealthCheckTimeoutSeconds(2); - - client = new RestClientNlpClient(props); - } - - @AfterEach - void tearDown() { - wireMock.stop(); - } - - private String makeParseResponseJson(String personNamesJson, String personRole, - String dateFrom, String dateTo, String keywordsJson, - String rawQuery) { - return String.format( - "{\"personNames\":%s,\"personRole\":\"%s\",\"dateFrom\":%s,\"dateTo\":%s,\"keywords\":%s,\"rawQuery\":\"%s\"}", - personNamesJson, personRole, - dateFrom == null ? "null" : "\"" + dateFrom + "\"", - dateTo == null ? "null" : "\"" + dateTo + "\"", - keywordsJson, rawQuery - ); - } - - @Test - void parse_returnsExtraction_whenNlpServiceReturnsValidJson() { - String body = makeParseResponseJson("[\"Walter\"]", "sender", "1914-01-01", "1914-12-31", - "[\"Krieg\"]", "Briefe von Walter im Krieg"); - wireMock.stubFor(post(urlEqualTo("/parse")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody(body))); - - NlpExtraction result = client.parse("Briefe von Walter im Krieg", "de"); - - assertThat(result.personNames()).containsExactly("Walter"); - assertThat(result.personRole()).isEqualTo("sender"); - assertThat(result.keywords()).containsExactly("Krieg"); - assertThat(result.dateFrom()).isNotNull(); - assertThat(result.dateTo()).isNotNull(); - } - - @Test - void parse_throwsSmartSearchUnavailable_whenNlpServiceReturns500() { - wireMock.stubFor(post(urlEqualTo("/parse")) - .willReturn(aResponse().withStatus(500))); - - assertThatThrownBy(() -> client.parse("some query", "de")) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); - } - - @Test - void parse_throwsSmartSearchUnavailable_whenNlpServiceExceedsTimeout() { - wireMock.stubFor(post(urlEqualTo("/parse")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withFixedDelay(6000) - .withBody("{\"personNames\":[],\"personRole\":\"any\",\"dateFrom\":null,\"dateTo\":null,\"keywords\":[],\"rawQuery\":\"q\"}"))); - - assertThatThrownBy(() -> client.parse("some query", "de")) - .isInstanceOf(DomainException.class) - .extracting(e -> ((DomainException) e).getCode()) - .isEqualTo(ErrorCode.SMART_SEARCH_UNAVAILABLE); - } - - @Test - void isHealthy_returnsTrue_whenPersonsLoadedIsPositive() { - wireMock.stubFor(get(urlEqualTo("/health")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody("{\"status\":\"ok\",\"persons_loaded\":42}"))); - - assertThat(client.isHealthy()).isTrue(); - } - - @Test - void isHealthy_returnsFalse_whenPersonsLoadedIsZero() { - wireMock.stubFor(get(urlEqualTo("/health")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody("{\"status\":\"ok\",\"persons_loaded\":0}"))); - - assertThat(client.isHealthy()).isFalse(); - } - - @Test - void isHealthy_returnsFalse_whenNlpServiceIsDown() { - wireMock.stubFor(get(urlEqualTo("/health")) - .willReturn(aResponse().withStatus(503))); - - assertThat(client.isHealthy()).isFalse(); - } -} diff --git a/docs/superpowers/plans/2026-06-07-remove-nlp-search.md b/docs/superpowers/plans/2026-06-07-remove-nlp-search.md new file mode 100644 index 00000000..635a4b66 --- /dev/null +++ b/docs/superpowers/plans/2026-06-07-remove-nlp-search.md @@ -0,0 +1,768 @@ +# Remove NLP/Smart Search Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove the NLP/smart-search feature entirely from the codebase — backend search package, frontend components, i18n keys, infrastructure config, and the nlp-service microservice. + +**Architecture:** Pure deletion + targeted edits. No new code. Each task deletes a self-contained layer, then verifies compilation passes before committing. Order: backend first (most isolated), then frontend, then infrastructure, then docs. + +**Tech Stack:** Spring Boot 4 (Java 21, Maven), SvelteKit 2 / Svelte 5, Docker Compose, Paraglide i18n. + +--- + +## File Map + +### Delete entirely +- `backend/src/main/java/org/raddatz/familienarchiv/search/` — 14 Java source files +- `backend/src/test/java/org/raddatz/familienarchiv/search/` — 6 Java test files +- `frontend/src/routes/search/SmartModeToggle.svelte` + `.spec.ts` +- `frontend/src/routes/search/SmartSearchStatus.svelte` + `.spec.ts` +- `frontend/src/routes/search/InterpretationChipRow.svelte` + `.spec.ts` +- `frontend/src/routes/search/DisambiguationPicker.svelte` + `.spec.ts` +- `frontend/src/routes/search/chip-types.ts` +- `frontend/src/routes/documents/theme-chip-removal.ts` + `.spec.ts` +- `infra/observability/grafana/provisioning/dashboards/ollama.json` +- `nlp-service/` (entire directory) +- `docs/adr/028-nl-search-ollama.md` +- `docs/adr/028-ollama-docker-compose-service.md` +- `docs/adr/034-ollama-production-deployment-and-keep-alive.md` +- `docs/adr/035-rule-based-nlp-service.md` + +### Modify +- `backend/src/main/java/org/raddatz/familienarchiv/exception/ErrorCode.java` — remove 2 enum values +- `backend/src/main/resources/application.yaml` — remove `nlp` + `nl-search` config blocks +- `backend/src/main/resources/application-dev.yaml` — remove `nlp` config block +- `frontend/src/routes/SearchFilterBar.svelte` — remove SmartModeToggle, smartMode prop, smart callbacks +- `frontend/src/routes/SearchFilterBar.svelte.spec.ts` — remove smart-mode describe block +- `frontend/src/routes/documents/+page.svelte` — remove all NL state, functions, template block +- `frontend/src/lib/shared/errors.ts` — remove 2 error codes + their switch cases +- `frontend/messages/de.json` — remove 8 smart-search keys +- `frontend/messages/en.json` — remove 8 smart-search keys +- `frontend/messages/es.json` — remove 8 smart-search keys +- `docker-compose.yml` — remove nlp-service block + backend depends_on + env var +- `docker-compose.prod.yml` — remove nlp-service block + backend depends_on + env var +- `infra/observability/prometheus/prometheus.yml` — remove ollama scrape job +- `CLAUDE.md` — remove search package reference + error code entries +- `backend/CLAUDE.md` — no change needed (search package already absent from structure) +- `frontend/CLAUDE.md` — update routes/search/ description + +--- + +### Task 1: Delete backend search package + +**Files:** +- Delete: `backend/src/main/java/org/raddatz/familienarchiv/search/` (14 files) +- Delete: `backend/src/test/java/org/raddatz/familienarchiv/search/` (6 files) + +- [ ] **Step 1: Delete all source files** + +```bash +rm -rf backend/src/main/java/org/raddatz/familienarchiv/search +rm -rf backend/src/test/java/org/raddatz/familienarchiv/search +``` + +- [ ] **Step 2: Verify backend compiles** + +```bash +cd backend && . ~/.sdkman/candidates/java/current/bin/../.. && source ~/.sdkman/bin/sdkman-init.sh && ./mvnw compile -q +``` + +Expected: BUILD SUCCESS with no errors. + +- [ ] **Step 3: Commit** + +```bash +git add -A +git commit -m "refactor(search): delete backend NLP search package" +``` + +--- + +### Task 2: Remove ErrorCode entries and backend config + +**Files:** +- Modify: `backend/src/main/java/org/raddatz/familienarchiv/exception/ErrorCode.java:138-142` +- Modify: `backend/src/main/resources/application.yaml:133-138` +- Modify: `backend/src/main/resources/application-dev.yaml:15-17` + +- [ ] **Step 1: Remove NL Search enum values from ErrorCode.java** + +Remove these lines (138–142): +```java + // --- NL Search --- + /** Ollama is unreachable or timed out. 503 */ + SMART_SEARCH_UNAVAILABLE, + /** NL search rate limit exceeded (5 requests per user per minute). 429 */ + SMART_SEARCH_RATE_LIMITED, +``` + +The block between `TAG_MERGE_INVALID_TARGET,` and `// --- Generic ---` becomes empty. + +- [ ] **Step 2: Remove nlp and nl-search config from application.yaml** + +Remove these lines (133–138): +```yaml + nlp: + base-url: http://nlp-service:8001 + + nl-search: + rate-limit: + max-requests-per-minute: 20 +``` + +- [ ] **Step 3: Remove nlp config from application-dev.yaml** + +Remove these lines (15–17): +```yaml +app: + nlp: + base-url: http://localhost:8001 +``` + +Note: only remove the `nlp:` sub-key under `app:`, preserving any other `app:` config above it. + +- [ ] **Step 4: Verify backend still compiles** + +```bash +cd backend && source ~/.sdkman/bin/sdkman-init.sh && ./mvnw compile -q +``` + +Expected: BUILD SUCCESS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/src/main/java/org/raddatz/familienarchiv/exception/ErrorCode.java \ + backend/src/main/resources/application.yaml \ + backend/src/main/resources/application-dev.yaml +git commit -m "refactor(search): remove NLP error codes and application config" +``` + +--- + +### Task 3: Delete frontend NL search components and utilities + +**Files:** +- Delete: `frontend/src/routes/search/SmartModeToggle.svelte` + `.spec.ts` +- Delete: `frontend/src/routes/search/SmartSearchStatus.svelte` + `.spec.ts` +- Delete: `frontend/src/routes/search/InterpretationChipRow.svelte` + `.spec.ts` +- Delete: `frontend/src/routes/search/DisambiguationPicker.svelte` + `.spec.ts` +- Delete: `frontend/src/routes/search/chip-types.ts` +- Delete: `frontend/src/routes/documents/theme-chip-removal.ts` + `.spec.ts` + +- [ ] **Step 1: Delete all NL search components, specs, and utilities** + +```bash +rm frontend/src/routes/search/SmartModeToggle.svelte \ + frontend/src/routes/search/SmartModeToggle.svelte.spec.ts \ + frontend/src/routes/search/SmartSearchStatus.svelte \ + frontend/src/routes/search/SmartSearchStatus.svelte.spec.ts \ + frontend/src/routes/search/InterpretationChipRow.svelte \ + frontend/src/routes/search/InterpretationChipRow.svelte.spec.ts \ + frontend/src/routes/search/DisambiguationPicker.svelte \ + frontend/src/routes/search/DisambiguationPicker.svelte.spec.ts \ + frontend/src/routes/search/chip-types.ts \ + frontend/src/routes/documents/theme-chip-removal.ts \ + frontend/src/routes/documents/theme-chip-removal.spec.ts +``` + +- [ ] **Step 2: Commit** + +```bash +git add -A +git commit -m "refactor(search): delete frontend NLP search components and utilities" +``` + +--- + +### Task 4: Remove NL search from SearchFilterBar + +**Files:** +- Modify: `frontend/src/routes/SearchFilterBar.svelte` +- Modify: `frontend/src/routes/SearchFilterBar.svelte.spec.ts:199-233` + +- [ ] **Step 1: Rewrite SearchFilterBar.svelte** + +Replace the entire ` +``` + +- [ ] **Step 2: Update the search input element in the template** + +Replace the `` element (lines 92–105) with: +```svelte + +``` + +- [ ] **Step 3: Remove the SmartModeToggle component from the template** + +Delete this line (135): +```svelte + +``` + +- [ ] **Step 4: Remove smart-mode describe block from SearchFilterBar.svelte.spec.ts** + +Delete lines 199–233 (the entire final `describe` block): +```typescript +describe('SearchFilterBar – smart-mode chip lifecycle hooks', () => { + // ... +}); +``` + +- [ ] **Step 5: Run the SearchFilterBar tests to verify they pass** + +```bash +cd frontend && source ~/.nvm/nvm.sh && npm run test -- --project=client src/routes/SearchFilterBar.svelte.spec.ts +``` + +Expected: all tests pass, no failures. + +- [ ] **Step 6: Commit** + +```bash +git add frontend/src/routes/SearchFilterBar.svelte \ + frontend/src/routes/SearchFilterBar.svelte.spec.ts +git commit -m "refactor(search): remove smart mode from SearchFilterBar" +``` + +--- + +### Task 5: Remove NL search from documents/+page.svelte + +**Files:** +- Modify: `frontend/src/routes/documents/+page.svelte` + +This is the largest edit. Remove all NL search state, derived values, functions, and the NL results template block. + +- [ ] **Step 1: Remove NL search imports (lines 11–16, 23–27)** + +Remove these import lines: +```typescript +import SmartSearchStatus from '../search/SmartSearchStatus.svelte'; +import InterpretationChipRow from '../search/InterpretationChipRow.svelte'; +import type { ChipType } from '../search/chip-types.js'; +import { buildThemeRemovalUrl } from './theme-chip-removal.js'; +import DisambiguationPicker from '../search/DisambiguationPicker.svelte'; +``` + +Remove these type aliases: +```typescript +type NlQueryInterpretation = components['schemas']['NlQueryInterpretation']; +type NlSearchResponse = components['schemas']['NlSearchResponse']; +type DocumentSearchResult = components['schemas']['DocumentSearchResult']; +type PersonHint = components['schemas']['PersonHint']; +type SmartSearchErrorCode = 'SMART_SEARCH_UNAVAILABLE' | 'SMART_SEARCH_RATE_LIMITED'; +``` + +Also remove the `import { csrfFetch } from '$lib/shared/cookies';` line — it is only used by `runSmartSearch`. + +- [ ] **Step 2: Remove all NL state and derived values (lines 51–70)** + +Remove these declarations: +```typescript +// Smart (NL) search — UI-local state, resets on real page navigation (away + back). +let smartMode = $state(false); +let nlSubmitted = $state(false); +let nlLoading = $state(false); +let nlError = $state(null); +let nlInterpretation = $state(null); +let nlResult = $state(null); + +const showNlView = $derived(smartMode && nlSubmitted); +const nlHasResults = $derived((nlResult?.items.length ?? 0) > 0); +const ambiguousPersons = $derived(nlInterpretation?.ambiguousPersons ?? []); +const nlIsAmbiguous = $derived(ambiguousPersons.length > 0); +const disambiguationHeading = $derived( + ambiguousPersons.length === 1 + ? m.search_disambiguation_did_you_mean({ name: ambiguousPersons[0].displayName }) + : m.search_disambiguation_heading() +); +const showDisambiguationCue = $derived(ambiguousPersons.length >= 2); +``` + +- [ ] **Step 3: Remove all NL search functions (lines 202–318)** + +Remove these functions entirely: +- `resetNlState()` +- `onModeToggle()` +- `runSmartSearch()` +- `switchToKeywordMode()` +- `applyResolvedAndSearch()` +- `paramsFromInterpretation()` +- `removeChip()` +- `selectDisambiguated()` + +- [ ] **Step 4: Update SearchFilterBar usage in the template** + +Replace the SearchFilterBar call with (removing `bind:smartMode`, `onSmartSearch`, `onModeToggle`): +```svelte + (qFocused = true)} + onblur={() => (qFocused = false)} + /> +``` + +- [ ] **Step 5: Remove the NL results template block** + +Replace the entire `{#if showNlView}...{:else}...{/if}` block with just the content of the `{:else}` branch — the ` diff --git a/frontend/src/routes/SearchFilterBar.svelte.spec.ts b/frontend/src/routes/SearchFilterBar.svelte.spec.ts index 1de166f2..446cd046 100644 --- a/frontend/src/routes/SearchFilterBar.svelte.spec.ts +++ b/frontend/src/routes/SearchFilterBar.svelte.spec.ts @@ -195,39 +195,3 @@ describe('SearchFilterBar – tagQ live filter', () => { vi.unstubAllGlobals(); }); }); - -describe('SearchFilterBar – smart-mode chip lifecycle hooks', () => { - // The interpretation chips live in the result area (parent page). SearchFilterBar - // drives chip-clearing through callbacks: onModeToggle (mode switch) and - // onSmartSearch (new query). These tests pin that contract. - it('invokes onModeToggle when toggling back to keyword mode (parent clears chips)', async () => { - const onModeToggle = vi.fn(); - render(SearchFilterBar, { - ...defaultProps, - sort: 'DATE', - dir: 'desc', - smartMode: true, - onModeToggle - }); - await page.getByRole('button', { name: /Smart/ }).click(); - expect(onModeToggle).toHaveBeenCalledOnce(); - }); - - it('invokes onSmartSearch when a new query is submitted in smart mode (parent resets chips)', async () => { - const onSmartSearch = vi.fn(); - render(SearchFilterBar, { - ...defaultProps, - sort: 'DATE', - dir: 'desc', - smartMode: true, - onSmartSearch - }); - const input = page.getByPlaceholder('Titel, Personen, Tags durchsuchen…'); - await input.fill('Walter im Krieg'); - await input.click(); - (document.activeElement as HTMLElement).dispatchEvent( - new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }) - ); - await vi.waitFor(() => expect(onSmartSearch).toHaveBeenCalled()); - }); -}); -- 2.49.1 From 1dd40721feccf05d9522f46abd6c3caa81c68e46 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 18:58:36 +0200 Subject: [PATCH 41/51] refactor(search): remove NLP smart search from documents page Co-Authored-By: Claude Sonnet 4.6 --- frontend/src/routes/documents/+page.svelte | 326 ++++----------------- 1 file changed, 60 insertions(+), 266 deletions(-) diff --git a/frontend/src/routes/documents/+page.svelte b/frontend/src/routes/documents/+page.svelte index ad3744f5..00abe3cd 100644 --- a/frontend/src/routes/documents/+page.svelte +++ b/frontend/src/routes/documents/+page.svelte @@ -8,23 +8,9 @@ import DocumentList from '../DocumentList.svelte'; import Pagination from '$lib/shared/primitives/Pagination.svelte'; import BulkSelectionBar from '$lib/document/BulkSelectionBar.svelte'; import TimelineDensityFilter from '$lib/document/TimelineDensityFilter.svelte'; -import SmartSearchStatus from '../search/SmartSearchStatus.svelte'; -import InterpretationChipRow from '../search/InterpretationChipRow.svelte'; -import type { ChipType } from '../search/chip-types.js'; -import { buildThemeRemovalUrl } from './theme-chip-removal.js'; -import DisambiguationPicker from '../search/DisambiguationPicker.svelte'; import { bulkSelectionStore } from '$lib/document/bulkSelection.svelte'; import { getErrorMessage, parseBackendError } from '$lib/shared/errors'; -import { csrfFetch } from '$lib/shared/cookies'; import * as m from '$lib/paraglide/messages.js'; -import { getLocale } from '$lib/paraglide/runtime'; -import type { components } from '$lib/generated/api'; - -type NlQueryInterpretation = components['schemas']['NlQueryInterpretation']; -type NlSearchResponse = components['schemas']['NlSearchResponse']; -type DocumentSearchResult = components['schemas']['DocumentSearchResult']; -type PersonHint = components['schemas']['PersonHint']; -type SmartSearchErrorCode = 'SMART_SEARCH_UNAVAILABLE' | 'SMART_SEARCH_RATE_LIMITED'; let { data } = $props(); @@ -48,27 +34,6 @@ let tagQ = $state(untrack(() => data.tagQ || '')); let tagOperator = $state<'AND' | 'OR'>(untrack(() => (data.tagOp as 'AND' | 'OR') || 'AND')); let undated = $state(untrack(() => data.undated ?? false)); -// Smart (NL) search — UI-local state, resets on real page navigation (away + back). -let smartMode = $state(false); -let nlSubmitted = $state(false); -let nlLoading = $state(false); -let nlError = $state(null); -let nlInterpretation = $state(null); -let nlResult = $state(null); - -const showNlView = $derived(smartMode && nlSubmitted); -const nlHasResults = $derived((nlResult?.items.length ?? 0) > 0); -const ambiguousPersons = $derived(nlInterpretation?.ambiguousPersons ?? []); -const nlIsAmbiguous = $derived(ambiguousPersons.length > 0); -// A 1-item picker is always a "did you mean …?" suggestion (a single direct match auto-selects -// and never reaches the picker); ≥2 keeps the "choose a person" framing and the action cue. -const disambiguationHeading = $derived( - ambiguousPersons.length === 1 - ? m.search_disambiguation_did_you_mean({ name: ambiguousPersons[0].displayName }) - : m.search_disambiguation_heading() -); -const showDisambiguationCue = $derived(ambiguousPersons.length >= 2); - function hasAdvancedFilters() { return ( (data.tags?.length ?? 0) > 0 || @@ -199,124 +164,6 @@ function handleImmediateSearch() { triggerSearchKeepZoom(); } -function resetNlState() { - nlSubmitted = false; - nlLoading = false; - nlError = null; - nlInterpretation = null; - nlResult = null; -} - -/** Toggling the mode (either direction) always clears any prior NL interpretation. */ -function onModeToggle() { - resetNlState(); -} - -/** Submit the natural-language query to the server-side parser. */ -async function runSmartSearch() { - const query = q.trim(); - if (query.length < 3) return; - nlSubmitted = true; - nlLoading = true; - nlError = null; - nlInterpretation = null; - nlResult = null; - try { - const res = await csrfFetch('/api/search/nl', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ query, lang: getLocale() }) - }); - if (!res.ok) { - const backend = await parseBackendError(res); - nlError = - backend?.code === 'SMART_SEARCH_RATE_LIMITED' - ? 'SMART_SEARCH_RATE_LIMITED' - : 'SMART_SEARCH_UNAVAILABLE'; - return; - } - const body: NlSearchResponse = await res.json(); - nlInterpretation = body.interpretation; - nlResult = body.result; - } catch { - nlError = 'SMART_SEARCH_UNAVAILABLE'; - } finally { - nlLoading = false; - } -} - -/** Option A empty/error fallback: drop NL mode, keep the raw query, run a keyword search. */ -function switchToKeywordMode() { - resetNlState(); - smartMode = false; - handleImmediateSearch(); -} - -/** Applies a resolved param set to the keyword filters and re-runs via GET. */ -function applyResolvedAndSearch(p: { - senderId: string; - receiverId: string; - from: string; - to: string; - q: string; -}) { - resetNlState(); - smartMode = false; - senderId = p.senderId; - receiverId = p.receiverId; - from = p.from; - to = p.to; - q = p.q; - handleImmediateSearch(); -} - -function paramsFromInterpretation(interp: NlQueryInterpretation) { - const resolved = interp.resolvedPersons; - return { - senderId: resolved.length >= 1 ? resolved[0].id : '', - receiverId: resolved.length >= 2 ? resolved[1].id : '', - from: interp.dateFrom ?? '', - to: interp.dateTo ?? '', - q: interp.keywordsApplied ? interp.keywords.join(' ') : '' - }; -} - -function removeChip(type: ChipType, value?: string) { - if (!nlInterpretation) return; - const p = paramsFromInterpretation(nlInterpretation); - if (type === 'sender') { - p.senderId = ''; - } else if (type === 'directional') { - p.senderId = ''; - p.receiverId = ''; - } else if (type === 'date') { - p.from = ''; - p.to = ''; - } else if (type === 'keyword' && value) { - const remaining = nlInterpretation.keywords.filter((keyword) => keyword !== value); - p.q = remaining.join(' '); - } else if (type === 'theme' && value) { - const url = buildThemeRemovalUrl(nlInterpretation, value); - resetNlState(); - goto(url, { keepFocus: true, noScroll: true }); - return; - } - applyResolvedAndSearch(p); -} - -/** Single-select disambiguation: resolved person becomes sender, chosen becomes receiver. */ -function selectDisambiguated(person: PersonHint) { - if (!nlInterpretation) return; - const resolved = nlInterpretation.resolvedPersons; - applyResolvedAndSearch({ - senderId: resolved.length >= 1 ? resolved[0].id : person.id, - receiverId: resolved.length >= 1 ? person.id : '', - from: nlInterpretation.dateFrom ?? '', - to: nlInterpretation.dateTo ?? '', - q: nlInterpretation.keywordsApplied ? nlInterpretation.keywords.join(' ') : '' - }); -} - // Trigger search reactively when the tag list changes. let prevTagStr = untrack(() => tagNames.map((t) => t.name).join(',')); $effect(() => { @@ -421,7 +268,6 @@ $effect(() => { bind:tagQ={tagQ} bind:tagOperator={tagOperator} bind:undated={undated} - bind:smartMode={smartMode} undatedCount={data.undatedCount ?? 0} initialSenderName={initialSenderName} initialReceiverName={initialReceiverName} @@ -429,71 +275,20 @@ $effect(() => { isLoading={navigating.to !== null} onSearch={handleTextSearch} onSearchImmediate={handleImmediateSearch} - onSmartSearch={runSmartSearch} - onModeToggle={onModeToggle} onfocus={() => (qFocused = true)} onblur={() => (qFocused = false)} /> - {#if showNlView} - -
- {#if nlLoading} - - {:else if nlError} - - {:else if nlInterpretation} - {#key nlInterpretation} -
- {#if nlIsAmbiguous} - - {:else} - - {/if} -
- - {#if !nlIsAmbiguous} - {#if nlHasResults} -

- {m.docs_result_count({ count: nlResult?.totalElements ?? 0 })} -

- - {:else} -
-

{m.search_empty_nl()}

- -
- {/if} - {/if} - {/key} - {/if} -
- {:else} - -
-

- {#if data.totalElements > 0}{m.docs_result_count({ count: data.totalElements })}{/if} -

- {#if data.canWrite} -
-
- {#if data.totalElements > 0} - - {/if} - +

+ {#if data.totalElements > 0}{m.docs_result_count({ count: data.totalElements })}{/if} +

+ {#if data.canWrite} +
+
+ {#if data.totalElements > 0} +
- {#if editAllError} - + {m.bulk_edit_all_x({ count: data.totalElements })} + {/if} + + + {m.docs_btn_new()} +
- {/if} -
+ {#if editAllError} + + {/if} +
+ {/if} +
- + - - {/if} + -- 2.49.1 From c139f2969ef41d36e51571158d104c8b71592d96 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 18:59:47 +0200 Subject: [PATCH 42/51] refactor(search): remove smart search error codes from frontend --- frontend/src/lib/shared/errors.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/frontend/src/lib/shared/errors.ts b/frontend/src/lib/shared/errors.ts index 4bff75e6..6efec2a7 100644 --- a/frontend/src/lib/shared/errors.ts +++ b/frontend/src/lib/shared/errors.ts @@ -53,8 +53,6 @@ export type ErrorCode = | 'FORBIDDEN' | 'CSRF_TOKEN_MISSING' | 'TOO_MANY_LOGIN_ATTEMPTS' - | 'SMART_SEARCH_UNAVAILABLE' - | 'SMART_SEARCH_RATE_LIMITED' | 'VALIDATION_ERROR' | 'BATCH_TOO_LARGE' | 'BULK_EDIT_TOO_MANY_IDS' @@ -180,10 +178,6 @@ export function getErrorMessage(code: ErrorCode | string | undefined): string { return m.error_csrf_token_missing(); case 'TOO_MANY_LOGIN_ATTEMPTS': return m.error_too_many_login_attempts(); - case 'SMART_SEARCH_UNAVAILABLE': - return m.error_smart_search_unavailable(); - case 'SMART_SEARCH_RATE_LIMITED': - return m.error_smart_search_rate_limited(); case 'VALIDATION_ERROR': return m.error_validation_error(); case 'BATCH_TOO_LARGE': -- 2.49.1 From a8a5130b02e36cf965e5ebfcd279167a2acf5911 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 19:01:17 +0200 Subject: [PATCH 43/51] refactor(search): remove smart search i18n keys from all language files --- frontend/messages/de.json | 15 --------------- frontend/messages/en.json | 15 --------------- frontend/messages/es.json | 15 --------------- 3 files changed, 45 deletions(-) diff --git a/frontend/messages/de.json b/frontend/messages/de.json index d8afe368..eaa970eb 100644 --- a/frontend/messages/de.json +++ b/frontend/messages/de.json @@ -22,22 +22,9 @@ "error_forbidden": "Sie haben keine Berechtigung für diese Aktion.", "error_csrf_token_missing": "Sitzungsfehler. Bitte laden Sie die Seite neu.", "error_too_many_login_attempts": "Zu viele Anmeldeversuche. Bitte versuchen Sie es später erneut.", - "error_smart_search_unavailable": "Die intelligente Suche ist momentan nicht verfügbar. Bitte nutzen Sie die normale Suche.", - "error_smart_search_rate_limited": "Sie haben die Suchfunktion zu häufig genutzt. Bitte warten Sie eine Minute.", - "smart_search_keywords_not_applied": "Schlüsselwörter konnten bei dieser Suche nicht berücksichtigt werden.", - "search_toggle_smart_label": "Smart", - "search_toggle_smart_label_suffix": "-Suche", - "search_toggle_keyword_label": "Text", - "search_toggle_keyword_label_suffix": "suche", "search_loading_nl": "Archiv wird befragt…", "search_loading_nl_sub": "Die Anfrage wird analysiert…", - "search_error_unavailable": "Intelligente Suche nicht verfügbar", - "search_error_unavailable_body": "Die intelligente Suche ist momentan nicht erreichbar. Sie können Ihre Anfrage als einfache Volltextsuche wiederholen.", "search_switch_to_keyword": "Zur Volltextsuche wechseln", - "search_error_rate_limited": "Zu viele Anfragen", - "search_error_rate_limited_body": "Sie haben die intelligente Suche zu häufig genutzt. Bitte warten Sie eine Minute und versuchen Sie es erneut.", - "search_empty_nl": "Keine Ergebnisse", - "search_empty_retry_keyword": "Als Volltextsuche wiederholen", "search_filter_remove_label": "Filter entfernen: {label}", "search_chip_sender": "Absender", "search_chip_date": "Zeitraum", @@ -46,8 +33,6 @@ "search_chip_directional_label": "Von {from} zu {to}, Filter entfernen", "search_disambiguation_trigger_label": "Mehrere Personen gefunden — zum Auswählen klicken", "search_disambiguation_cue": "(auswählen…)", - "search_disambiguation_heading": "Person auswählen", - "search_disambiguation_did_you_mean": "Meintest du {name}?", "search_disambiguation_select_label": "{name} auswählen", "error_validation_error": "Die Eingabe ist ungültig.", "error_internal_error": "Ein unerwarteter Fehler ist aufgetreten.", diff --git a/frontend/messages/en.json b/frontend/messages/en.json index e473d770..1534ebc7 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -22,22 +22,9 @@ "error_forbidden": "You do not have permission for this action.", "error_csrf_token_missing": "Session error. Please reload the page.", "error_too_many_login_attempts": "Too many login attempts. Please try again later.", - "error_smart_search_unavailable": "The smart search is currently unavailable. Please use the regular search.", - "error_smart_search_rate_limited": "You have used the search function too frequently. Please wait a minute.", - "smart_search_keywords_not_applied": "Keywords could not be applied to this search.", - "search_toggle_smart_label": "Smart", - "search_toggle_smart_label_suffix": " search", - "search_toggle_keyword_label": "Text", - "search_toggle_keyword_label_suffix": " search", "search_loading_nl": "Querying the archive…", "search_loading_nl_sub": "Your request is being analysed…", - "search_error_unavailable": "Smart search unavailable", - "search_error_unavailable_body": "The smart search is currently unreachable. You can repeat your request as a plain full-text search.", "search_switch_to_keyword": "Switch to full-text search", - "search_error_rate_limited": "Too many requests", - "search_error_rate_limited_body": "You have used the smart search too frequently. Please wait a minute and try again.", - "search_empty_nl": "No results", - "search_empty_retry_keyword": "Repeat as full-text search", "search_filter_remove_label": "Remove filter: {label}", "search_chip_sender": "Sender", "search_chip_date": "Period", @@ -46,8 +33,6 @@ "search_chip_directional_label": "From {from} to {to}, remove filter", "search_disambiguation_trigger_label": "Several people found — click to choose", "search_disambiguation_cue": "(choose…)", - "search_disambiguation_heading": "Choose a person", - "search_disambiguation_did_you_mean": "Did you mean {name}?", "search_disambiguation_select_label": "Select {name}", "error_validation_error": "The input is invalid.", "error_internal_error": "An unexpected error occurred.", diff --git a/frontend/messages/es.json b/frontend/messages/es.json index 5337f39c..60f7a2a8 100644 --- a/frontend/messages/es.json +++ b/frontend/messages/es.json @@ -22,22 +22,9 @@ "error_forbidden": "No tiene permiso para realizar esta acción.", "error_csrf_token_missing": "Error de sesión. Recargue la página.", "error_too_many_login_attempts": "Demasiados intentos. Por favor, inténtelo más tarde.", - "error_smart_search_unavailable": "La búsqueda inteligente no está disponible en este momento. Por favor, usa la búsqueda normal.", - "error_smart_search_rate_limited": "Has utilizado la función de búsqueda demasiadas veces. Por favor, espera un minuto.", - "smart_search_keywords_not_applied": "Las palabras clave no pudieron aplicarse a esta búsqueda.", - "search_toggle_smart_label": "búsqueda inteligente", - "search_toggle_smart_label_suffix": "", - "search_toggle_keyword_label": "Texto", - "search_toggle_keyword_label_suffix": " búsqueda", "search_loading_nl": "Consultando el archivo…", "search_loading_nl_sub": "Su solicitud está siendo analizada…", - "search_error_unavailable": "Búsqueda inteligente no disponible", - "search_error_unavailable_body": "La búsqueda inteligente no está disponible en este momento. Puede repetir su solicitud como una búsqueda de texto completo.", "search_switch_to_keyword": "Cambiar a búsqueda de texto completo", - "search_error_rate_limited": "Demasiadas solicitudes", - "search_error_rate_limited_body": "Ha utilizado la búsqueda inteligente con demasiada frecuencia. Espere un minuto e inténtelo de nuevo.", - "search_empty_nl": "Sin resultados", - "search_empty_retry_keyword": "Repetir como búsqueda de texto completo", "search_filter_remove_label": "Eliminar filtro: {label}", "search_chip_sender": "Remitente", "search_chip_date": "Período", @@ -46,8 +33,6 @@ "search_chip_directional_label": "De {from} a {to}, eliminar filtro", "search_disambiguation_trigger_label": "Se encontraron varias personas — haga clic para elegir", "search_disambiguation_cue": "(elegir…)", - "search_disambiguation_heading": "Elegir una persona", - "search_disambiguation_did_you_mean": "¿Quería decir {name}?", "search_disambiguation_select_label": "Seleccionar {name}", "error_validation_error": "La entrada no es válida.", "error_internal_error": "Se ha producido un error inesperado.", -- 2.49.1 From f226f31fee3ca4f0638bd1cadc7396798e962480 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 19:02:17 +0200 Subject: [PATCH 44/51] refactor(infra): remove nlp-service from docker-compose files --- docker-compose.prod.yml | 36 ------------------------------------ docker-compose.yml | 39 --------------------------------------- 2 files changed, 75 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9a328b36..26e07442 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -200,39 +200,6 @@ services: security_opt: - no-new-privileges:true - # --- NLP service: rule-based NL query parser --- - # Lightweight FastAPI service; replaces Ollama for smart search query parsing. - # Connects to the DB at startup to build person/tag lookup tables. - nlp-service: - build: - context: ./nlp-service - restart: unless-stopped - expose: - - "8001" - networks: - - archiv-net - environment: - DATABASE_URL: "postgresql://archiv:${POSTGRES_PASSWORD}@db:5432/archiv" - NLP_FUZZY_THRESHOLD: "${NLP_FUZZY_THRESHOLD:-80}" - mem_limit: 256m - memswap_limit: 256m - read_only: true - tmpfs: - - /tmp:size=32m - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8001/health"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 15s - depends_on: - db: - condition: service_healthy - backend: image: familienarchiv/backend:${TAG:-nightly} build: @@ -251,8 +218,6 @@ services: # is a one-shot that must complete successfully. See #510. create-buckets: condition: service_completed_successfully - nlp-service: - condition: service_healthy # Bound to localhost only — Caddy fronts external traffic. ports: - "127.0.0.1:${PORT_BACKEND}:8080" @@ -287,7 +252,6 @@ services: APP_ADMIN_PASSWORD: ${APP_ADMIN_PASSWORD} APP_OCR_BASE_URL: http://ocr-service:8000 APP_OCR_TRAINING_TOKEN: ${OCR_TRAINING_TOKEN} - APP_NLP_BASE_URL: http://nlp-service:8001 MAIL_HOST: ${MAIL_HOST} MAIL_PORT: ${MAIL_PORT:-587} MAIL_USERNAME: ${MAIL_USERNAME:-} diff --git a/docker-compose.yml b/docker-compose.yml index 4de4245d..74f1bd3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,42 +141,6 @@ services: security_opt: - no-new-privileges:true - # --- NLP service: rule-based NL query parser --- - # FastAPI Python service; replaces Ollama for smart search query parsing. - # Not started in CI — CI uses explicit service selection - # (docker-compose.ci.yml: db minio create-buckets) - nlp-service: - build: - context: ./nlp-service - dockerfile: Dockerfile - container_name: archive-nlp - restart: unless-stopped - expose: - - "8001" - networks: - - archiv-net - environment: - DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}" - NLP_FUZZY_THRESHOLD: "${NLP_FUZZY_THRESHOLD:-80}" - mem_limit: 256m - memswap_limit: 256m - read_only: true - tmpfs: - - /tmp:size=32m - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8001/health"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 15s - depends_on: - db: - condition: service_healthy - # --- Backend: Spring Boot --- backend: build: @@ -195,8 +159,6 @@ services: condition: service_started ocr-service: condition: service_started - nlp-service: - condition: service_healthy environment: SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${POSTGRES_DB} SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER} @@ -222,7 +184,6 @@ services: SPRING_MAIL_PROPERTIES_MAIL_SMTP_STARTTLS_ENABLE: ${MAIL_STARTTLS_ENABLE:-false} APP_OCR_BASE_URL: http://ocr-service:8000 APP_OCR_TRAINING_TOKEN: "${OCR_TRAINING_TOKEN:-}" - APP_NLP_BASE_URL: "http://nlp-service:8001" SENTRY_DSN: ${SENTRY_DSN:-} SENTRY_TRACES_SAMPLE_RATE: ${SENTRY_TRACES_SAMPLE_RATE:-1.0} # Observability: send traces to Tempo inside archiv-net (OTLP gRPC port 4317) -- 2.49.1 From 894e92327e426e5765776e593272129a162b1d5d Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 19:02:56 +0200 Subject: [PATCH 45/51] refactor(infra): remove Ollama/NLP observability config --- .../provisioning/dashboards/ollama.json | 218 ------------------ infra/observability/prometheus/prometheus.yml | 5 - 2 files changed, 223 deletions(-) delete mode 100644 infra/observability/grafana/provisioning/dashboards/ollama.json diff --git a/infra/observability/grafana/provisioning/dashboards/ollama.json b/infra/observability/grafana/provisioning/dashboards/ollama.json deleted file mode 100644 index 47536e2d..00000000 --- a/infra/observability/grafana/provisioning/dashboards/ollama.json +++ /dev/null @@ -1,218 +0,0 @@ -{ - "id": null, - "uid": "ollama-dashboard", - "title": "Ollama", - "description": "Ollama inference latency and request rate", - "version": 1, - "schemaVersion": 39, - "tags": ["ollama", "inference"], - "timezone": "browser", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "liveNow": false, - "refresh": "30s", - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "weekStart": "", - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { "type": "datasource", "uid": "grafana" }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "panels": [ - { - "id": 1, - "type": "timeseries", - "title": "Inference Latency p50", - "description": "50th percentile of Ollama request duration over a 5-minute window", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 80 } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "single", "sort": "none" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, rate(ollama_request_duration_seconds_bucket[5m]))", - "instant": false, - "legendFormat": "p50", - "range": true, - "refId": "A" - } - ] - }, - { - "id": 2, - "type": "timeseries", - "title": "Inference Latency p95", - "description": "95th percentile of Ollama request duration over a 5-minute window", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 80 } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "single", "sort": "none" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, rate(ollama_request_duration_seconds_bucket[5m]))", - "instant": false, - "legendFormat": "p95", - "range": true, - "refId": "A" - } - ] - }, - { - "id": 3, - "type": "timeseries", - "title": "Request Rate", - "description": "Ollama requests per second over a 5-minute window", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 80 } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "single", "sort": "none" } - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "rate(ollama_requests_total[5m])", - "instant": false, - "legendFormat": "req/s", - "range": true, - "refId": "A" - } - ] - } - ], - "preload": false, - "templating": { - "list": [] - } -} diff --git a/infra/observability/prometheus/prometheus.yml b/infra/observability/prometheus/prometheus.yml index 53121566..ecffc410 100644 --- a/infra/observability/prometheus/prometheus.yml +++ b/infra/observability/prometheus/prometheus.yml @@ -22,8 +22,3 @@ scrape_configs: static_configs: - targets: ['ocr-service:8000'] - - job_name: ollama - metrics_path: /metrics - static_configs: - # Uses the Docker service name for reliable DNS resolution. - - targets: ['ollama:11434'] -- 2.49.1 From 3b54175945c8415127e5822651a846cba3f69d2a Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 19:04:00 +0200 Subject: [PATCH 46/51] refactor(search): delete nlp-service microservice and Ollama ADRs --- docs/adr/028-nl-search-ollama.md | 67 ---- docs/adr/028-ollama-docker-compose-service.md | 239 ------------- ...ma-production-deployment-and-keep-alive.md | 125 ------- docs/adr/035-rule-based-nlp-service.md | 105 ------ nlp-service/.dockerignore | 6 - nlp-service/CLAUDE.md | 59 --- nlp-service/Dockerfile | 24 -- nlp-service/extractor.py | 310 ---------------- nlp-service/main.py | 79 ----- nlp-service/models.py | 17 - nlp-service/person_matcher.py | 184 ---------- nlp-service/requirements.txt | 6 - nlp-service/test_extractor.py | 335 ------------------ nlp-service/test_main.py | 117 ------ nlp-service/test_sentences.md | 126 ------- 15 files changed, 1799 deletions(-) delete mode 100644 docs/adr/028-nl-search-ollama.md delete mode 100644 docs/adr/028-ollama-docker-compose-service.md delete mode 100644 docs/adr/034-ollama-production-deployment-and-keep-alive.md delete mode 100644 docs/adr/035-rule-based-nlp-service.md delete mode 100644 nlp-service/.dockerignore delete mode 100644 nlp-service/CLAUDE.md delete mode 100644 nlp-service/Dockerfile delete mode 100644 nlp-service/extractor.py delete mode 100644 nlp-service/main.py delete mode 100644 nlp-service/models.py delete mode 100644 nlp-service/person_matcher.py delete mode 100644 nlp-service/requirements.txt delete mode 100644 nlp-service/test_extractor.py delete mode 100644 nlp-service/test_main.py delete mode 100644 nlp-service/test_sentences.md diff --git a/docs/adr/028-nl-search-ollama.md b/docs/adr/028-nl-search-ollama.md deleted file mode 100644 index ab166acc..00000000 --- a/docs/adr/028-nl-search-ollama.md +++ /dev/null @@ -1,67 +0,0 @@ -# ADR-028 — Natural language search is powered by Ollama (Qwen 2.5 7B), not a cloud API - -**Date:** 2026-06-06 -**Status:** Accepted -**Issue:** #738 (NL search backend); part of epic #735 -**Milestone:** Archive Intelligence — NL Search - ---- - -## Context - -Family members write their search intent in plain German ("Was hat Walter im Krieg an Emma geschrieben?"), not in structured filter forms. Issue #735 defines NL search as a core product goal. Three delivery options were evaluated: - -**Option A — extend the OCR service.** The OCR Python microservice already runs on the same host. Adding LLM inference there avoids a new container. Rejected: the OCR service is a single-purpose, CPU-bound pipeline optimised for Kraken; bundling a 4.5 GB LLM weight into the same image would bloat it, complicate model lifecycle management, and create an unrelated failure domain (OOM on large OCR batches vs. LLM load time). ADR-001 was explicit about keeping OCR single-purpose. - -**Option B — call an external API (OpenAI, Anthropic, etc.).** Cloud inference is instant and requires no local hardware. Rejected: the archive contains real person names and private family correspondence from 1899–1950 — sending query content to a third party violates the project's data-residency principle (family data stays on the family server). Additionally, API cost and availability are outside the operator's control; the system must work air-gapped. - -**Option C — local Ollama service (chosen).** Ollama is a purpose-built LLM runtime with a simple REST API, model lifecycle management (`ollama pull`), and support for grammar-constrained JSON output. It runs entirely on the existing server (i7-6700, 64 GB RAM) with no cloud dependency. - -**Model selection:** Qwen 2.5 7B Q4_K_M (`qwen2.5:7b-instruct-q4_K_M`) was chosen over larger models because: -- Quantised weight is ~4.5 GB — fits comfortably in 64 GB RAM alongside PostgreSQL and the JVM. -- Instruction-tuned variant follows the structured JSON schema reliably without fine-tuning. -- CPU-only inference at Q4_K_M takes 2–15 seconds per query, acceptable for a search that replaces a multi-step filter form. - -**Prompt injection mitigation:** The backend sends the raw user query to Ollama. To prevent the model from being prompted to return schema-breaking output, the API call uses Ollama's `format` parameter with a grammar-constrained JSON schema. Output length is further bounded by `maxLength` constraints in the schema (names ≤ 200 chars, keywords ≤ 100 chars). `NlQueryParserService` enforces these limits in code before any LLM-extracted fragment is passed to `PersonRepository.searchByName()` — defence in depth. - -**DB-blind name resolution:** The Ollama prompt stays small (the raw query only); person database records are never sent to the model. Name resolution happens as a cheap SQL query after the model returns. This keeps the prompt short, avoids data leakage, and means adding 1,000 new persons requires no prompt change. - -**Graceful degradation:** In-path Ollama failures surface via `OllamaClient.parse()` — any `IOException`, read timeout, or non-2xx response is caught by `RestClientOllamaClient` and re-thrown as `DomainException(SMART_SEARCH_UNAVAILABLE, HTTP 503)`. `isHealthy()` has no callers inside `search/`; it is reserved for the ops/health-endpoint polling path only (e.g. a future `/api/health/ollama` endpoint). The regular structured search (`GET /api/documents/search`) is unaffected — it never calls Ollama. - -**Expected inference latency:** 2–15 seconds on the current CPU-only hardware. The frontend issue must show a persistent "Suche läuft…" indicator for the full duration (see `aria-live="polite"` requirement in issue #738 frontend notes). The backend timeout is 30 seconds (`app.ollama.timeout-seconds=30`) — chosen as a safe upper bound for Q4_K_M on the i7-6700 with a realistic 500-character query under modest concurrent load. - -**NL query logging policy:** Only metadata is logged — query length, resolved person count, latency in milliseconds. The raw query is never written to the log file. Rationale: queries contain real family names (PII); log files persist to disk and may be shipped to Loki. Structured metadata is sufficient for debugging latency regressions. - -**Prompt-amplification abuse:** A malicious user could submit a long or crafted query to cause slow Ollama inference, consuming CPU. Mitigated by `NlSearchRateLimiter` (5 requests per user per minute, Bucket4j + Caffeine) and by `@Size(max=500)` on the request body. The rate limiter is node-local; in multi-replica deployments the effective limit multiplies by replica count — acceptable at the current single-node deployment scale. - -**Ollama model pre-pull requirement:** The Docker image contains only the Ollama binary, not the model weights. The operator must run `ollama pull qwen2.5:7b-instruct-q4_K_M` (≈4.5 GB download, 10–30 minutes) before the backend starts inference. If skipped, every NL search request returns 503 until the pull completes. The deployment runbook in `docs/DEPLOYMENT.md` covers this explicitly. - -**Startup dependency:** The `backend` Compose service declares `depends_on: ollama: condition: service_healthy`. The Ollama healthcheck polls `GET http://localhost:11434/api/tags`; `start_period: 120s` provides margin for weight loading (20–60 s on SSD). Note: `service_healthy` confirms the API is responding, not that the model is downloaded — if the pull was skipped, inference still returns 404. - -**Multi-name resolution heuristic:** For 2-name queries (e.g. "Was hat Walter an Emma geschrieben?"), the first extracted name is treated as sender and the second as receiver. Per-name role annotation (e.g. `{name: "Walter", role: "sender"}`) was rejected because it would require a combinatorially complex Ollama schema and the most natural German phrasing strongly implies sender→receiver order. For single-name queries, a `personRole` field (`sender`/`receiver`/`any`) is returned. - -**`personRole: "any"` keyword limitation:** When `personRole` is `"any"` and the name resolves to exactly one person, `DocumentService.searchDocumentsByPersonId()` is called (OR semantics: person as sender or receiver). Keyword filtering is not applied on this path — only person identity and date range. `keywordsApplied = false` is returned in the response. Rationale: the JPQL for OR-semantics person queries has no text predicate; adding FTS would require a native query or a separate pass, adding complexity for a case that is already well-narrowed by person identity. - -**`search/` → `person/` + `document/` dependency direction:** `NlQueryParserService` calls `PersonService.findByDisplayNameContaining()` and `DocumentService.searchDocuments()` — both are legitimate cross-domain service calls, not repository leaks. The `search/` package has no JPA entities of its own and never accesses `PersonRepository` or `DocumentRepository` directly. - -**Keyword→tag resolution** (issue #743): After Ollama extracts the `keywords` list, `NlQueryParserService` calls `TagService.findByNameContaining()` for each keyword. Keywords that match one or more tags are removed from the FTS text list and added as OR-union tag filters; keywords with no tag match remain as FTS text. Resolved tags are returned to the frontend as `TagHint` objects in `NlQueryInterpretation.resolvedTags` and rendered as removable "Thema: X" chips. The `tagsApplied` flag signals whether the OR-union filter was actually passed to `DocumentService.searchDocuments()` — it is `false` when the `personRole:any` single-person path is taken, because that path has no tag filter slot. See ADR-033 for the tag name resolution and case-collision rules that `TagService.findByNameContaining()` relies on. - -## Decision - -**Introduce a new `search/` domain package** with a local Ollama integration via `RestClientOllamaClient`. The Ollama service runs as a separate Docker container, reachable only on the internal Docker network (`expose: ["11434"]`, not `ports:`). The backend calls Ollama's `/api/generate` endpoint with grammar-constrained JSON output. Name resolution and document search are performed by existing services after the model returns. - -Key component structure: -- `OllamaClient` / `OllamaHealthClient` interfaces — mockable for tests, modelled on `OcrClient`/`OcrHealthClient` -- `RestClientOllamaClient` — two `RestClient` instances (30 s inference, 2 s health-check) -- `NlQueryParserService` — orchestrates Ollama → name resolution → document search -- `NlSearchRateLimiter` — Bucket4j + Caffeine, 5 req/min per user -- `NlSearchController` — `POST /api/search/nl`, `@RequirePermission(READ_ALL)` - -## Consequences - -- Family members can query in natural German without learning filter UI. Expected search satisfaction improvement for the 60+ age cohort (primary transcription audience) is significant. -- NL search is unavailable when Ollama is down or the model pull is not complete. The regular search is unaffected. The 503 response includes a CTA directing users to the regular search. -- Operator responsibility: run `ollama pull` on first deploy and after model updates. The backup runbook must exclude `ollama_models` volume (model weights are re-downloadable, not user data). -- Inference takes 2–15 seconds. The frontend loading indicator is a hard requirement (see issue #738 frontend notes). -- The rate limiter is node-local. At the current single-node deployment scale this is correct. If the service is ever scaled horizontally, the rate limiter must be moved to Redis (same caveat as `LoginRateLimiter`). -- The `search/` package introduces a new cross-domain dependency direction (`search` → `person`, `search` → `document`). This is intentional and documented in `docs/architecture/c4/l3-backend-search.puml`. diff --git a/docs/adr/028-ollama-docker-compose-service.md b/docs/adr/028-ollama-docker-compose-service.md deleted file mode 100644 index 24a2d1bd..00000000 --- a/docs/adr/028-ollama-docker-compose-service.md +++ /dev/null @@ -1,239 +0,0 @@ -# ADR-028: Ollama Docker Compose service for NL search - -**Date:** 2026-06-06 -**Status:** Accepted -**Deciders:** Marcel Raddatz -**Relates to:** #737 (infrastructure), #735 (NL search epic) - ---- - -## Context - -Issue #735 introduces natural-language document search, requiring a local LLM to generate embeddings and/or run inference at query time. The family archive stores personal family history — data privacy is non-negotiable, so cloud-based inference APIs are excluded. The production target is a Hetzner CX42 (16 GB RAM, 8 vCPUs, CPU-only, ~32 EUR/month). - -Alternatives considered: - -| Option | Reason rejected | -|---|---| -| **llama.cpp** | No HTTP API out of the box; requires custom wrapper; higher ops burden | -| **vLLM** | GPU-first; significant overhead on CPU-only hardware; overkill for this scale | -| **Cloud APIs** (OpenAI, Gemini, etc.) | Vendor lock-in; per-token cost at scale; data leaves the server — unacceptable for a private family archive | -| **Ollama** | Self-contained Docker image; built-in HTTP REST API; actively maintained; CPU-compatible; zero egress | - -**Decision:** run Ollama as a Docker Compose service alongside the existing stack. - ---- - -## Decisions - -### 1. Hardware minimums and CPU-only constraint - -All inference runs on CPU. The target is the Hetzner CX42 (16 GB RAM, 8 vCPUs). - -| Tier | RAM | NL search | -|---|---|---| -| CX42 | 16 GB | Supported — full stack including Ollama | -| CX32 | 8 GB | Disabled — set `APP_OLLAMA_BASE_URL=` (empty) to skip Ollama entirely | -| CX22 | 4 GB | Unsupported for NL search | - -### 2. Memory budget on CX42 - -| Component | `mem_limit` | Typical active RSS | -|---|---|---| -| OCR service | 12g (hard ceiling) | ~6 GB | -| Ollama | 8g | ~8 GB | -| **Total** | | **~14 GB active** | - -`memswap_limit` on the Ollama service is set to `8g` (matching `mem_limit`) to prevent Linux from swapping model weights into swap under OCR memory pressure. Swapping model weights does not crash the container but silently degrades inference latency. This mirrors the pattern already applied to the OCR service. - -**Operational constraint:** do NOT run `docker-compose.observability.yml` continuously alongside both OCR and Ollama on a CX42. The observability stack adds ~2 GB, which leaves no headroom. - -### 3. Graceful-degradation contract - -`app.ollama.base-url` absent OR blank → Ollama bean NOT registered → NL search returns HTTP 503 with `ErrorCode: NL_SEARCH_UNAVAILABLE`. - -This single code path covers all unavailability scenarios: base-url unset, service unreachable, health check failed, and request timeout. - -#### Why not `@ConditionalOnProperty` - -`@ConditionalOnProperty` registers the bean when the property is present but blank (`APP_OLLAMA_BASE_URL=`). This produces a `RestClient` with an empty base URL that fails at runtime with an opaque error rather than a clean 503. - -#### Correct condition expression - -```java -@ConditionalOnExpression("!'${app.ollama.base-url:}'.isBlank()") -``` - -When the property is absent, the placeholder resolves to `''`; `.isBlank()` returns `true`; negation makes the condition `false`; the bean is not registered. Same result for an explicit empty string (`APP_OLLAMA_BASE_URL=`). - -### 4. Backend configuration pattern - -Use a `@ConfigurationProperties` record, not separate `@Value` injections: - -```java -@ConfigurationProperties("app.ollama") -record OllamaProperties(String baseUrl, String apiKey) {} -``` - -`OllamaProperties` is registered unconditionally — it is a plain value holder with no side effects. - -`@ConditionalOnExpression` belongs **only** on `RestClientOllamaClient` (the bean that creates a live network client). - -**Deliberate divergence from the OCR pattern:** the OCR service uses `@Value`-with-default because OCR is always-on and `http://ocr-service:8000` is a safe default. Ollama is truly optional — a missing URL means "feature disabled", not "use this default server". There is no safe default Ollama URL. - -### 5. Optional injection - -The NL search service uses constructor injection with `Optional`: - -```java -private final Optional ollamaClient; -``` - -When empty (bean not registered), the service method returns 503 immediately: - -```java -var client = ollamaClient.orElseThrow( - () -> DomainException.internal(ErrorCode.NL_SEARCH_UNAVAILABLE, "Ollama not configured")); -``` - -Prefer this over `@Autowired(required = false)` with a null check — the null-check pattern is noisy when the service already uses `@RequiredArgsConstructor`. - -### 6. Empty API key guard - -`RestClientOllamaClient` omits the `Authorization` header entirely when `apiKey` is blank: - -```java -if (!apiKey.isBlank()) { - request.header("Authorization", "Bearer " + apiKey); -} -``` - -Sending `Authorization: Bearer ` (empty token) has undefined or potentially broken behavior depending on the Ollama version. This mirrors the `trainingToken` guard in `RestClientOcrClient.java:107`. - -### 7. OLLAMA_API_KEY behavior in Ollama 0.6.5 and 0.30.6 - -**Empirically verified (2026-06-06) on both `0.6.5` and `0.30.6`:** `OLLAMA_API_KEY` does **not** enforce request authentication in either version. - -Test matrix run against `/api/tags`: - -| Configuration | No auth header | `Authorization: Bearer ` (empty) | `Authorization: Bearer wrongkey` | `Authorization: Bearer correctkey` | -|---|---|---|---|---| -| `OLLAMA_API_KEY=` (empty) | 200 | 200 | — | — | -| `OLLAMA_API_KEY` unset | 200 | — | — | — | -| `OLLAMA_API_KEY=testkey99` | 200 | 200 | 200 | 200 | - -**Finding:** The `OLLAMA_API_KEY` environment variable is not listed in Ollama's startup config dump and does not gate any HTTP request in either tested version. All configurations — empty string, fully unset, and a real key — accept all requests without authentication. - -**Practical implication:** `OLLAMA_API_KEY` provides no defense-in-depth in the tested versions. `archiv-net` network isolation is the only effective security control. The env var is retained in the Compose definition and `.env.example` for forward compatibility if Ollama enables enforcement in a future version, but operators must not rely on it for access control. - -**Backend guard still valid:** the `RestClientOllamaClient` code-level guard (omit `Authorization` header when `apiKey.isBlank()`) remains correct behavior regardless — it prevents a malformed `Authorization: Bearer ` header from being sent. - -### 8. read_only: true feasibility - -**Empirically verified (2026-06-06) on both `0.6.5` and `0.30.6`:** `read_only: true` works with Ollama. All three operations — `ollama serve`, `ollama pull qwen2.5:7b-instruct-q4_K_M`, and `ollama list` — succeeded with exit code 0 in both versions. - -Test run: -```bash -docker run --rm --read-only \ - -v ollama_models:/root/.ollama \ - --tmpfs /tmp \ - --entrypoint sh ollama/ollama:0.30.6 \ - -c "ollama serve & sleep 5 && ollama pull qwen2.5:7b-instruct-q4_K_M && ollama list" -``` - -**Note:** the entrypoint must be overridden to `sh` for the test command — the container's default entrypoint is `/bin/ollama` and does not accept `sh` as a subcommand. This is a Docker invocation detail; the Compose service definition uses the image's default entrypoint and `command:` override for the init container, which works correctly. - -**Result:** `read_only: true` and `tmpfs: - /tmp:size=512m` are applied to both `ollama` and `ollama-model-init`. The `ollama_models` volume handles all persistent writes; no other paths require write access during normal operation. - -### 9. Peak RSS of init container during pull - -**Empirically verified (2026-06-06):** Peak RSS during `qwen2.5:7b-instruct-q4_K_M` pull was **~108 MiB**. - -`docker stats` samples during the pull (15-second intervals): - -| Sample | MEM | -|---|---| -| 1 | 54.89 MiB | -| 2 | 66.3 MiB | -| 5 | 97.25 MiB | -| 9 | **107.8 MiB** (peak) | - -`mem_limit: 2g` is adequate — the model weights stream directly to the named volume; RSS is dominated by the Ollama server process alone (~100 MB), not the model data. No bump to 4 GB needed. - -### 10. Init container pull mechanism - -The `ollama-model-init` container uses a curl-based readiness loop with captured PID: - -```sh -ollama serve & SERVE_PID=$! -until curl -sf http://localhost:11434/api/tags; do sleep 1; done -ollama pull qwen2.5:7b-instruct-q4_K_M -kill $SERVE_PID -``` - -`kill %1` (job-control syntax) is unreliable in non-interactive `sh -c` contexts. Capturing the PID via `SERVE_PID=$!` is reliable. - -The same endpoint (`/api/tags`) is used for both the init container readiness loop and the main service `healthcheck`. - -### 11. start_period: 60s rationale - -The model is pre-pulled by `ollama-model-init` before the main service starts (via `condition: service_completed_successfully`). At main service startup, Ollama only loads model weights from the named volume and binds port 11434. - -60 seconds is appropriate for this cold-start profile. 300 seconds was considered — that would be appropriate if the service pulled the model itself — but overstates actual startup time when the model is already present on the volume. - -### 12. Security threat model - -**Primary control:** `archiv-net` network isolation. Ollama has no externally exposed port (`expose:` only, not `ports:`). The Caddyfile must not route any path to the Ollama service. - -**Note on `OLLAMA_API_KEY`:** Per §7, `OLLAMA_API_KEY` is not enforced in Ollama 0.6.5 or 0.30.6 and provides no authentication barrier against a compromised backend container. `archiv-net` network isolation is the sole effective security control. The env var is retained for forward compatibility only — do not rely on it for access control. - -Both `ollama` and `ollama-model-init` receive the ADR-019 hardening baseline: - -```yaml -cap_drop: [ALL] -security_opt: [no-new-privileges:true] -``` - -### 13. CI exclusion strategy - -Docker Compose profiles are not used — they would add developer friction (requiring `--profile ...` for all local dev commands). - -CI uses explicit service selection in `docker-compose.ci.yml`: -```bash -docker compose -f docker-compose.ci.yml up -d db minio create-buckets -``` - -Ollama is simply not listed and is never started in CI. A YAML comment on the `ollama` service block documents this: - -```yaml -# Not started in CI — CI uses explicit service selection -# (docker-compose.ci.yml: db minio create-buckets) -``` - -### 14. ollama_models volume operational note - -The `ollama_models` named volume holds model weights only — fully reproducible by re-pull. No backup is needed. - -If the volume fills after a model upgrade: -```bash -docker volume rm ollama_models && docker compose up -d -``` -The init container re-pulls the model on next startup. - ---- - -## Consequences - -### Positive - -- NL search runs entirely on-premises; no data leaves the server and no per-token cloud cost. -- Graceful degradation is a first-class concern: smaller or budget-constrained instances can run the app without Ollama with a single env var change. -- The init container pattern keeps model pull out of the critical startup path for the main service, giving accurate healthcheck timings. -- `@ConditionalOnExpression` with a blank-check is more correct than `@ConditionalOnProperty` for optional features with no safe default URL. - -### Risks and operational implications - -- **Memory pressure:** OCR + Ollama together consume ~14 GB on a 16 GB host. Running the observability stack simultaneously risks OOM kills. Monitor with `docker stats`. -- **CPU inference latency:** `qwen2.5:7b-instruct-q4_K_M` is chosen for CPU viability, but inference on 8 vCPUs will be noticeably slower than GPU-accelerated alternatives. This is acceptable for the family archive use case (low concurrency, not real-time). -- All three empirical TBD items from the original issue spec were resolved — see §7 (OLLAMA_API_KEY not enforced), §8 (`read_only: true` works), §9 (peak RSS ~108 MiB). -- Model upgrades require a `docker volume rm` to free old weights before pulling the replacement. Document this in runbook/DEPLOYMENT.md. diff --git a/docs/adr/034-ollama-production-deployment-and-keep-alive.md b/docs/adr/034-ollama-production-deployment-and-keep-alive.md deleted file mode 100644 index 0ff4a790..00000000 --- a/docs/adr/034-ollama-production-deployment-and-keep-alive.md +++ /dev/null @@ -1,125 +0,0 @@ -# ADR-034: Ollama in production — deployment, keep-alive pinning, and corrected init recipe - -**Date:** 2026-06-06 -**Status:** Accepted -**Deciders:** Marcel Raddatz -**Relates to:** #758 (bug), #759 (fix), #737 (NL search infrastructure) -**Corrects:** ADR-028 §10–§11 (init recipe and readiness probe) - ---- - -## Context - -ADR-028 introduced Ollama as a Docker Compose service for NL search and documented -its topology, graceful-degradation contract, and memory budget. Two defects survived -that work and only surfaced once NL search reached staging (#758): - -1. **Ollama was added only to the dev `docker-compose.yml`.** Staging and production - deploy from the self-contained `docker-compose.prod.yml`, which had no `ollama` - service. The backend defaults to `app.ollama.base-url: http://ollama:11434`, so its - client bean was active and resolved to a non-existent host → `ResourceAccessException` - → HTTP 503 on every NL search. -2. **The init recipe documented in ADR-028 §10 never worked.** The `ollama/ollama` image - `ENTRYPOINT` is `ollama`, so a bare `command: sh -c "…"` ran as `ollama sh -c "…"` - (`unknown command "sh"`), and the image ships **no curl**, so the curl-based readiness - loop and the curl healthcheck could never pass. - -This ADR records the production deployment decision and the corrected operational -contract. It is also the durable record of *why* `OLLAMA_KEEP_ALIVE=-1` is set, so a -future maintainer does not "optimize" it away and reintroduce the cold-load 503. - ---- - -## Decisions - -### 1. Ollama is a first-class production service - -`docker-compose.prod.yml` now defines `ollama` + `ollama-model-init` + the -`ollama-models` volume, mirroring the dev stack. The graceful-degradation contract from -ADR-028 §3 is preserved: `backend` has **no** hard `depends_on` on `ollama`, so an absent -or unhealthy Ollama still yields a clean 503 rather than blocking backend startup. - -### 2. Corrected init recipe (supersedes ADR-028 §10) - -The init container overrides the image entrypoint to a shell and probes readiness with -`ollama list` (not curl, which the image lacks): - -```sh -ollama serve & until ollama list >/dev/null 2>&1; do sleep 1; done && \ - (ollama list | grep -q 'qwen2.5:7b-instruct-q4_K_M' || ollama pull qwen2.5:7b-instruct-q4_K_M) -``` - -```yaml -entrypoint: ["/bin/sh", "-c"] -``` - -The pull is **guarded by a grep on the cached model list**. A model already on the volume -exits clean without any registry round-trip. This makes re-up offline-safe: a host reboot -during a registry/network blip can no longer fail init (which, via -`condition: service_completed_successfully`, would otherwise block the `ollama` service -and take NL search down until the registry was reachable again). The same recipe is used -in dev and prod — one mental model. - -### 3. Healthcheck uses `ollama list` (supersedes ADR-028 §11 probe) - -```yaml -healthcheck: - test: ["CMD", "ollama", "list"] -``` - -`ollama list` hits the local API and exits non-zero when the server is down — the correct -probe for a curl-less image. The `start_period: 60s` rationale from ADR-028 §11 still holds. - -### 4. `OLLAMA_KEEP_ALIVE=-1` — pin the model in memory - -```yaml -environment: - OLLAMA_KEEP_ALIVE: "-1" -``` - -By default Ollama evicts an idle model after ~5 minutes. The next query then pays a -cold-load penalty that exceeds the backend read timeout, producing an NL search 503 after -any idle period. Pinning the model (`-1` = never unload) keeps warm-path latency -predictable (~18 s on CPU). **Do not remove this** without re-introducing the post-idle -cold-load 503. - -### 5. Read timeout raised 30 → 60 s - -`app.ollama.timeout-seconds` is raised from 30 to 60 (`application.yaml`, mirrored in -`DEPLOYMENT.md`). Warm CPU inference is ~18 s; the higher ceiling absorbs the one cold -model load on the first query after an Ollama (re)start, before §4's pin takes hold. - -**Implicit NFR made explicit:** NL search shall return a result or a 503 within 60 s; the -cold-start path immediately after an Ollama restart is the only path that approaches this -ceiling. - -### 6. Hard-OOM trade-off (refines ADR-028 §2) - -`memswap_limit == mem_limit` (both `${OLLAMA_MEM_LIMIT:-8g}`) disables swap for the -container. Combined with §4's pinned model, a memory-pressure event is a **hard OOM-kill, -not graceful latency degradation**. This is deliberate — swap-thrashing an LLM is worse -than a clean restart — but it means the 8 GB envelope is a real ceiling. `qwen2.5-7B-q4` -plus its KV cache under load sits close enough to 8 GB that this needs a Prometheus -memory alert on the `ollama` container before it bites in production (tracked as -observability follow-up, not in this PR). - ---- - -## Consequences - -### Positive - -- NL search works on staging/production, not just dev — the actual deploy artifact now - matches the documented architecture. -- Re-up is offline-safe: a cached model never depends on registry reachability. -- The keep-alive pin and timeout ceiling make NL search latency predictable on CPU. - -### Risks and operational implications - -- **Hard OOM under memory pressure** (§6): a Prometheus alert on `ollama` container memory - is required before this is load-bearing in prod. Tracked as an observability follow-up. -- **Unauthenticated inference** relies entirely on `archiv-net` isolation (ADR-028 §7/§12, - unchanged). Sending an `Authorization` header from `RestClientOllamaClient` is a separate - durable hardening item, tracked outside this PR. -- ADR-028 §10–§11 describe a recipe that never functioned; this ADR is the authoritative - init/healthcheck contract going forward. diff --git a/docs/adr/035-rule-based-nlp-service.md b/docs/adr/035-rule-based-nlp-service.md deleted file mode 100644 index 3955e423..00000000 --- a/docs/adr/035-rule-based-nlp-service.md +++ /dev/null @@ -1,105 +0,0 @@ -# ADR-035: Replace Ollama with a rule-based NLP service for smart search - -**Date:** 2026-06-07 -**Status:** Accepted -**Deciders:** Marcel Raddatz -**Supersedes:** ADR-028 (Ollama for NL search), ADR-034 (Ollama production deployment) -**Relates to:** #771 (implementation) - ---- - -## Context - -ADR-028 introduced Ollama + qwen2.5-7B to parse free-text search queries into structured -extractions (person names, date ranges, person role, keywords). After deploying to -staging (ADR-034) the approach showed three problems: - -1. **Cold-start latency:** even with `OLLAMA_KEEP_ALIVE=-1` a Qwen inference on CPU takes - ~18 s. This blows the UX budget for a search feature and requires a 60 s timeout. -2. **Resource cost:** 8 GB resident RAM + 4 vCPU cap for an LLM whose only job is regex- - level entity extraction from short (< 500 char) German family-history queries. -3. **Fragility:** model-weight downloads, version pinning, and init-container orchestration - add operational surface area with no quality benefit over a deterministic parser. - -The query set is narrow and well-understood: person names are all in the PostgreSQL -`persons` table; date patterns are a fixed repertoire of German/English/Spanish formats; -person role (sender vs. receiver) is reliably signalled by a handful of prepositions -("von", "an", "von … an"); keywords are nouns/proper nouns not consumed by the other -extractors. - ---- - -## Decision - -Replace Ollama with a lightweight, rule-based Python FastAPI service (`nlp-service`). - -### Architecture - -``` -POST /api/search/nl (NlSearchController) - → NlQueryParserService - → RestClientNlpClient.parse(query, lang) - → POST http://nlp-service:8001/parse - ← { personNames, personRole, dateFrom, dateTo, keywords, rawQuery } -``` - -The response contract is identical to the old `OllamaExtraction`; only the transport -and implementation change. Java callers see `NlpExtraction` (renamed, same shape). - -### Implementation - -- **`nlp-service/`** — standalone FastAPI app (Python 3.11.12-slim image, ~256 MB RAM) - - `extractor.py` — pipeline: person extraction → role detection → date parsing → keywords - - `person_matcher.py` — two-pass fuzzy lookup (rapidfuzz 3.x) against the `persons` DB table; - loaded at startup, no live DB queries during extraction - - `models.py` — Pydantic `ParseRequest` (max 500 chars), `ParseResponse` - - `main.py` — lifespan loads persons from `DATABASE_URL`; `/health` reports `persons_loaded` - -- **`backend/search/`** — `OllamaClient` / `OllamaExtraction` renamed to `NlpClient` / - `NlpExtraction`; `NlpProperties` (`@ConfigurationProperties("app.nlp")`) replaces - `OllamaProperties`; `lang` parameter added to `/parse` and threaded through the stack. - -### Tunable parameters - -| Env var | Default | Effect | -|---|---|---| -| `DATABASE_URL` | — | PostgreSQL DSN; unset → person matching disabled | -| `NLP_FUZZY_THRESHOLD` | `80` | rapidfuzz similarity floor (0–100) | - -### Graceful degradation - -The backend's `RestClientNlpClient` wraps all HTTP errors and timeouts in -`DomainException.serviceUnavailable(SMART_SEARCH_UNAVAILABLE)`, returning HTTP 503 to -the client — identical behaviour to the Ollama path. The rate limiter is relaxed from -5 to 20 requests/min (rule-based extraction completes in < 50 ms vs. ~18 s for LLM). - ---- - -## Consequences - -### Positive - -- **Latency:** < 50 ms per extraction vs. ~18 s — smart search is now interactive. -- **Memory:** ~256 MB vs. 8 GB — frees 7.75 GB on the production host. -- **No model downloads:** the image ships no weights; startup is a single DB query. -- **Deterministic:** same query always produces the same result; no temperature/sampling. -- **Testable without infrastructure:** pytest with a seeded `PersonMatcher` fixture; no - WireMock stubs needed for most unit tests. - -### Trade-offs - -- **No semantic generalisation.** The LLM could handle novel phrasing; the rule-based - parser only handles the preposition patterns it was written for. Edge cases that fall - outside the pattern produce an empty extraction rather than a best-effort result. -- **Person matching depends on DB content.** A person not yet in the archive will never - match, even if the user types their exact name. The LLM could surface the name as a - raw string; this service surfaces nothing. This is acceptable for the current archive - size and query patterns. -- **Language support is fixed at de/en/es** (Paraglide locales). Adding a fourth locale - requires adding its stopword list and preposition table to `extractor.py`. - -### Superseded ADRs - -ADR-028 and ADR-034 documented the Ollama topology, init recipe, keep-alive pin, and -memory budget. All of that is now moot. The `ollama`, `ollama-model-init`, and -`ollama_models` volume are removed from `docker-compose.yml`. diff --git a/nlp-service/.dockerignore b/nlp-service/.dockerignore deleted file mode 100644 index 4a5db792..00000000 --- a/nlp-service/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -venv/ -.env -__pycache__/ -.pytest_cache/ -test_*.py -*.md diff --git a/nlp-service/CLAUDE.md b/nlp-service/CLAUDE.md deleted file mode 100644 index 821dc996..00000000 --- a/nlp-service/CLAUDE.md +++ /dev/null @@ -1,59 +0,0 @@ -# NLP Service - -Lightweight FastAPI service that parses free-text search queries into structured extractions, -replacing Ollama for the Familienarchiv NL search feature. - -## Stack - -- Python 3.11, FastAPI 0.115, rapidfuzz 3.x, psycopg2-binary - -No ML models — persons are matched against the live DB via fuzzy lookup. - -## Endpoints - -- `POST /parse` — parse a free-text query, return extraction matching `NlpExtraction` contract -- `GET /health` — returns `{"status": "ok", "persons_loaded": N}` - -## Running locally - -```bash -pip install -r requirements.txt - -# Without DB (empty person matcher — dates and keywords still work): -uvicorn main:app --reload --port 8001 - -# With DB (full person matching): -DATABASE_URL=postgresql://archive_user:secret@localhost:5432/family_archive_db \ - uvicorn main:app --reload --port 8001 - -curl -X POST http://localhost:8001/parse \ - -H "Content-Type: application/json" \ - -d '{"query": "Briefe von Clara Cram an Walter de Gruyter vor 1920", "lang": "de"}' -``` - -## Testing - -```bash -pytest -v -``` - -No DB required for tests — fixture pre-seeds the PersonMatcher with a small test corpus. - -## Architecture - -- `person_matcher.py` — DB-backed name lookup: loads all persons at startup, fuzzy-matches query tokens after person prepositions -- `extractor.py` — pipeline: persons → role → dates (regex) → keywords (stopword filter) -- `main.py` — FastAPI app; reads `DATABASE_URL` env var at startup - -## Design spec - -See `docs/superpowers/specs/2026-06-07-spacy-nlp-service-design.md`. - -## Notes - -This service is fully wired into `docker-compose.yml` (container `archive-nlp`, port 8001 -internal-only) and the Java search path (`RestClientNlpClient` → `NlQueryParserService` → -`NlSearchController`). The extraction contract matches `NlpExtraction` in -`backend/src/main/java/org/raddatz/familienarchiv/search/`. - -Test sentences for manual evaluation are in `test_sentences.md`. diff --git a/nlp-service/Dockerfile b/nlp-service/Dockerfile deleted file mode 100644 index ccb9e7e6..00000000 --- a/nlp-service/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM python:3.11.12-slim - -WORKDIR /app - -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl \ - && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -COPY . . - -RUN useradd --no-create-home --shell /usr/sbin/nologin --uid 1001 nlp \ - && chown -R nlp:nlp /app - -USER nlp - -EXPOSE 8001 - -HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ - CMD curl -f http://localhost:8001/health || exit 1 - -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001"] diff --git a/nlp-service/extractor.py b/nlp-service/extractor.py deleted file mode 100644 index dace65cf..00000000 --- a/nlp-service/extractor.py +++ /dev/null @@ -1,310 +0,0 @@ -"""Rule-based NLP pipeline: dates via regex, persons via DB-backed matcher.""" -from __future__ import annotations - -import re -from datetime import date -from typing import TYPE_CHECKING - -from models import ParseResponse -from person_matcher import PersonMatcher - -if TYPE_CHECKING: - pass - -# ── Module-level PersonMatcher and fuzzy threshold (set at startup) ────────── - -_matcher: PersonMatcher | None = None -_fuzzy_threshold: int = 80 - - -def set_person_matcher(m: PersonMatcher) -> None: - global _matcher - _matcher = m - - -def get_person_matcher() -> PersonMatcher | None: - return _matcher - - -def set_fuzzy_threshold(threshold: int) -> None: - global _fuzzy_threshold - _fuzzy_threshold = threshold - - -# ── Preposition sets ────────────────────────────────────────────────────────── - -_SENDER_PREPS: dict[str, frozenset[str]] = { - "de": frozenset({"von", "vom"}), - "en": frozenset({"from", "by"}), - "es": frozenset({"de", "por"}), -} - -_RECEIVER_PREPS: dict[str, frozenset[str]] = { - "de": frozenset({"an", "nach", "für"}), - "en": frozenset({"to", "for"}), - "es": frozenset({"para", "a"}), -} - -_ALL_PERSON_PREPS: dict[str, frozenset[str]] = { - lang: _SENDER_PREPS[lang] | _RECEIVER_PREPS[lang] - for lang in ("de", "en", "es") -} - -# ── Date direction tokens ───────────────────────────────────────────────────── - -_DATE_BEFORE: dict[str, frozenset[str]] = { - "de": frozenset({"vor"}), - "en": frozenset({"before"}), - "es": frozenset({"antes"}), -} - -_DATE_AFTER: dict[str, frozenset[str]] = { - "de": frozenset({"nach"}), - "en": frozenset({"after"}), - "es": frozenset({"después", "despues"}), -} - -_DATE_BETWEEN: dict[str, frozenset[str]] = { - "de": frozenset({"zwischen"}), - "en": frozenset({"between"}), - "es": frozenset({"entre"}), -} - -# ── Extra span-termination tokens (function words that cannot be in a name) ── - -_EXTRA_SPAN_STOPS: dict[str, frozenset[str]] = { - # German articles, possessives, and particles that end a name span - "de": frozenset({ - "im", "am", "beim", "zum", "zur", - "dem", "den", "des", - "sein", "seine", "seinen", "seiner", - "ihr", "ihre", "ihrem", "ihren", "ihrer", - "unser", "unsere", "unseren", - "über", "auch", "oder", "und", - }), - "en": frozenset(), - "es": frozenset({"el", "la", "los", "las", "su", "sus", "mi"}), -} - -# ── Stopword lists ──────────────────────────────────────────────────────────── - -_STOPWORDS: dict[str, frozenset[str]] = { - "de": frozenset({ - "der", "die", "das", "des", "dem", "den", - "ein", "eine", "einem", "einen", "einer", "eines", - "er", "sie", "es", "wir", "ihr", "ich", "du", - "und", "oder", "aber", "doch", "auch", "noch", "nur", - "in", "an", "auf", "aus", "bei", "mit", "nach", "von", "vom", - "vor", "zu", "zur", "zum", "durch", "für", "über", "unter", - "zwischen", "gegen", "ohne", "um", "bis", "seit", "wegen", - "ist", "sind", "war", "waren", "wird", "werden", - "hat", "haben", "hatte", "hatten", - "sein", "seine", "seinen", "seiner", "seines", - "ihre", "ihren", "ihrer", "ihrem", "ihres", - "nicht", "kein", "keine", "keinen", "keinem", "keines", - "so", "wie", "als", "da", "hier", "dort", "wo", "wer", "was", - "im", "am", "beim", "ins", "ans", - "ja", "nein", "denn", "wenn", "weil", "dass", "ob", "damit", - "alle", "alles", "mehr", "sehr", "viel", "wenig", - "diesem", "dieser", "dieses", "diese", "diesen", - "jetzt", "dann", "nun", "schon", "wohl", "wurde", "wurden", - "worden", "geschrieben", "seinen", "ihrer", - "beim", "nach", "zum", "zur", "dem", "den", - "seine", "ihrem", "Jahr", "Jahren", "jahre", "jahr", - }), - "en": frozenset({ - "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", - "for", "of", "with", "by", "from", "about", "as", "into", - "through", "is", "are", "was", "were", "be", "been", "being", - "have", "has", "had", "do", "does", "did", "will", "would", - "could", "should", "may", "might", "must", "shall", "can", - "i", "you", "he", "she", "it", "we", "they", "their", "our", - "his", "her", "its", "my", "your", - "this", "that", "these", "those", "all", "not", "no", "nor", - "very", "more", "most", "much", "many", "some", "any", - "before", "after", "between", "during", "since", "until", - "when", "where", "who", "which", "what", "how", - }), - "es": frozenset({ - "el", "la", "los", "las", "un", "una", "unos", "unas", - "y", "o", "pero", "sin", "con", "en", "de", "del", "al", - "a", "ante", "bajo", "desde", "entre", "hacia", "hasta", - "para", "por", "sobre", "tras", - "es", "son", "era", "eran", "fue", "fueron", "ser", "estar", - "ha", "han", "he", "tener", "tiene", - "yo", "su", "sus", "mi", "tu", - "este", "esta", "estos", "estas", "ese", "esa", - "no", "muy", "todo", "todos", "toda", - "que", "cuando", "donde", "como", - "antes", "después", "durante", "desde", "hasta", - }), -} - -# ── Year regex ──────────────────────────────────────────────────────────────── - -_YEAR_RE = re.compile(r"\b(\d{4})\b") -_WORD_RE = re.compile(r"\b[^\W\d_]{3,}\b", re.UNICODE) - - -# ── Step 1 + 2: Person extraction and role detection ───────────────────────── - -def _extract_persons_and_role( - query: str, - lang: str, -) -> tuple[list[str], str]: - """Return (person_names, role) using the DB-backed PersonMatcher.""" - m = _matcher - if m is None or len(m) == 0: - return [], "any" - - preps = _ALL_PERSON_PREPS[lang] - stops = preps | _DATE_BEFORE[lang] | _DATE_AFTER[lang] | _DATE_BETWEEN[lang] | _EXTRA_SPAN_STOPS[lang] - matches = m.find_in_query(query, preps, stop_tokens=stops, threshold=_fuzzy_threshold) - - person_names = [text for text, _ in matches] - - if len(matches) != 1: - return person_names, "any" - - _, prep = matches[0] - if prep is None: - return person_names, "any" - if prep in _SENDER_PREPS[lang]: - return person_names, "sender" - if prep in _RECEIVER_PREPS[lang]: - return person_names, "receiver" - return person_names, "any" - - -# ── Step 3: Date extraction ─────────────────────────────────────────────────── - -def _find_years(query: str) -> list[tuple[int, int, int]]: - """Return list of (start, end, year_int) for valid 4-digit year tokens.""" - return [ - (m.start(), m.end(), int(m.group())) - for m in _YEAR_RE.finditer(query) - if 1000 < int(m.group()) < 3000 - ] - - -def _direction_before_year( - query: str, - year_start: int, - lang: str, - person_names: list[str], -) -> str: - """Classify direction of the date span as 'before', 'after', or 'bare'. - - Looks at the two tokens immediately preceding the year. If the closer - token is a matched person name part, the direction word belongs to that - person — not to the year — so we return 'bare'. - """ - prefix_words = query[:year_start].split() - if not prefix_words: - return "bare" - - person_tokens = {w.lower() for name in person_names for w in name.split()} - recent = [w.lower() for w in prefix_words[-2:]] - - before_set = _DATE_BEFORE[lang] - after_set = _DATE_AFTER[lang] - - for direction_tok in reversed(recent): # closest first - if direction_tok in before_set: - # Only use this if the word immediately before the year is not a person - if recent[-1] in person_tokens: - return "bare" - return "before" - if direction_tok in after_set: - if recent[-1] in person_tokens: - return "bare" - return "after" - - return "bare" - - -def extract_dates( - query: str, - lang: str, - person_names: list[str] | None = None, -) -> tuple[str | None, str | None]: - """Return (date_from, date_to) as ISO strings or None.""" - if person_names is None: - person_names = [] - - year_spans = _find_years(query) - if not year_spans: - return None, None - - # "zwischen X und Y" / "between X and Y" — two years form a range - query_lower = query.lower() - if any(w in query_lower.split() for w in _DATE_BETWEEN[lang]) and len(year_spans) >= 2: - years = sorted([y for _, _, y in year_spans[:2]]) - return date(years[0], 1, 1).isoformat(), date(years[1], 12, 31).isoformat() - - start, end, year = year_spans[0] - direction = _direction_before_year(query, start, lang, person_names) - - if direction == "before": - return None, date(year, 12, 31).isoformat() - if direction == "after": - return date(year, 1, 1).isoformat(), None - # bare year → closed year range - return date(year, 1, 1).isoformat(), date(year, 12, 31).isoformat() - - -# ── Step 4: Keyword extraction ──────────────────────────────────────────────── - -def extract_keywords( - query: str, - lang: str, - person_spans: list[str], - year_strings: list[str], -) -> list[str]: - """Return lowercased content words after removing persons, years, stopwords.""" - text = query - - # Remove matched person spans (longest first to avoid partial replacements) - for span in sorted(person_spans, key=len, reverse=True): - text = re.sub( - r"(? ParseResponse: - """Run the full rule-based pipeline and return a ParseResponse.""" - person_names, person_role = _extract_persons_and_role(query, lang) - year_strings = [str(y) for _, _, y in _find_years(query)] - date_from, date_to = extract_dates(query, lang, person_names) - keywords = extract_keywords(query, lang, person_names, year_strings) - - return ParseResponse( - personNames=person_names, - personRole=person_role, - dateFrom=date_from, - dateTo=date_to, - keywords=keywords, - rawQuery=query, - ) diff --git a/nlp-service/main.py b/nlp-service/main.py deleted file mode 100644 index 509ae7b1..00000000 --- a/nlp-service/main.py +++ /dev/null @@ -1,79 +0,0 @@ -"""FastAPI app — /parse and /health endpoints.""" -from __future__ import annotations - -import logging -import os -from contextlib import asynccontextmanager - -from fastapi import FastAPI, HTTPException - -logger = logging.getLogger(__name__) - -from extractor import extract, get_person_matcher, set_fuzzy_threshold, set_person_matcher -from models import ParseRequest, ParseResponse -from person_matcher import PersonMatcher - -_DEFAULT_FUZZY_THRESHOLD = 80 - - -def _parse_fuzzy_threshold(val: str) -> int: - """Parse and validate NLP_FUZZY_THRESHOLD — must be integer in [0, 100].""" - try: - n = int(val) - except ValueError: - raise ValueError(f"NLP_FUZZY_THRESHOLD must be an integer, got: {val!r}") - if not (0 <= n <= 100): - raise ValueError(f"NLP_FUZZY_THRESHOLD must be between 0 and 100, got: {n}") - return n - - -def _load_persons_from_db(db_url: str) -> list[tuple[str | None, str | None]]: - import psycopg2 # deferred — not available in test environments without a DB - - conn = psycopg2.connect(db_url) - try: - cur = conn.cursor() - cur.execute("SELECT first_name, last_name FROM persons") - return cur.fetchall() - finally: - conn.close() - - -@asynccontextmanager -async def lifespan(app: FastAPI): - threshold_raw = os.environ.get("NLP_FUZZY_THRESHOLD", str(_DEFAULT_FUZZY_THRESHOLD)) - threshold = _parse_fuzzy_threshold(threshold_raw) - set_fuzzy_threshold(threshold) - - # Only initialise the matcher when nothing was pre-seeded (e.g., by tests). - if get_person_matcher() is None: - m = PersonMatcher() - db_url = os.environ.get("DATABASE_URL") - if db_url: - try: - rows = _load_persons_from_db(db_url) - m.load(rows) - logger.info("PersonMatcher loaded %d name variants from DB", len(m)) - except Exception: - logger.error("Failed to load persons from DB — person matching disabled", exc_info=True) - else: - logger.warning("DATABASE_URL not set — person matching disabled") - set_person_matcher(m) - yield - - -app = FastAPI(lifespan=lifespan) - - -@app.get("/health") -def health() -> dict: - m = get_person_matcher() - return {"status": "ok", "persons_loaded": len(m) if m else 0} - - -@app.post("/parse", response_model=ParseResponse) -def parse(request: ParseRequest) -> ParseResponse: - try: - return extract(request.query, request.lang) - except Exception as exc: - raise HTTPException(status_code=500, detail="internal error") from exc diff --git a/nlp-service/models.py b/nlp-service/models.py deleted file mode 100644 index 7015d36a..00000000 --- a/nlp-service/models.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations -from typing import Literal -from pydantic import BaseModel, Field - - -class ParseRequest(BaseModel): - query: str = Field(max_length=500) - lang: Literal["de", "en", "es"] - - -class ParseResponse(BaseModel): - personNames: list[str] - personRole: Literal["sender", "receiver", "any"] - dateFrom: str | None - dateTo: str | None - keywords: list[str] - rawQuery: str diff --git a/nlp-service/person_matcher.py b/nlp-service/person_matcher.py deleted file mode 100644 index 2374208c..00000000 --- a/nlp-service/person_matcher.py +++ /dev/null @@ -1,184 +0,0 @@ -"""DB-backed person name matcher with fuzzy search.""" -from __future__ import annotations - -import re - -from rapidfuzz import fuzz, process - -_PUNCT_RE = re.compile(r"[^\w\s\-]", re.UNICODE) -_YEAR_PAT = re.compile(r"^\d{4}$") - -# Tokens that cannot appear in a real person's first name — used to filter DB -# records that are annotations or descriptions masquerading as persons. -_NON_NAME_TOKENS: frozenset[str] = frozenset({ - # German prepositions - "an", "in", "im", "am", "aus", "von", "vom", "nach", "zu", "zum", "zur", - "für", "bei", "beim", "mit", "über", "unter", "durch", "gegen", "ohne", - "bis", "seit", "des", "dem", "den", - # German possessives / pronouns - "sein", "seine", "seinen", "seiner", - "ihr", "ihre", "ihren", "ihrem", - # English prepositions - "for", "from", "by", "of", - # Spanish prepositions - "del", "por", "para", -}) - - -class PersonMatcher: - """Match person name fragments from free-text queries against known persons. - - Loaded once at startup from (first_name, last_name) DB rows. At query - time, scans for tokens following person-indicator prepositions and fuzzy- - matches them against the loaded name variants. Returns the original query - text (not the resolved DB name) so the Java resolveNames() mechanism can - do its own disambiguation. - """ - - def __init__(self) -> None: - self._names: list[str] = [] # lowercase name variants - - # ── Loading ─────────────────────────────────────────────────────────────── - - def load(self, rows: list[tuple[str | None, str | None]]) -> None: - """Populate from DB rows of (first_name, last_name).""" - seen: set[str] = set() - for first, last in rows: - first = (first or "").strip() - last = (last or "").strip() - # Skip records whose first_name contains function words — these are - # annotations or descriptions in the DB, not real person names. - if any(w in _NON_NAME_TOKENS for w in first.lower().split()): - continue - for variant in _name_variants(first, last): - key = variant.lower() - if key not in seen: - seen.add(key) - self._names.append(key) - - def __len__(self) -> int: - return len(self._names) - - # ── Query-time matching ─────────────────────────────────────────────────── - - def find_in_query( - self, - query: str, - prepositions: frozenset[str], - stop_tokens: frozenset[str] | None = None, - threshold: int = 80, - ) -> list[tuple[str, str | None]]: - """Find person name spans in *query*. - - Returns a list of ``(original_query_text, anchoring_prep_or_None)`` - in left-to-right order. - - Parameters - ---------- - prepositions: - Person-indicator prepositions for the query language (triggers a - scan for the tokens that follow). - stop_tokens: - Tokens that terminate a name span (prepositions + date-direction - words). "de" is a special exception: when immediately followed by - a capitalised word it is treated as a name connector (e.g. - "de Gruyter") rather than a stop. - threshold: - Minimum rapidfuzz token_sort_ratio score to accept a match. - - Strategy - -------- - Pass 1 — prep-anchored: for each person-indicator preposition found in - the token list, collect up to 3 consecutive non-stop, non-year tokens - after it and fuzzy-match the resulting span against loaded names. - Longest match wins. - - Pass 2 — full-name scan: scan positions not yet consumed for exact - multi-word full-name matches (no preposition anchor required). - """ - tokens = query.split() - clean = [_PUNCT_RE.sub("", t) for t in tokens] - lower = [t.lower() for t in clean] - - # Prepositions always terminate a name span, even without explicit stop_tokens. - stops = (stop_tokens or frozenset()) | prepositions - consumed: set[int] = set() - hits: list[tuple[int, str, str | None]] = [] # (position, text, prep) - - # Pass 1 — prep-anchored - for i, ltok in enumerate(lower): - if ltok not in prepositions or i + 1 >= len(tokens): - continue - - # Build candidate span — stop at stop tokens or 4-digit years. - # Exception: "de" before a capitalised word is a name connector. - span_indices: list[int] = [] - j = i + 1 - while j < len(tokens) and len(span_indices) < 3: - if j in consumed: - break - t = lower[j] - if t in stops or _YEAR_PAT.match(clean[j]): - # Allow "de" when the *next* token starts with a capital — - # e.g. "Walter de Gruyter". - next_clean = clean[j + 1] if j + 1 < len(tokens) else "" - if t == "de" and next_clean[:1].isupper(): - pass # connector — keep going - else: - break - span_indices.append(j) - j += 1 - - # Try longest match first, then shorter spans - for span_len in range(len(span_indices), 0, -1): - idx = span_indices[:span_len] - span_lower = " ".join(lower[k] for k in idx) - if self._is_match(span_lower, threshold): - hits.append((idx[0], " ".join(tokens[k] for k in idx), ltok)) - consumed.update(idx) - break - - # Pass 2 — full multi-word name scan (exact only, no preposition needed) - for span_len in (3, 2): - for i in range(len(tokens) - span_len + 1): - span_idx = range(i, i + span_len) - if any(j in consumed for j in span_idx): - continue - span_lower = " ".join(lower[i : i + span_len]) - if span_lower in self._names: - hits.append((i, " ".join(tokens[i : i + span_len]), None)) - consumed.update(span_idx) - - hits.sort(key=lambda h: h[0]) - return [(text, prep) for _, text, prep in hits] - - # ── Internal helpers ────────────────────────────────────────────────────── - - def _is_match(self, text: str, threshold: int) -> bool: - """Return True if *text* fuzzy-matches any loaded name at >= threshold.""" - if not self._names or len(text.strip()) < 3: - return False - text_lower = text.strip().lower() - if text_lower in self._names: - return True # exact match — fast path - result = process.extractOne( - text_lower, - self._names, - scorer=fuzz.token_sort_ratio, - score_cutoff=threshold, - ) - return result is not None - - -# ── helpers ─────────────────────────────────────────────────────────────────── - -def _name_variants(first: str, last: str) -> list[str]: - """Return the name variants to index for a single person.""" - variants = [] - if first and last: - variants.append(f"{first} {last}") - if first: - variants.append(first) - if last: - variants.append(last) - return variants diff --git a/nlp-service/requirements.txt b/nlp-service/requirements.txt deleted file mode 100644 index 42935fcc..00000000 --- a/nlp-service/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -fastapi[standard]==0.115.6 -uvicorn[standard]==0.34.0 -rapidfuzz>=3.0,<4.0 -psycopg2-binary>=2.9,<3.0 -pytest>=8.0,<9.0 -httpx>=0.28,<1.0 diff --git a/nlp-service/test_extractor.py b/nlp-service/test_extractor.py deleted file mode 100644 index 1bf47a9c..00000000 --- a/nlp-service/test_extractor.py +++ /dev/null @@ -1,335 +0,0 @@ -"""Tests for the rule-based extractor and PersonMatcher.""" -import pytest - -from extractor import extract, extract_dates, extract_keywords, set_person_matcher -from person_matcher import PersonMatcher - -# ── Shared test fixture ─────────────────────────────────────────────────────── - -_TEST_PERSONS = [ - ("Clara", "Cram"), - ("Herbert", "Cram"), - ("Eugenie", "de Gruyter"), - ("Walter", "de Gruyter"), - ("Marie", "Cram"), - ("Juan", "Cram"), - ("Hilde", "de Gruyter"), - ("Hans", "de Gruyter"), - ("Albert", "de Gruyter"), - ("Anita", "Wöhler"), - ("Else", "Bohrmann"), - ("Lili", "Duvenbeck"), -] - - -@pytest.fixture(scope="session", autouse=True) -def seeded_matcher(): - """Load test persons into the global matcher before any test runs.""" - m = PersonMatcher() - m.load(_TEST_PERSONS) - set_person_matcher(m) - return m - - -# ── PersonMatcher unit tests ────────────────────────────────────────────────── - -class TestPersonMatcher: - DE_PREPS = frozenset({"von", "vom", "an", "nach", "für"}) - - def test_load_populates_names(self, seeded_matcher): - assert len(seeded_matcher) > 0 - - def test_exact_full_name_match(self, seeded_matcher): - hits = seeded_matcher.find_in_query("Briefe von Clara Cram", self.DE_PREPS) - assert hits == [("Clara Cram", "von")] - - def test_exact_first_name_only(self, seeded_matcher): - hits = seeded_matcher.find_in_query("Briefe von Eugenie", self.DE_PREPS) - assert hits == [("Eugenie", "von")] - - def test_exact_first_name_receiver(self, seeded_matcher): - hits = seeded_matcher.find_in_query("Briefe an Herbert", self.DE_PREPS) - assert hits == [("Herbert", "an")] - - def test_fuzzy_typo(self, seeded_matcher): - hits = seeded_matcher.find_in_query("Briefe von Herrbert Cram", self.DE_PREPS) - assert len(hits) == 1 - assert hits[0][1] == "von" - - def test_two_persons_extracted(self, seeded_matcher): - hits = seeded_matcher.find_in_query( - "Briefe von Clara Cram an Herbert Cram", self.DE_PREPS - ) - assert len(hits) == 2 - assert hits[0][0] == "Clara Cram" - assert hits[0][1] == "von" - assert hits[1][0] == "Herbert Cram" - assert hits[1][1] == "an" - - def test_no_match_for_place_name(self, seeded_matcher): - hits = seeded_matcher.find_in_query("Reise nach Mexiko", self.DE_PREPS) - assert hits == [] - - def test_no_match_for_topic_word(self, seeded_matcher): - hits = seeded_matcher.find_in_query("Briefe aus dem Krieg", self.DE_PREPS) - assert hits == [] - - def test_first_name_eugenie_regression(self, seeded_matcher): - # spaCy NER missed standalone first names - hits = seeded_matcher.find_in_query("Briefe von Eugenie", self.DE_PREPS) - assert len(hits) == 1 - - def test_merged_names_regression(self, seeded_matcher): - # spaCy NER merged "Herbert an Eugenie de Gruyter" into one PER span - hits = seeded_matcher.find_in_query( - "Briefe von Herbert an Eugenie de Gruyter nach 1914", self.DE_PREPS - ) - assert len(hits) == 2 - names = [h[0] for h in hits] - assert "Herbert" in names - assert "Eugenie de Gruyter" in names - - def test_english_preps(self, seeded_matcher): - en_preps = frozenset({"from", "by", "to", "for"}) - hits = seeded_matcher.find_in_query( - "Letters from Clara Cram to Walter de Gruyter in 1920", en_preps - ) - assert len(hits) == 2 - assert hits[0][0] == "Clara Cram" - assert hits[1][0] == "Walter de Gruyter" - - def test_double_preposition_de(self, seeded_matcher): - hits = seeded_matcher.find_in_query( - "Briefe von Clara nach Herbert", self.DE_PREPS - ) - assert len(hits) == 2 - names = [h[0] for h in hits] - assert "Clara" in names - assert "Herbert" in names - - -# ── Date extraction tests ───────────────────────────────────────────────────── - -class TestExtractDates: - def test_bare_year_gives_range(self): - assert extract_dates("Briefe 1920", "de") == ("1920-01-01", "1920-12-31") - - def test_im_jahr(self): - assert extract_dates("Schriften im Jahr 1905", "de") == ( - "1905-01-01", "1905-12-31" - ) - - def test_vor_year(self): - assert extract_dates("Briefe vor 1920", "de") == (None, "1920-12-31") - - def test_nach_year(self): - assert extract_dates("Schriften nach 1920", "de") == ("1920-01-01", None) - - def test_zwischen(self): - assert extract_dates("Dokumente zwischen 1914 und 1918", "de") == ( - "1914-01-01", "1918-12-31" - ) - - def test_before_en(self): - assert extract_dates("Letters before 1918", "en") == (None, "1918-12-31") - - def test_after_en(self): - assert extract_dates("Letters after 1939", "en") == ("1939-01-01", None) - - def test_between_en(self): - assert extract_dates("Letters between 1914 and 1918", "en") == ( - "1914-01-01", "1918-12-31" - ) - - def test_antes_de_es(self): - assert extract_dates("Cartas antes de 1900", "es") == (None, "1900-12-31") - - def test_entre_es(self): - assert extract_dates("entre 1915 y 1920", "es") == ( - "1915-01-01", "1920-12-31" - ) - - def test_no_year(self): - assert extract_dates("Briefe aus dem Krieg", "de") == (None, None) - - def test_nach_before_person_then_year(self): - # "nach Marie 1920" — "nach" belongs to person, not date - date_from, date_to = extract_dates("Briefe nach Marie 1920", "de", ["Marie"]) - assert date_from == "1920-01-01" - assert date_to == "1920-12-31" - - def test_bare_year_alone(self): - assert extract_dates("1918", "de") == ("1918-01-01", "1918-12-31") - - -# ── Keyword extraction tests ────────────────────────────────────────────────── - -class TestExtractKeywords: - def test_basic_topic_words(self): - kws = extract_keywords("Briefe aus dem Krieg", "de", [], []) - assert "krieg" in kws - - def test_stopwords_excluded(self): - kws = extract_keywords("von der nach dem aus", "de", [], []) - for sw in ("von", "der", "nach", "dem", "aus"): - assert sw not in kws - - def test_person_spans_excluded(self): - kws = extract_keywords( - "Briefe von Clara Cram nach Herbert", "de", - ["Clara Cram", "Herbert"], [] - ) - assert "clara" not in kws - assert "cram" not in kws - assert "herbert" not in kws - - def test_years_excluded(self): - kws = extract_keywords("Schriften 1920 über Reise", "de", [], ["1920"]) - assert "1920" not in kws - - def test_deduplication(self): - kws = extract_keywords("Krieg Krieg Krieg", "de", [], []) - assert kws.count("krieg") == 1 - - def test_en_stopwords(self): - kws = extract_keywords("Letters about the war", "en", [], []) - assert "the" not in kws - assert "war" in kws - - def test_short_words_excluded(self): - kws = extract_keywords("ab cd ef xy", "de", [], []) - assert all(len(k) >= 3 for k in kws) - - -# ── Full pipeline integration tests ────────────────────────────────────────── - -class TestExtract: - def test_full_sentence_de(self): - r = extract("Briefe von Clara Cram an Walter de Gruyter im Jahr 1920", "de") - assert "Clara Cram" in r.personNames - assert "Walter de Gruyter" in r.personNames - assert r.personRole == "any" - assert r.dateFrom == "1920-01-01" - assert r.dateTo == "1920-12-31" - - def test_sender_role_de(self): - r = extract("Briefe von Clara Cram vor 1910", "de") - assert r.personNames == ["Clara Cram"] - assert r.personRole == "sender" - assert r.dateTo == "1910-12-31" - assert r.dateFrom is None - - def test_receiver_role_de(self): - r = extract("Briefe an Walter de Gruyter", "de") - assert r.personNames == ["Walter de Gruyter"] - assert r.personRole == "receiver" - - def test_first_name_only_eugenie(self): - r = extract("Briefe von Eugenie", "de") - assert "Eugenie" in r.personNames - assert r.personRole == "sender" - - def test_first_name_only_herbert(self): - r = extract("Kriegsbriefe von Herbert", "de") - assert "Herbert" in r.personNames - - def test_merged_names_bug_fixed(self): - r = extract("Briefe von Herbert an Eugenie de Gruyter nach 1914", "de") - assert "Herbert" in r.personNames - assert "Eugenie de Gruyter" in r.personNames - assert r.dateFrom == "1914-01-01" - - def test_topic_only_krieg(self): - r = extract("Briefe aus dem Krieg", "de") - assert r.personNames == [] - assert "krieg" in r.keywords - - def test_topic_only_single_word(self): - r = extract("Kriegspost", "de") - assert r.personNames == [] - - def test_date_range_only(self): - r = extract("Dokumente zwischen 1914 und 1918", "de") - assert r.personNames == [] - assert r.dateFrom == "1914-01-01" - assert r.dateTo == "1918-12-31" - - def test_colloquial_von(self): - r = extract("von Clara", "de") - assert r.personNames == ["Clara"] - assert r.personRole == "sender" - - def test_colloquial_an(self): - r = extract("an Walter", "de") - assert r.personNames == ["Walter"] - assert r.personRole == "receiver" - - def test_bare_year_alone(self): - r = extract("1918", "de") - assert r.dateFrom == "1918-01-01" - assert r.dateTo == "1918-12-31" - assert r.personNames == [] - - def test_english_full_sentence(self): - r = extract("Letters from Clara Cram to Walter de Gruyter in 1920", "en") - assert "Clara Cram" in r.personNames - assert "Walter de Gruyter" in r.personNames - assert r.dateFrom == "1920-01-01" - - def test_english_receiver_with_date(self): - r = extract("Letters to Herbert Cram after 1939", "en") - assert "Herbert Cram" in r.personNames - assert r.personRole == "receiver" - assert r.dateFrom == "1939-01-01" - - def test_english_birthday(self): - r = extract("Birthday greetings from Anita Wöhler", "en") - assert "Anita Wöhler" in r.personNames - assert r.personRole == "sender" - - def test_english_between_dates(self): - r = extract("Letters between 1914 and 1918", "en") - assert r.dateFrom == "1914-01-01" - assert r.dateTo == "1918-12-31" - - def test_spanish_full_sentence(self): - r = extract("Cartas de Clara Cram a Walter de Gruyter en 1920", "es") - assert "Clara Cram" in r.personNames - assert "Walter de Gruyter" in r.personNames - assert r.dateFrom == "1920-01-01" - - def test_spanish_before(self): - r = extract("Cartas antes de 1900", "es") - assert r.dateTo == "1900-12-31" - assert r.dateFrom is None - - def test_rawquery_echoed(self): - q = "test query" - r = extract(q, "de") - assert r.rawQuery == q - - def test_false_positive_compound_noun_regression(self): - # spaCy tagged "Geburtstagsglückwünsche" as a PER entity - r = extract("Geburtstagsglückwünsche", "de") - assert r.personNames == [] - - def test_question_phrasing(self): - r = extract("Wer hat an Herbert Cram 1918 geschrieben?", "de") - assert "Herbert Cram" in r.personNames - assert r.personRole == "receiver" - assert r.dateFrom == "1918-01-01" - - def test_lowercase_query(self): - r = extract("briefe von clara cram an herbert 1920", "de") - # Should still find persons despite lowercase - assert len(r.personNames) >= 1 - - def test_empty_matcher_returns_no_persons(self, seeded_matcher): - from extractor import get_person_matcher, set_person_matcher - original = get_person_matcher() - try: - set_person_matcher(PersonMatcher()) - r = extract("Briefe von Clara Cram", "de") - assert r.personNames == [] - finally: - set_person_matcher(original) diff --git a/nlp-service/test_main.py b/nlp-service/test_main.py deleted file mode 100644 index 5a81156a..00000000 --- a/nlp-service/test_main.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Integration tests for the FastAPI app.""" -import pytest -from fastapi.testclient import TestClient - -from extractor import set_person_matcher -from person_matcher import PersonMatcher - -_TEST_PERSONS = [ - ("Clara", "Cram"), - ("Herbert", "Cram"), - ("Eugenie", "de Gruyter"), - ("Walter", "de Gruyter"), - ("Marie", "Cram"), - ("Anita", "Wöhler"), -] - - -@pytest.fixture(scope="session") -def client(): - # Pre-seed the matcher so the lifespan doesn't overwrite it with an empty one. - m = PersonMatcher() - m.load(_TEST_PERSONS) - set_person_matcher(m) - from main import app - with TestClient(app) as c: - yield c - - -def test_health(client): - r = client.get("/health") - assert r.status_code == 200 - assert r.json()["status"] == "ok" - assert r.json()["persons_loaded"] > 0 - - -def test_parse_returns_200_with_all_fields(client): - r = client.post("/parse", json={"query": "Briefe vor 1920", "lang": "de"}) - assert r.status_code == 200 - d = r.json() - assert "personNames" in d - assert d["personRole"] in ("sender", "receiver", "any") - assert "dateFrom" in d - assert "dateTo" in d - assert "keywords" in d - assert d["rawQuery"] == "Briefe vor 1920" - assert d["dateTo"] == "1920-12-31" - - -def test_parse_person_with_date(client): - r = client.post( - "/parse", - json={"query": "Briefe von Clara Cram an Walter de Gruyter im Jahr 1920", "lang": "de"}, - ) - assert r.status_code == 200 - d = r.json() - assert "Clara Cram" in d["personNames"] - assert "Walter de Gruyter" in d["personNames"] - assert d["dateFrom"] == "1920-01-01" - assert d["dateTo"] == "1920-12-31" - - -def test_parse_unknown_lang_returns_422(client): - r = client.post("/parse", json={"query": "test", "lang": "fr"}) - assert r.status_code == 422 - - -def test_parse_missing_query_returns_422(client): - r = client.post("/parse", json={"lang": "de"}) - assert r.status_code == 422 - - -def test_parse_all_languages(client): - cases = [ - ("de", "Briefe vor 1920"), - ("en", "letters before 1920"), - ("es", "cartas antes de 1920"), - ] - for lang, query in cases: - r = client.post("/parse", json={"query": query, "lang": lang}) - assert r.status_code == 200, f"Failed for lang={lang}" - assert r.json()["dateTo"] == "1920-12-31", f"Wrong dateTo for lang={lang}" - - -def test_fuzzy_threshold_valid_range(): - from main import _parse_fuzzy_threshold - assert _parse_fuzzy_threshold("80") == 80 - assert _parse_fuzzy_threshold("0") == 0 - assert _parse_fuzzy_threshold("100") == 100 - - -def test_fuzzy_threshold_out_of_range_raises(): - from main import _parse_fuzzy_threshold - with pytest.raises(ValueError): - _parse_fuzzy_threshold("101") - with pytest.raises(ValueError): - _parse_fuzzy_threshold("-1") - with pytest.raises(ValueError): - _parse_fuzzy_threshold("abc") - - -def test_parse_exceeds_max_length_returns_422(client): - r = client.post("/parse", json={"query": "a" * 501, "lang": "de"}) - assert r.status_code == 422 - - -def test_parse_internal_exception_does_not_leak_detail(client, monkeypatch): - """500 errors must return generic message — never expose internal details.""" - import main as main_module - - def _boom(query, lang): - raise RuntimeError("postgresql://archive_user:s3cr3t@db:5432/family_archive_db") - - monkeypatch.setattr(main_module, "extract", _boom) - r = client.post("/parse", json={"query": "test", "lang": "de"}) - assert r.status_code == 500 - assert "s3cr3t" not in r.text - assert r.json()["detail"] == "internal error" diff --git a/nlp-service/test_sentences.md b/nlp-service/test_sentences.md deleted file mode 100644 index 66143c3f..00000000 --- a/nlp-service/test_sentences.md +++ /dev/null @@ -1,126 +0,0 @@ -# NLP Service — Test Sentences - -Real data drawn from the Familienarchiv DB (2026-06-07). -Top persons: Clara Cram, Herbert Cram, Eugenie de Gruyter, Walter de Gruyter, Marie Cram, -Juan Cram, Albert de Gruyter, Hilde de Gruyter, Else Bohrmann, Anita Wöhler, Lili Duvenbeck. -Date range: ~1895–1945. Key tags: Krieg, Hochzeit, Reise, Geburtstag, Tod, Alltag, Briefwechsel. - ---- - -## German — full sentences - -```json -{"query": "Briefe von Clara Cram an Walter de Gruyter im Jahr 1920", "lang": "de"} -{"query": "Briefe von Herbert an Eugenie de Gruyter nach 1914", "lang": "de"} -{"query": "Schreiben von Albert de Gruyter an seine Kinder vor 1900", "lang": "de"} -{"query": "Briefe von Juan Cram an Marie zwischen 1915 und 1918", "lang": "de"} -{"query": "Telegramm von Walter de Gruyter an Clara im Jahr 1930", "lang": "de"} -{"query": "Briefe von Else Bohrmann an Herbert Cram nach 1939", "lang": "de"} -``` - -## German — medium (person + date, no strong role signal) - -```json -{"query": "Briefe von Clara Cram vor 1910", "lang": "de"} -{"query": "Dokumente über Walter de Gruyter aus den 1920er Jahren", "lang": "de"} -{"query": "Briefe an Herbert Cram nach dem Krieg", "lang": "de"} -{"query": "Schriften von Eugenie de Gruyter im Jahr 1905", "lang": "de"} -``` - -## German — short (person only) - -```json -{"query": "Briefe an Walter de Gruyter", "lang": "de"} -{"query": "Dokumente über Clara Cram", "lang": "de"} -{"query": "Herbert Cram", "lang": "de"} -{"query": "Anita Wöhler", "lang": "de"} -``` - -## German — topic only (keywords → tag resolution on Java side) - -```json -{"query": "Briefe aus dem Krieg", "lang": "de"} -{"query": "Kriegspost", "lang": "de"} -{"query": "Hochzeitsbriefe", "lang": "de"} -{"query": "Reisebriefe", "lang": "de"} -{"query": "Geburtstagsglückwünsche", "lang": "de"} -{"query": "Briefe über die Hochzeitsreise", "lang": "de"} -{"query": "Kinderbriefe", "lang": "de"} -{"query": "Familienbriefe aus dem Alltag", "lang": "de"} -{"query": "Brautbriefe", "lang": "de"} -{"query": "Kondolenzbriefe nach dem Tod von Eugenie", "lang": "de"} -``` - -## German — date range only - -```json -{"query": "Briefe aus dem Ersten Weltkrieg", "lang": "de"} -{"query": "Dokumente zwischen 1914 und 1918", "lang": "de"} -{"query": "Briefe vor 1900", "lang": "de"} -{"query": "Schriften nach 1920", "lang": "de"} -``` - -## German — combined (all fields) - -```json -{"query": "Briefe von Clara Cram an ihre Kinder über die Reise nach Mexiko im Jahr 1925", "lang": "de"} -{"query": "Kriegspost von Herbert Cram an Eugenie de Gruyter zwischen 1916 und 1918", "lang": "de"} -{"query": "Glückwünsche von Hilde de Gruyter zur Hochzeit im Jahr 1910", "lang": "de"} -{"query": "Kondolenzschreiben an Walter de Gruyter nach dem Tod von Eugenie", "lang": "de"} -``` - -## English - -```json -{"query": "Letters from Clara Cram to Walter de Gruyter in 1920", "lang": "en"} -{"query": "Letters about the war before 1918", "lang": "en"} -{"query": "Letters to Herbert Cram after 1939", "lang": "en"} -{"query": "Birthday greetings from Anita Wöhler", "lang": "en"} -{"query": "Letters between 1914 and 1918", "lang": "en"} -``` - -## Spanish - -```json -{"query": "Cartas de Clara Cram a Walter de Gruyter en 1920", "lang": "es"} -{"query": "Cartas antes de 1900", "lang": "es"} -{"query": "Cartas después de la guerra", "lang": "es"} -{"query": "Cartas de Juan Cram a sus hijos entre 1915 y 1920", "lang": "es"} -``` - ---- - -## Edge cases — lazy / missing words / typos - -```json -{"query": "Clara", "lang": "de"} -{"query": "Eugenie", "lang": "de"} -{"query": "Herbert", "lang": "de"} -{"query": "de Gruyter", "lang": "de"} -{"query": "Briefe von Klara Kram an Herbert", "lang": "de"} -{"query": "briefe von clara cram an herbert 1920", "lang": "de"} -{"query": "1918", "lang": "de"} -{"query": "1914 1918", "lang": "de"} -{"query": "Krieg", "lang": "de"} -{"query": "Briefe von Eugenie", "lang": "de"} -{"query": "Clara Cram Herbert Cram 1920", "lang": "de"} -{"query": "Wer hat an Herbert Cram 1918 geschrieben?", "lang": "de"} -{"query": "von Clara", "lang": "de"} -{"query": "an Walter", "lang": "de"} -{"query": "Clara 1920", "lang": "de"} -{"query": "Kriegsbriefe von Herbert", "lang": "de"} -{"query": "Briefe von Clara nach Herbert", "lang": "de"} -{"query": "Briefe von Herrbert Cram", "lang": "de"} -``` - ---- - -## Known spaCy failures now fixed by DB-backed matcher - -| Query | spaCy result | Expected | -|---|---|---| -| `Briefe von Eugenie` | persons=[] | persons=["Eugenie"] | -| `Kriegsbriefe von Herbert` | keywords=["herbert"] | persons=["Herbert"] | -| `Briefe von Herbert an Eugenie de Gruyter nach 1914` | persons=["Herbert an Eugenie de Gruyter"] (merged!) | persons=["Herbert", "Eugenie de Gruyter"] | -| `Letters from Clara Cram to Walter de Gruyter` | persons=[] (EN model doesn't know German names) | persons=["Clara Cram", "Walter de Gruyter"] | -| `Geburtstagsglückwünsche` | persons=["Geburtstagsglückwünsche"] (false positive!) | persons=[] | -- 2.49.1 From 62eadcacd67999d9d6985468b03d84a94ca0dfdc Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 19:04:52 +0200 Subject: [PATCH 47/51] docs(claude): remove NLP search references from CLAUDE.md files --- CLAUDE.md | 5 ++--- frontend/CLAUDE.md | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b0203020..8364399a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,7 +92,6 @@ backend/src/main/java/org/raddatz/familienarchiv/ ├── ocr/ OCR domain — OcrService, OcrBatchService, training ├── person/ Person domain │ └── relationship/ PersonRelationship sub-domain -├── search/ NL search domain — NlSearchController, NlQueryParserService, RestClientOllamaClient, NlSearchRateLimiter ├── security/ SecurityConfig, Permission, @RequirePermission, PermissionAspect ├── tag/ Tag domain └── user/ User domain — AppUser, UserGroup, UserService @@ -161,7 +160,7 @@ Input DTOs live flat in the domain package. Response types are the model entitie → See [CONTRIBUTING.md §Error handling](./CONTRIBUTING.md#error-handling) -**LLM reminder:** use `DomainException.notFound/forbidden/conflict/internal()` from service methods — never throw raw exceptions. When adding a new `ErrorCode`: (1) add to `ErrorCode.java`, (2) add to `ErrorCode` type in `frontend/src/lib/shared/errors.ts`, (3) add a `case` in `getErrorMessage()`, (4) add i18n keys in `messages/{de,en,es}.json`. Valid error codes include: `TOO_MANY_LOGIN_ATTEMPTS` (returned by `LoginRateLimiter` as HTTP 429 when a brute-force threshold is exceeded); `SMART_SEARCH_UNAVAILABLE` (HTTP 503 — Ollama inference service offline or timed out); `SMART_SEARCH_RATE_LIMITED` (HTTP 429 — user exceeded 5 NL search requests per minute). +**LLM reminder:** use `DomainException.notFound/forbidden/conflict/internal()` from service methods — never throw raw exceptions. When adding a new `ErrorCode`: (1) add to `ErrorCode.java`, (2) add to `ErrorCode` type in `frontend/src/lib/shared/errors.ts`, (3) add a `case` in `getErrorMessage()`, (4) add i18n keys in `messages/{de,en,es}.json`. Valid error codes include: `TOO_MANY_LOGIN_ATTEMPTS` (returned by `LoginRateLimiter` as HTTP 429 when a brute-force threshold is exceeded). ### Security / Permissions @@ -269,7 +268,7 @@ Back button pattern — use the shared `` component from `$lib/share → See [CONTRIBUTING.md §Error handling](./CONTRIBUTING.md#error-handling) -**LLM reminder:** when adding a new `ErrorCode`: (1) add to `ErrorCode.java`, (2) add to `ErrorCode` type in `frontend/src/lib/shared/errors.ts`, (3) add a `case` in `getErrorMessage()`, (4) add i18n keys in `messages/{de,en,es}.json`. Valid error codes include: `TOO_MANY_LOGIN_ATTEMPTS` (returned by `LoginRateLimiter` as HTTP 429 when a brute-force threshold is exceeded); `SMART_SEARCH_UNAVAILABLE` (HTTP 503 — Ollama inference service offline or timed out); `SMART_SEARCH_RATE_LIMITED` (HTTP 429 — user exceeded 5 NL search requests per minute). +**LLM reminder:** when adding a new `ErrorCode`: (1) add to `ErrorCode.java`, (2) add to `ErrorCode` type in `frontend/src/lib/shared/errors.ts`, (3) add a `case` in `getErrorMessage()`, (4) add i18n keys in `messages/{de,en,es}.json`. Valid error codes include: `TOO_MANY_LOGIN_ATTEMPTS` (returned by `LoginRateLimiter` as HTTP 429 when a brute-force threshold is exceeded). --- diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index 3301675e..a6fa8df7 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -28,7 +28,6 @@ src/ │ ├── +layout.server.ts # Loads current user, injects auth cookie │ ├── +page.svelte # Home / document search dashboard │ ├── documents/ # Document CRUD, detail, edit, upload -│ ├── search/ # Smart (NL) search sub-components — SmartModeToggle, InterpretationChipRow, SmartSearchStatus, DisambiguationPicker (no +page; consumed by documents/ and SearchFilterBar) │ ├── persons/ # Person directory (filtered, paginated), detail, edit, merge, review (triage) │ ├── aktivitaeten/ # Unified activity feed (Chronik) │ ├── admin/ # User, group, tag, OCR, system management -- 2.49.1 From 5301b52e0f326ac6ae901c5d9122668df6b89b80 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 19:12:20 +0200 Subject: [PATCH 48/51] docs: remove nlp-service and NL search references from DEPLOYMENT.md and GLOSSARY.md --- docs/DEPLOYMENT.md | 72 +++++----------------------------------------- docs/GLOSSARY.md | 7 ----- 2 files changed, 7 insertions(+), 72 deletions(-) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index e8bc6b67..4692964a 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -51,17 +51,15 @@ graph TD The OCR service requires significant RAM for model loading. The dev compose sets `mem_limit: 12g`. -| Production target | RAM | Recommended OCR limit | NL Search | Notes | -|---|---|---|---|---| -| Current server (Hetzner Serverbörse, i7-6700) | 64 GB | 12 GB | Supported | Default `mem_limit: 12g` works comfortably; nlp-service adds only ~256 MB | -| ≥ 16 GB RAM | 16+ GB | 12 GB | Supported | Default works | -| 8 GB RAM | 8 GB | 6 GB | Supported | Set `OCR_MEM_LIMIT=6g`; accept reduced batch sizes; nlp-service is lightweight | -| 4 GB RAM | 4 GB | — | Supported | Disable OCR service (`profiles: [ocr]`); run OCR on demand only; nlp-service still runs | +| Production target | RAM | Recommended OCR limit | Notes | +|---|---|---|---| +| Current server (Hetzner Serverbörse, i7-6700) | 64 GB | 12 GB | Default `mem_limit: 12g` works comfortably | +| ≥ 16 GB RAM | 16+ GB | 12 GB | Default works | +| 8 GB RAM | 8 GB | 6 GB | Set `OCR_MEM_LIMIT=6g`; accept reduced batch sizes | +| 4 GB RAM | 4 GB | — | Disable OCR service (`profiles: [ocr]`); run OCR on demand only | On servers with less than 16 GB RAM the default `mem_limit: 12g` cannot be honoured — set the `OCR_MEM_LIMIT` env var (in `.env.production` / `.env.staging`, or as a Gitea secret consumed by the workflow). The prod compose interpolates this var with a 12g default. -> **Memory budget:** OCR (~6 GB active) + nlp-service (~256 MB) = ~6.25 GB. The previous Ollama LLM (~8 GB) has been replaced by the rule-based nlp-service — significant memory headroom freed on all server tiers. - ### Dev vs production differences | Concern | Dev (`docker-compose.yml`) | Prod (`docker-compose.prod.yml`) | @@ -148,19 +146,6 @@ All vars are set in `.env` at the repo root (copy from `.env.example`). The back | `XDG_CACHE_HOME` | XDG cache base dir — redirects Matplotlib and other XDG-aware libraries away from the read-only `HOME` (`/home/ocr`) to the writable cache volume | `/app/cache` | — | — | | `TORCH_HOME` | PyTorch model cache — redirects `~/.cache/torch` to the writable models volume | `/app/models/torch` | — | — | -### NLP service (NL search) - -| Variable | Purpose | Default | Required? | Sensitive? | -|---|---|---|---|---| -| `APP_NLP_BASE_URL` | Internal URL of the nlp-service container. Wired automatically in compose via `http://nlp-service:8001`. | `http://nlp-service:8001` | YES | — | -| `NLP_FUZZY_THRESHOLD` | Rapidfuzz similarity floor for person-name matching (0–100). Lower values match more aggressively; raise if false positives appear. | `80` | — | — | - -The nlp-service reads `DATABASE_URL` at startup (composed from `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB`). Any credential rotation that touches those three vars must be followed by a restart of **both** `backend` and `nlp-service`: - -```bash -docker compose restart nlp-service backend -``` - ### Observability stack (`docker-compose.observability.yml`) | Variable | Purpose | Default | Required? | Sensitive? | @@ -281,14 +266,6 @@ git.raddatz.cloud A ### 3.4 First deploy -> **NL search startup:** `nlp-service` loads person names from the database at startup (single query, ~1–2 s). No model weights to download. The backend waits for `nlp-service` to pass its healthcheck (`/health` returns `{"status":"ok","persons_loaded":N}`) before starting, so `docker compose up -d --wait` is safe to use on first deploy. -> -> **Verify NL search is active:** -> ```bash -> curl -s http://localhost:8001/health -> # Returns {"status":"ok","persons_loaded":N} with N > 0 → person matching enabled -> # Returns {"status":"ok","persons_loaded":0} → DB not reachable or persons table empty -> ``` ```bash # 1. Trigger nightly.yml manually (Repo → Actions → nightly → "Run workflow") @@ -328,7 +305,7 @@ docker compose logs --follow # Single snapshot docker compose logs --tail=200 -# services: frontend, backend, db, minio, ocr-service, nlp-service +# services: frontend, backend, db, minio, ocr-service ``` ### Log locations @@ -585,41 +562,6 @@ bash scripts/download-kraken-models.sh > Downloads the Kurrent/Sütterlin HTR models. Run once after a fresh clone or when models are updated. -### NLP service — natural-language search (NL Search) - -NL search uses the rule-based `nlp-service` FastAPI container for query parsing. It has no model weights — it loads person names from the database at startup and applies regex + fuzzy matching. See ADR-035. - -**Health check:** - -```bash -curl -s http://localhost:8001/health -# {"status":"ok","persons_loaded":1247} -``` - -`persons_loaded: 0` means the service started but could not reach the database (check `DATABASE_URL` and that `db` is healthy). - -If `POST /api/search/nl` returns HTTP 503 `SMART_SEARCH_UNAVAILABLE`, the backend cannot reach `nlp-service`. Check with: - -```bash -docker compose logs nlp-service --tail=50 -docker compose ps nlp-service -``` - -**Configuration** (see `application.yaml` under `app.nlp`): - -| Property | Default | Description | -|---|---|---| -| `app.nlp.base-url` | `http://nlp-service:8001` | nlp-service URL; set via `APP_NLP_BASE_URL` env var | -| `app.nl-search.rate-limit.max-requests-per-minute` | `20` | Per-user rate limit | - -**Tuning person matching:** - -Set `NLP_FUZZY_THRESHOLD` in `.env` (default: `80`, range: `0–100`). Lower values match more aggressively at the cost of false positives. Restart nlp-service after changing: - -```bash -docker compose restart nlp-service -``` - ### Trigger a canonical import The importer no longer parses the raw spreadsheet. It consumes the **canonical artifacts** diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md index e30cf875..0af61be9 100644 --- a/docs/GLOSSARY.md +++ b/docs/GLOSSARY.md @@ -165,13 +165,6 @@ _See also [Chronik](#chronik-internal)._ **Domain** — a Tier-1 bounded context with its own entities, controller, service, repository, and DTOs. Backend domains: `document`, `person`, `tag`, `user`, `geschichte`, `notification`, `ocr`, `audit`, `dashboard`. Frontend domains mirror this structure under `src/lib/`. ---- - -## NL Search Terms - -**NlSearch** — the natural-language document search feature. Users type a plain-German query (e.g. "Was hat Walter im Krieg an Emma geschrieben?"); the backend parses it via Ollama, resolves person names to database UUIDs, and delegates to the standard `DocumentService.searchDocuments()` path. Endpoint: `POST /api/search/nl`. - -**NlQueryInterpretation** — the structured result of parsing a natural-language query. Contains: `resolvedPersons` (persons whose names unambiguously matched one DB record), `ambiguousPersons` (all candidates when a name matched more than one person), `keywords` (LLM-extracted search terms), `dateFrom`/`dateTo` (extracted date range), `rawQuery` (the original user input), `keywordsApplied` (whether keyword FTS was used), `resolvedTags` (tags matched by keyword→tag resolution), and `tagsApplied` (whether the OR-union tag filter was applied). **keyword→tag resolution** — the post-Ollama step in `NlQueryParserService` where each LLM-extracted keyword is substring-matched against the tag taxonomy via `TagService.findByNameContaining()`. Keywords that hit one or more tags are removed from the FTS text list and become an OR-union tag filter; keywords with no match remain as FTS text. Matching is case-insensitive and traverses the tag hierarchy via the recursive CTE `findDescendantIdsByName`. See ADR-033. -- 2.49.1 From 75c9dc4be9308e2636aa0ec231be520a69391b1f Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 19:37:17 +0200 Subject: [PATCH 49/51] docs(c4): remove NLP service from L2 container diagram; delete NL search L3 diagram --- docs/architecture/c4/l2-containers.puml | 2 - .../architecture/c4/l3-backend-3h-search.puml | 37 ------------------- 2 files changed, 39 deletions(-) delete mode 100644 docs/architecture/c4/l3-backend-3h-search.puml diff --git a/docs/architecture/c4/l2-containers.puml b/docs/architecture/c4/l2-containers.puml index 9592b649..46d80101 100644 --- a/docs/architecture/c4/l2-containers.puml +++ b/docs/architecture/c4/l2-containers.puml @@ -12,7 +12,6 @@ System_Boundary(archiv, "Familienarchiv (Docker Compose)") { Container(frontend, "Web Frontend", "SvelteKit / Node adapter / port 3000", "Server-side rendered UI. Handles auth session cookies, document search and viewer, transcription editor, annotation layer, family tree (Stammbaum), stories (Geschichten), activity feed (Chronik), enrichment workflow, and admin panel.") Container(backend, "API Backend", "Spring Boot 4 / Java 21 / Jetty / port 8080", "REST API. Implements document management, search, user auth, file upload/download, transcription, OCR orchestration, and SSE notifications. Trusts X-Forwarded-* headers from Caddy.") Container(ocr, "OCR Service", "Python FastAPI / port 8000", "Handwritten text recognition (HTR) and OCR microservice. Single-node by design — see ADR-001. Reachable only on the internal Docker network; no external port exposed.") - Container(nlp, "NLP Service", "Python FastAPI / port 8001 (internal only)", "Rule-based NL search query parser. Extracts person names (fuzzy DB lookup), date ranges (regex), person role, and keywords. Reachable only on the internal Docker network; no external port exposed.") ContainerDb(db, "Relational Database", "PostgreSQL 16", "Stores document metadata, persons, users, permission groups, tags, transcription blocks, audit log, and Spring Session data.") ContainerDb(storage, "Object Storage", "MinIO (S3-compatible)", "Stores the actual document files (PDFs, scans). Backend uses a bucket-scoped service account (archiv-app), not MinIO root.") Container(mc, "Bucket / Service-Account Init", "MinIO Client (mc)", "One-shot container on startup. Idempotent: creates the archive bucket, the archiv-app service account, and attaches the readwrite policy.") @@ -42,7 +41,6 @@ Rel(backend, ocr, "OCR job requests with presigned MinIO URL", "HTTP / REST / JS Rel(backend, mail, "Sends notification and password-reset emails (optional)", "SMTP") Rel(ocr, storage, "Fetches PDF via presigned URL", "HTTP / S3 presigned") Rel(mc, storage, "Bootstraps bucket + service account on startup", "MinIO Client CLI") -Rel(backend, nlp, "NL query parsing (POST /parse)", "HTTP / REST / JSON") Rel(promtail, loki, "Pushes log streams", "HTTP/Loki push API") Rel(backend, tempo, "Sends distributed traces via OTLP", "HTTP / OTLP / port 4318 (archiv-net)") Rel(prometheus, backend, "Scrapes JVM + HTTP metrics", "HTTP 8081 /actuator/prometheus") diff --git a/docs/architecture/c4/l3-backend-3h-search.puml b/docs/architecture/c4/l3-backend-3h-search.puml deleted file mode 100644 index 7ffa28be..00000000 --- a/docs/architecture/c4/l3-backend-3h-search.puml +++ /dev/null @@ -1,37 +0,0 @@ -@startuml -!include - -title Component Diagram: API Backend — NL Search - -Container(frontend, "Web Frontend", "SvelteKit") -ContainerDb(db, "PostgreSQL", "PostgreSQL 16") -Container(ollama, "Ollama", "ollama/ollama — port 11434 (internal only)") - -System_Boundary(backend, "API Backend (Spring Boot)") { - Component(nlCtrl, "NlSearchController", "Spring MVC — POST /api/search/nl", "REST entry point for natural language search. Enforces READ_ALL permission. Uses @AuthenticationPrincipal UserDetails to obtain the caller's email for rate limiting. Delegates to NlQueryParserService and returns NlSearchResponse.") - Component(rateLimiter, "NlSearchRateLimiter", "Spring Service", "Bucket4j + Caffeine LoadingCache keyed on user email. Allows 5 NL search requests per minute per user. Throws DomainException(SMART_SEARCH_RATE_LIMITED / HTTP 429) when the bucket is exhausted. Node-local — same caveat as LoginRateLimiter.") - Component(parserSvc, "NlQueryParserService", "Spring Service", "Orchestrates the full NL search pipeline: (1) validates query length, (2) calls OllamaClient.parse() to extract structured intent, (3) resolves keywords to tags via TagService.findByNameContaining(), (4) resolves each person name via PersonService.findByDisplayNameContaining(), (5) applies multi-name / personRole heuristics, (6) delegates to DocumentService.searchDocuments() or searchDocumentsByPersonId(). Returns NlSearchResponse. Never logs raw query content (PII).") - Component(ollamaClient, "RestClientOllamaClient", "Spring Service — implements OllamaClient + OllamaHealthClient", "HTTP client for the Ollama API. Uses two separate RestClient instances: inference client (30 s read timeout) and health-check client (2 s connect timeout). Calls POST /api/generate with grammar-constrained JSON schema (personNames, personRole, dateFrom, dateTo, keywords). isHealthy() polls GET /api/tags. Null-coalesces absent personNames/keywords to List.of(). Defaults unknown personRole to 'any' with a warning log. Maps timeout/5xx/parse errors to DomainException(SMART_SEARCH_UNAVAILABLE / HTTP 503).") - Component(ollamaProps, "OllamaProperties", "@ConfigurationProperties(\"app.ollama\")", "Config bean: baseUrl, model (qwen2.5:7b-instruct-q4_K_M), timeoutSeconds (default: 30), healthCheckTimeoutSeconds (default: 2).") - Component(rateLimitProps, "NlSearchRateLimitProperties", "@ConfigurationProperties(\"app.nl-search.rate-limit\")", "Config bean: maxRequestsPerMinute (default: 5).") -} - -Component(personSvc, "PersonService", "Spring Service", "See diagram 3e. findByDisplayNameContaining(fragment) delegates to PersonRepository.searchByName() — covers first+last name, alias, and name aliases via LEFT JOIN.") -Component(documentSvc, "DocumentService", "Spring Service", "See diagram 3b. searchDocuments() for keyword/sender/receiver/date queries. searchDocumentsByPersonId() for OR-semantics single-person queries (person as sender OR receiver, no keyword filter).") -Component(tagSvc, "TagService", "Spring Service", "See diagram 3b. findByNameContaining(fragment) delegates to TagRepository.findByNameContainingIgnoreCase(). resolveEffectiveColors() applies one-level color inheritance in-place on a collection of Tag entities.") - -Rel(frontend, nlCtrl, "POST /api/search/nl with JSON query", "HTTP / JSON") -Rel(nlCtrl, rateLimiter, "checkAndConsume(userEmail)") -Rel(nlCtrl, parserSvc, "parse(query)") -Rel(parserSvc, ollamaClient, "parse(rawQuery) — extracts intent", "HTTP / JSON") -Rel(ollamaClient, ollama, "POST /api/generate (grammar-constrained JSON schema)", "HTTP / REST") -Rel(ollamaClient, ollama, "GET /api/tags (health check)", "HTTP / REST") -Rel(parserSvc, tagSvc, "findByNameContaining(keyword) — keyword→tag resolution") -Rel(parserSvc, tagSvc, "resolveEffectiveColors(tags)") -Rel(parserSvc, personSvc, "findByDisplayNameContaining(name) for each extracted name") -Rel(parserSvc, documentSvc, "searchDocuments() or searchDocumentsByPersonId()") -Rel(documentSvc, db, "JPA queries", "JDBC") -Rel(personSvc, db, "JPA queries", "JDBC") -Rel(tagSvc, db, "JPA queries", "JDBC") - -@enduml -- 2.49.1 From acc73fd3e1227b62b9303a6e6fe84fd223ff6793 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 7 Jun 2026 19:50:48 +0200 Subject: [PATCH 50/51] fix(review): resolve all review blockers and concerns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delete frontend/e2e/nl-search.spec.ts (was left alive; would have crashed CI when Playwright couldn't find the deleted SmartModeToggle) - Fix docs/DEPLOYMENT.md: remove NLP service arrow + key-facts bullet that were accidentally added instead of removed in the prior commit - Clean docs/GLOSSARY.md: remove keyword→tag resolution, PersonHint, TagHint, theme chip entries; trim NameMatches to drop the NlQueryParserService reference - Remove @ConfigurationPropertiesScan from FamilienarchivApplication (all remaining @ConfigurationProperties beans carry @Component) - Remove 12 orphaned i18n keys from de/en/es message files (search_loading_nl, search_chip_*, search_disambiguation_*, etc.) - Fix SearchFilterBar.svelte input padding: pr-20 → pr-4 (SmartModeToggle that justified the right padding is gone) - Delete docs/superpowers/plans/2026-06-07-remove-nlp-search.md (scaffolding artefact; plan files belong in Gitea issues, not the repo) - Add docs/adr/034-remove-nl-search.md documenting the removal decision (supersedes deleted ADR-028 ×2, ADR-034-ollama, ADR-035) Co-Authored-By: Claude Sonnet 4.6 --- .../FamilienarchivApplication.java | 2 - docs/DEPLOYMENT.md | 3 +- docs/GLOSSARY.md | 11 +- docs/adr/034-remove-nl-search.md | 53 ++ .../plans/2026-06-07-remove-nlp-search.md | 768 ------------------ frontend/e2e/nl-search.spec.ts | 115 --- frontend/messages/de.json | 12 - frontend/messages/en.json | 12 - frontend/messages/es.json | 12 - frontend/src/routes/SearchFilterBar.svelte | 2 +- 10 files changed, 56 insertions(+), 934 deletions(-) create mode 100644 docs/adr/034-remove-nl-search.md delete mode 100644 docs/superpowers/plans/2026-06-07-remove-nlp-search.md delete mode 100644 frontend/e2e/nl-search.spec.ts diff --git a/backend/src/main/java/org/raddatz/familienarchiv/FamilienarchivApplication.java b/backend/src/main/java/org/raddatz/familienarchiv/FamilienarchivApplication.java index 0b358e80..4fef338f 100644 --- a/backend/src/main/java/org/raddatz/familienarchiv/FamilienarchivApplication.java +++ b/backend/src/main/java/org/raddatz/familienarchiv/FamilienarchivApplication.java @@ -2,10 +2,8 @@ package org.raddatz.familienarchiv; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.context.properties.ConfigurationPropertiesScan; @SpringBootApplication -@ConfigurationPropertiesScan public class FamilienarchivApplication { public static void main(String[] args) { diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 4692964a..9368d782 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -33,7 +33,6 @@ graph TD Backend -->|JDBC :5432| DB[(PostgreSQL 16)] Backend -->|S3 API :9000| MinIO[(MinIO)] Backend -->|HTTP :8000 internal| OCR["OCR Service\nPython FastAPI"] - Backend -->|HTTP :8001 internal| NLP["NLP Service\nPython FastAPI"] OCR -->|presigned URL| MinIO Caddy -->|SSE proxy_pass| Backend ``` @@ -41,7 +40,7 @@ graph TD **Key facts:** - Caddy terminates TLS and reverse-proxies to frontend (`:3000`) and backend (`:8080`). The Caddyfile is committed at [`infra/caddy/Caddyfile`](../infra/caddy/Caddyfile) and is installed on the host as `/etc/caddy/Caddyfile` (symlink). - The host binds all docker-published ports to `127.0.0.1` only; Caddy is the sole external entry point. -- The OCR service and NLP service have **no published ports** — reachable only on the internal Docker network from the backend. +- The OCR service has **no published port** — reachable only on the internal Docker network from the backend. - SSE notifications transit Caddy (browser → Caddy → backend); the backend is never reachable directly from the public internet. The SvelteKit SSR layer is bypassed for SSE, but Caddy is not. - The Caddyfile responds `404` on `/actuator/*` (defense in depth). Internal monitoring scrapes the backend on the docker network, not through Caddy. - Production and staging cohabit on the same host via docker compose project names: `archiv-production` (ports 8080/3000) and `archiv-staging` (ports 8081/3001). diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md index 0af61be9..d7f05bb7 100644 --- a/docs/GLOSSARY.md +++ b/docs/GLOSSARY.md @@ -165,16 +165,7 @@ _See also [Chronik](#chronik-internal)._ **Domain** — a Tier-1 bounded context with its own entities, controller, service, repository, and DTOs. Backend domains: `document`, `person`, `tag`, `user`, `geschichte`, `notification`, `ocr`, `audit`, `dashboard`. Frontend domains mirror this structure under `src/lib/`. - -**keyword→tag resolution** — the post-Ollama step in `NlQueryParserService` where each LLM-extracted keyword is substring-matched against the tag taxonomy via `TagService.findByNameContaining()`. Keywords that hit one or more tags are removed from the FTS text list and become an OR-union tag filter; keywords with no match remain as FTS text. Matching is case-insensitive and traverses the tag hierarchy via the recursive CTE `findDescendantIdsByName`. See ADR-033. - -**PersonHint** — a lightweight `{id, displayName}` pair used in `NlQueryInterpretation` to describe a resolved or ambiguous person without exposing the full `Person` entity to the frontend. - -**NameMatches** — the Person-domain result of `PersonService.resolveByName(name)`: candidate persons split by name-match strength into `direct` and `partial`. A match is **direct** when every query token is a whole-token match (order-independent, alias/maiden-name aware) across all of a person's name components (`firstName`, `lastName`, `alias`, each `PersonNameAlias` first+last, `title`); a **partial** matched the substring fetch but is not direct (e.g. "Cram" → "Clara Cramer"). The vocabulary is deliberately match strength, not the search layer's resolved/ambiguous buckets — `NlQueryParserService` maps one direct → resolved (auto-select), ≥2 direct → ambiguous, partial-only → ambiguous suggestions ("Meintest du …?"), and no candidates → folded into full-text search. - -**TagHint** — a lightweight `{id, name, color?}` triple used in `NlQueryInterpretation.resolvedTags` to describe a tag matched by keyword→tag resolution. `color` is the tag's effective color (one-level inheritance from parent when the tag has no own color), or null if neither tag nor parent has a color. - -**theme chip** `[frontend]` — a removable chip rendered in `InterpretationChipRow` for each entry in `NlQueryInterpretation.resolvedTags` when `tagsApplied` is `true`. Displays "Thema: {tag.name}" (prefix varies by locale). Clicking × removes the tag from the OR-union filter and navigates to `/documents?tag=…&tagOp=OR` with remaining tag and person parameters preserved. +**NameMatches** — the Person-domain result of `PersonService.resolveByName(name)`: candidate persons split by name-match strength into `direct` and `partial`. A match is **direct** when every query token is a whole-token match (order-independent, alias/maiden-name aware) across all of a person's name components (`firstName`, `lastName`, `alias`, each `PersonNameAlias` first+last, `title`); a **partial** matched the substring fetch but is not direct (e.g. "Cram" → "Clara Cramer"). --- diff --git a/docs/adr/034-remove-nl-search.md b/docs/adr/034-remove-nl-search.md new file mode 100644 index 00000000..6f9e1ca6 --- /dev/null +++ b/docs/adr/034-remove-nl-search.md @@ -0,0 +1,53 @@ +# ADR-034 — Remove NL/smart-search (supersedes ADR-028 ×2, ADR-034-ollama, ADR-035) + +**Date:** 2026-06-07 +**Status:** Accepted +**Issue:** #772 +**Supersedes:** ADR-028 (nl-search-ollama), ADR-028 (ollama-docker-compose-service), ADR-034 (ollama-production-deployment-and-keep-alive), ADR-035 (rule-based-nlp-service) + +--- + +## Context + +The natural-language search feature ("KI-Suche" / smart search) allowed users to enter +free-form queries like *"Was hat Walter an Emma im Krieg geschrieben?"* and have them +interpreted by an LLM into structured filters (persons, tags, date range, keywords). + +The feature went through two major iterations: +1. **Ollama integration** (ADR-028): an `ollama` Docker service running a local LLM + (llama3.2/gemma3) parsed queries via a JSON-mode prompt. +2. **Rule-based NLP service** (ADR-035): after Ollama proved too slow and unreliable on + CPU-only hardware, a Python FastAPI microservice (`nlp-service`, port 8001) replaced + it with deterministic regex + spaCy parsing plus a lightweight LLM call. + +Both approaches shared the same fundamental problem: inference on the production server +(Hetzner Serverbörse, no GPU, 64 GB RAM, i7-6700) was too slow to be useful, with +typical query latencies of 10–30 seconds. Users got better and faster results from +the existing keyword search with date/person/tag filters. + +## Decision + +**Remove the NL search feature entirely.** The Python `nlp-service` microservice, the +Spring Boot `search/` package (`NlSearchController`, `NlQueryParserService`, +`RestClientNlpClient`, `NlSearchRateLimiter`, and all supporting classes), the frontend +NL search components (`SmartModeToggle`, `SmartSearchStatus`, `InterpretationChipRow`, +`DisambiguationPicker`), the related Docker Compose services, Prometheus scrape job, +Grafana dashboard, and all i18n keys are removed. + +The existing structured search (FTS keyword + person/tag/date/directional filters) is +sufficient for the archive's current audience and search workload. + +## Consequences + +- **Capability removed:** users can no longer enter free-form natural-language queries. + They must use the structured filter bar (keyword text box + person/tag/date/directional + dropdowns). For documents where these filters are sufficient, there is no regression. +- **Operational simplification:** the Docker Compose stack loses two services + (`nlp-service` and previously `ollama`/`ollama-model-init`). Memory budget on the + production host is freed. No external model weights need to be kept warm. +- **Future reinstatement:** if a GPU-capable host becomes available, re-implementing + server-side LLM inference would be straightforward given the clean separation of the + `NlSearchController` entry point. However, this ADR deliberately avoids leaving dead + infrastructure or stub code in place — start clean if and when that becomes viable. +- **No data or schema change:** only query/endpoint code and Docker services are removed. + The `documents`, `persons`, and `tags` tables and their FTS indexes are untouched. diff --git a/docs/superpowers/plans/2026-06-07-remove-nlp-search.md b/docs/superpowers/plans/2026-06-07-remove-nlp-search.md deleted file mode 100644 index 635a4b66..00000000 --- a/docs/superpowers/plans/2026-06-07-remove-nlp-search.md +++ /dev/null @@ -1,768 +0,0 @@ -# Remove NLP/Smart Search Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Remove the NLP/smart-search feature entirely from the codebase — backend search package, frontend components, i18n keys, infrastructure config, and the nlp-service microservice. - -**Architecture:** Pure deletion + targeted edits. No new code. Each task deletes a self-contained layer, then verifies compilation passes before committing. Order: backend first (most isolated), then frontend, then infrastructure, then docs. - -**Tech Stack:** Spring Boot 4 (Java 21, Maven), SvelteKit 2 / Svelte 5, Docker Compose, Paraglide i18n. - ---- - -## File Map - -### Delete entirely -- `backend/src/main/java/org/raddatz/familienarchiv/search/` — 14 Java source files -- `backend/src/test/java/org/raddatz/familienarchiv/search/` — 6 Java test files -- `frontend/src/routes/search/SmartModeToggle.svelte` + `.spec.ts` -- `frontend/src/routes/search/SmartSearchStatus.svelte` + `.spec.ts` -- `frontend/src/routes/search/InterpretationChipRow.svelte` + `.spec.ts` -- `frontend/src/routes/search/DisambiguationPicker.svelte` + `.spec.ts` -- `frontend/src/routes/search/chip-types.ts` -- `frontend/src/routes/documents/theme-chip-removal.ts` + `.spec.ts` -- `infra/observability/grafana/provisioning/dashboards/ollama.json` -- `nlp-service/` (entire directory) -- `docs/adr/028-nl-search-ollama.md` -- `docs/adr/028-ollama-docker-compose-service.md` -- `docs/adr/034-ollama-production-deployment-and-keep-alive.md` -- `docs/adr/035-rule-based-nlp-service.md` - -### Modify -- `backend/src/main/java/org/raddatz/familienarchiv/exception/ErrorCode.java` — remove 2 enum values -- `backend/src/main/resources/application.yaml` — remove `nlp` + `nl-search` config blocks -- `backend/src/main/resources/application-dev.yaml` — remove `nlp` config block -- `frontend/src/routes/SearchFilterBar.svelte` — remove SmartModeToggle, smartMode prop, smart callbacks -- `frontend/src/routes/SearchFilterBar.svelte.spec.ts` — remove smart-mode describe block -- `frontend/src/routes/documents/+page.svelte` — remove all NL state, functions, template block -- `frontend/src/lib/shared/errors.ts` — remove 2 error codes + their switch cases -- `frontend/messages/de.json` — remove 8 smart-search keys -- `frontend/messages/en.json` — remove 8 smart-search keys -- `frontend/messages/es.json` — remove 8 smart-search keys -- `docker-compose.yml` — remove nlp-service block + backend depends_on + env var -- `docker-compose.prod.yml` — remove nlp-service block + backend depends_on + env var -- `infra/observability/prometheus/prometheus.yml` — remove ollama scrape job -- `CLAUDE.md` — remove search package reference + error code entries -- `backend/CLAUDE.md` — no change needed (search package already absent from structure) -- `frontend/CLAUDE.md` — update routes/search/ description - ---- - -### Task 1: Delete backend search package - -**Files:** -- Delete: `backend/src/main/java/org/raddatz/familienarchiv/search/` (14 files) -- Delete: `backend/src/test/java/org/raddatz/familienarchiv/search/` (6 files) - -- [ ] **Step 1: Delete all source files** - -```bash -rm -rf backend/src/main/java/org/raddatz/familienarchiv/search -rm -rf backend/src/test/java/org/raddatz/familienarchiv/search -``` - -- [ ] **Step 2: Verify backend compiles** - -```bash -cd backend && . ~/.sdkman/candidates/java/current/bin/../.. && source ~/.sdkman/bin/sdkman-init.sh && ./mvnw compile -q -``` - -Expected: BUILD SUCCESS with no errors. - -- [ ] **Step 3: Commit** - -```bash -git add -A -git commit -m "refactor(search): delete backend NLP search package" -``` - ---- - -### Task 2: Remove ErrorCode entries and backend config - -**Files:** -- Modify: `backend/src/main/java/org/raddatz/familienarchiv/exception/ErrorCode.java:138-142` -- Modify: `backend/src/main/resources/application.yaml:133-138` -- Modify: `backend/src/main/resources/application-dev.yaml:15-17` - -- [ ] **Step 1: Remove NL Search enum values from ErrorCode.java** - -Remove these lines (138–142): -```java - // --- NL Search --- - /** Ollama is unreachable or timed out. 503 */ - SMART_SEARCH_UNAVAILABLE, - /** NL search rate limit exceeded (5 requests per user per minute). 429 */ - SMART_SEARCH_RATE_LIMITED, -``` - -The block between `TAG_MERGE_INVALID_TARGET,` and `// --- Generic ---` becomes empty. - -- [ ] **Step 2: Remove nlp and nl-search config from application.yaml** - -Remove these lines (133–138): -```yaml - nlp: - base-url: http://nlp-service:8001 - - nl-search: - rate-limit: - max-requests-per-minute: 20 -``` - -- [ ] **Step 3: Remove nlp config from application-dev.yaml** - -Remove these lines (15–17): -```yaml -app: - nlp: - base-url: http://localhost:8001 -``` - -Note: only remove the `nlp:` sub-key under `app:`, preserving any other `app:` config above it. - -- [ ] **Step 4: Verify backend still compiles** - -```bash -cd backend && source ~/.sdkman/bin/sdkman-init.sh && ./mvnw compile -q -``` - -Expected: BUILD SUCCESS. - -- [ ] **Step 5: Commit** - -```bash -git add backend/src/main/java/org/raddatz/familienarchiv/exception/ErrorCode.java \ - backend/src/main/resources/application.yaml \ - backend/src/main/resources/application-dev.yaml -git commit -m "refactor(search): remove NLP error codes and application config" -``` - ---- - -### Task 3: Delete frontend NL search components and utilities - -**Files:** -- Delete: `frontend/src/routes/search/SmartModeToggle.svelte` + `.spec.ts` -- Delete: `frontend/src/routes/search/SmartSearchStatus.svelte` + `.spec.ts` -- Delete: `frontend/src/routes/search/InterpretationChipRow.svelte` + `.spec.ts` -- Delete: `frontend/src/routes/search/DisambiguationPicker.svelte` + `.spec.ts` -- Delete: `frontend/src/routes/search/chip-types.ts` -- Delete: `frontend/src/routes/documents/theme-chip-removal.ts` + `.spec.ts` - -- [ ] **Step 1: Delete all NL search components, specs, and utilities** - -```bash -rm frontend/src/routes/search/SmartModeToggle.svelte \ - frontend/src/routes/search/SmartModeToggle.svelte.spec.ts \ - frontend/src/routes/search/SmartSearchStatus.svelte \ - frontend/src/routes/search/SmartSearchStatus.svelte.spec.ts \ - frontend/src/routes/search/InterpretationChipRow.svelte \ - frontend/src/routes/search/InterpretationChipRow.svelte.spec.ts \ - frontend/src/routes/search/DisambiguationPicker.svelte \ - frontend/src/routes/search/DisambiguationPicker.svelte.spec.ts \ - frontend/src/routes/search/chip-types.ts \ - frontend/src/routes/documents/theme-chip-removal.ts \ - frontend/src/routes/documents/theme-chip-removal.spec.ts -``` - -- [ ] **Step 2: Commit** - -```bash -git add -A -git commit -m "refactor(search): delete frontend NLP search components and utilities" -``` - ---- - -### Task 4: Remove NL search from SearchFilterBar - -**Files:** -- Modify: `frontend/src/routes/SearchFilterBar.svelte` -- Modify: `frontend/src/routes/SearchFilterBar.svelte.spec.ts:199-233` - -- [ ] **Step 1: Rewrite SearchFilterBar.svelte** - -Replace the entire ` -``` - -- [ ] **Step 2: Update the search input element in the template** - -Replace the `` element (lines 92–105) with: -```svelte - -``` - -- [ ] **Step 3: Remove the SmartModeToggle component from the template** - -Delete this line (135): -```svelte - -``` - -- [ ] **Step 4: Remove smart-mode describe block from SearchFilterBar.svelte.spec.ts** - -Delete lines 199–233 (the entire final `describe` block): -```typescript -describe('SearchFilterBar – smart-mode chip lifecycle hooks', () => { - // ... -}); -``` - -- [ ] **Step 5: Run the SearchFilterBar tests to verify they pass** - -```bash -cd frontend && source ~/.nvm/nvm.sh && npm run test -- --project=client src/routes/SearchFilterBar.svelte.spec.ts -``` - -Expected: all tests pass, no failures. - -- [ ] **Step 6: Commit** - -```bash -git add frontend/src/routes/SearchFilterBar.svelte \ - frontend/src/routes/SearchFilterBar.svelte.spec.ts -git commit -m "refactor(search): remove smart mode from SearchFilterBar" -``` - ---- - -### Task 5: Remove NL search from documents/+page.svelte - -**Files:** -- Modify: `frontend/src/routes/documents/+page.svelte` - -This is the largest edit. Remove all NL search state, derived values, functions, and the NL results template block. - -- [ ] **Step 1: Remove NL search imports (lines 11–16, 23–27)** - -Remove these import lines: -```typescript -import SmartSearchStatus from '../search/SmartSearchStatus.svelte'; -import InterpretationChipRow from '../search/InterpretationChipRow.svelte'; -import type { ChipType } from '../search/chip-types.js'; -import { buildThemeRemovalUrl } from './theme-chip-removal.js'; -import DisambiguationPicker from '../search/DisambiguationPicker.svelte'; -``` - -Remove these type aliases: -```typescript -type NlQueryInterpretation = components['schemas']['NlQueryInterpretation']; -type NlSearchResponse = components['schemas']['NlSearchResponse']; -type DocumentSearchResult = components['schemas']['DocumentSearchResult']; -type PersonHint = components['schemas']['PersonHint']; -type SmartSearchErrorCode = 'SMART_SEARCH_UNAVAILABLE' | 'SMART_SEARCH_RATE_LIMITED'; -``` - -Also remove the `import { csrfFetch } from '$lib/shared/cookies';` line — it is only used by `runSmartSearch`. - -- [ ] **Step 2: Remove all NL state and derived values (lines 51–70)** - -Remove these declarations: -```typescript -// Smart (NL) search — UI-local state, resets on real page navigation (away + back). -let smartMode = $state(false); -let nlSubmitted = $state(false); -let nlLoading = $state(false); -let nlError = $state(null); -let nlInterpretation = $state(null); -let nlResult = $state(null); - -const showNlView = $derived(smartMode && nlSubmitted); -const nlHasResults = $derived((nlResult?.items.length ?? 0) > 0); -const ambiguousPersons = $derived(nlInterpretation?.ambiguousPersons ?? []); -const nlIsAmbiguous = $derived(ambiguousPersons.length > 0); -const disambiguationHeading = $derived( - ambiguousPersons.length === 1 - ? m.search_disambiguation_did_you_mean({ name: ambiguousPersons[0].displayName }) - : m.search_disambiguation_heading() -); -const showDisambiguationCue = $derived(ambiguousPersons.length >= 2); -``` - -- [ ] **Step 3: Remove all NL search functions (lines 202–318)** - -Remove these functions entirely: -- `resetNlState()` -- `onModeToggle()` -- `runSmartSearch()` -- `switchToKeywordMode()` -- `applyResolvedAndSearch()` -- `paramsFromInterpretation()` -- `removeChip()` -- `selectDisambiguated()` - -- [ ] **Step 4: Update SearchFilterBar usage in the template** - -Replace the SearchFilterBar call with (removing `bind:smartMode`, `onSmartSearch`, `onModeToggle`): -```svelte - (qFocused = true)} - onblur={() => (qFocused = false)} - /> -``` - -- [ ] **Step 5: Remove the NL results template block** - -Replace the entire `{#if showNlView}...{:else}...{/if}` block with just the content of the `{:else}` branch — the `