feat(nlp-service): replace spaCy NER with DB-backed PersonMatcher
Rule-based pipeline: persons matched via rapidfuzz against all known names loaded from DB at startup. Fixes first-name-only extraction (Eugenie, Herbert), merged-span bug (Herbert + Eugenie de Gruyter), false positives on compound nouns, and EN/ES model failures. Date extraction unchanged (regex). No spaCy models required. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Marcel
@@ -1,350 +1,337 @@
|
||||
"""Tests for the rule-based extractor and PersonMatcher."""
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
||||
# ── Models ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_parse_request_valid():
|
||||
from models import ParseRequest
|
||||
req = ParseRequest(query="Briefe von Opa", lang="de")
|
||||
assert req.query == "Briefe von Opa"
|
||||
assert req.lang == "de"
|
||||
|
||||
|
||||
def test_parse_request_rejects_unknown_lang():
|
||||
from models import ParseRequest
|
||||
with pytest.raises(ValidationError):
|
||||
ParseRequest(query="Letters from grandpa", lang="fr")
|
||||
|
||||
|
||||
def test_parse_response_serializes_nulls():
|
||||
from models import ParseResponse
|
||||
resp = ParseResponse(
|
||||
personNames=["Opa"],
|
||||
personRole="sender",
|
||||
dateFrom=None,
|
||||
dateTo="1920-12-31",
|
||||
keywords=["brief"],
|
||||
rawQuery="Briefe von Opa",
|
||||
)
|
||||
data = resp.model_dump()
|
||||
assert data["dateFrom"] is None
|
||||
assert data["dateTo"] == "1920-12-31"
|
||||
assert data["personRole"] == "sender"
|
||||
|
||||
|
||||
# ── Model loading ────────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def nlp_de():
|
||||
from extractor import get_nlp
|
||||
return get_nlp("de")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def nlp_en():
|
||||
from extractor import get_nlp
|
||||
return get_nlp("en")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def nlp_es():
|
||||
from extractor import get_nlp
|
||||
return get_nlp("es")
|
||||
|
||||
|
||||
def test_get_nlp_de_loads(nlp_de):
|
||||
doc = nlp_de("Test")
|
||||
assert doc is not None
|
||||
|
||||
|
||||
def test_get_nlp_en_loads(nlp_en):
|
||||
doc = nlp_en("Test")
|
||||
assert doc is not None
|
||||
|
||||
|
||||
def test_get_nlp_es_loads(nlp_es):
|
||||
doc = nlp_es("Prueba")
|
||||
assert doc is not None
|
||||
|
||||
|
||||
def test_get_nlp_unknown_lang_raises():
|
||||
from extractor import get_nlp
|
||||
with pytest.raises(ValueError, match="Unsupported language"):
|
||||
get_nlp("fr")
|
||||
|
||||
|
||||
# ── Person name extraction ───────────────────────────────────────────────────
|
||||
|
||||
def _make_doc_with_ents(nlp, text: str, char_ents: list[tuple[int, int, str]]):
|
||||
"""Create a Doc with manually injected entity spans (no NER model needed)."""
|
||||
doc = nlp.make_doc(text)
|
||||
spans = [doc.char_span(s, e, label=lbl) for s, e, lbl in char_ents]
|
||||
doc.ents = [sp for sp in spans if sp is not None]
|
||||
return doc
|
||||
|
||||
|
||||
def test_extract_person_names_two_persons(nlp_de):
|
||||
from extractor import extract_person_names
|
||||
# "Briefe von Opa Hermann an Marie"
|
||||
# "Opa Hermann" = chars 11..22, "Marie" = chars 26..31
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe von Opa Hermann an Marie", [
|
||||
(11, 22, "PER"),
|
||||
(26, 31, "PER"),
|
||||
])
|
||||
assert extract_person_names(doc) == ["Opa Hermann", "Marie"]
|
||||
|
||||
|
||||
def test_extract_person_names_preserves_order(nlp_de):
|
||||
from extractor import extract_person_names
|
||||
# "Marie von Opa" — Marie comes first in text
|
||||
# "Marie" = 0..5, "Opa" = 10..13
|
||||
doc = _make_doc_with_ents(nlp_de, "Marie von Opa", [
|
||||
(0, 5, "PER"),
|
||||
(10, 13, "PER"),
|
||||
])
|
||||
assert extract_person_names(doc) == ["Marie", "Opa"]
|
||||
|
||||
|
||||
def test_extract_person_names_empty(nlp_de):
|
||||
from extractor import extract_person_names
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe aus dem Krieg", [])
|
||||
assert extract_person_names(doc) == []
|
||||
|
||||
|
||||
def test_extract_person_names_ignores_non_per(nlp_de):
|
||||
from extractor import extract_person_names
|
||||
# DATE entity should not appear in personNames
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe 1920", [(7, 11, "DATE")])
|
||||
assert extract_person_names(doc) == []
|
||||
|
||||
|
||||
# ── Role detection ───────────────────────────────────────────────────────────
|
||||
|
||||
def test_role_sender_von(nlp_de):
|
||||
from extractor import detect_person_role
|
||||
# "Briefe von Marie" — "von" immediately before "Marie"
|
||||
# "Marie" = chars 11..16
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe von Marie", [(11, 16, "PER")])
|
||||
per_spans = list(doc.ents)
|
||||
assert detect_person_role(doc, per_spans, "de") == "sender"
|
||||
|
||||
|
||||
def test_role_receiver_an(nlp_de):
|
||||
from extractor import detect_person_role
|
||||
# "Briefe an Marie" — "an" immediately before "Marie"
|
||||
# "Marie" = chars 10..15
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe an Marie", [(10, 15, "PER")])
|
||||
per_spans = list(doc.ents)
|
||||
assert detect_person_role(doc, per_spans, "de") == "receiver"
|
||||
|
||||
|
||||
def test_role_two_persons_returns_any(nlp_de):
|
||||
from extractor import detect_person_role
|
||||
# "von Opa an Marie" — two PER spans → always "any"
|
||||
# "Opa" = chars 4..7, "Marie" = chars 11..16
|
||||
doc = _make_doc_with_ents(nlp_de, "von Opa an Marie", [
|
||||
(4, 7, "PER"),
|
||||
(11, 16, "PER"),
|
||||
])
|
||||
per_spans = list(doc.ents)
|
||||
assert detect_person_role(doc, per_spans, "de") == "any"
|
||||
|
||||
|
||||
def test_role_no_prep_returns_any(nlp_de):
|
||||
from extractor import detect_person_role
|
||||
# "Briefe Marie" — no preposition
|
||||
# "Marie" = chars 7..12
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe Marie", [(7, 12, "PER")])
|
||||
per_spans = list(doc.ents)
|
||||
assert detect_person_role(doc, per_spans, "de") == "any"
|
||||
|
||||
|
||||
def test_role_empty_returns_any(nlp_de):
|
||||
from extractor import detect_person_role
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe 1920", [])
|
||||
assert detect_person_role(doc, [], "de") == "any"
|
||||
|
||||
|
||||
def test_role_sender_from_english(nlp_en):
|
||||
from extractor import detect_person_role
|
||||
# "letters from Marie" — "from" before "Marie"
|
||||
# "Marie" = chars 13..18
|
||||
doc = _make_doc_with_ents(nlp_en, "letters from Marie", [(13, 18, "PER")])
|
||||
per_spans = list(doc.ents)
|
||||
assert detect_person_role(doc, per_spans, "en") == "sender"
|
||||
|
||||
|
||||
def test_role_receiver_to_english(nlp_en):
|
||||
from extractor import detect_person_role
|
||||
# "letters to Marie" — "to" before "Marie"
|
||||
# "letters" = 0..7, " " = 7, "to" = 8..10, " " = 10, "Marie" = 11..16
|
||||
doc = _make_doc_with_ents(nlp_en, "letters to Marie", [(11, 16, "PER")])
|
||||
per_spans = list(doc.ents)
|
||||
assert detect_person_role(doc, per_spans, "en") == "receiver"
|
||||
|
||||
|
||||
# ── Date parsing ─────────────────────────────────────────────────────────────
|
||||
|
||||
def test_date_vor_1920(nlp_de):
|
||||
from extractor import extract_dates
|
||||
# "Briefe vor 1920" — "1920" at chars 11..15
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe vor 1920", [(11, 15, "DATE")])
|
||||
date_from, date_to = extract_dates(doc, "de")
|
||||
assert date_from is None
|
||||
assert date_to == "1920-12-31"
|
||||
|
||||
|
||||
def test_date_nach_1900(nlp_de):
|
||||
from extractor import extract_dates
|
||||
# "Briefe nach 1900" — "1900" at chars 12..16
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe nach 1900", [(12, 16, "DATE")])
|
||||
date_from, date_to = extract_dates(doc, "de")
|
||||
assert date_from == "1900-01-01"
|
||||
assert date_to is None
|
||||
|
||||
|
||||
def test_date_zwischen_1900_und_1920(nlp_de):
|
||||
from extractor import extract_dates
|
||||
# "zwischen 1900 und 1920"
|
||||
# "1900" = chars 9..13, "1920" = chars 18..22
|
||||
doc = _make_doc_with_ents(nlp_de, "zwischen 1900 und 1920", [
|
||||
(9, 13, "DATE"),
|
||||
(18, 22, "DATE"),
|
||||
])
|
||||
date_from, date_to = extract_dates(doc, "de")
|
||||
assert date_from == "1900-01-01"
|
||||
assert date_to == "1920-12-31"
|
||||
|
||||
|
||||
def test_date_bare_year_makes_range(nlp_de):
|
||||
from extractor import extract_dates
|
||||
# "Briefe 1920" — no direction token → year-range
|
||||
# "1920" = chars 7..11
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe 1920", [(7, 11, "DATE")])
|
||||
date_from, date_to = extract_dates(doc, "de")
|
||||
assert date_from == "1920-01-01"
|
||||
assert date_to == "1920-12-31"
|
||||
|
||||
|
||||
def test_date_no_date_entity(nlp_de):
|
||||
from extractor import extract_dates
|
||||
doc = _make_doc_with_ents(nlp_de, "Briefe von Opa", [])
|
||||
date_from, date_to = extract_dates(doc, "de")
|
||||
assert date_from is None
|
||||
assert date_to is None
|
||||
|
||||
|
||||
def test_date_before_english(nlp_en):
|
||||
from extractor import extract_dates
|
||||
# "letters before 1920" — "1920" at chars 15..19
|
||||
doc = _make_doc_with_ents(nlp_en, "letters before 1920", [(15, 19, "DATE")])
|
||||
date_from, date_to = extract_dates(doc, "en")
|
||||
assert date_from is None
|
||||
assert date_to == "1920-12-31"
|
||||
|
||||
|
||||
def test_date_after_english(nlp_en):
|
||||
from extractor import extract_dates
|
||||
# "letters after 1900" — "1900" at chars 14..18
|
||||
doc = _make_doc_with_ents(nlp_en, "letters after 1900", [(14, 18, "DATE")])
|
||||
date_from, date_to = extract_dates(doc, "en")
|
||||
assert date_from == "1900-01-01"
|
||||
assert date_to is None
|
||||
|
||||
|
||||
# ── Keyword extraction ───────────────────────────────────────────────────────
|
||||
|
||||
def test_keywords_extracts_nouns(nlp_de):
|
||||
from extractor import extract_keywords
|
||||
# Use real NLP for POS tags; disable NER to avoid interference
|
||||
doc = nlp_de("Briefe aus dem Krieg", disable=["ner"])
|
||||
keywords = extract_keywords(doc, [])
|
||||
# "Brief" (NOUN) and "Krieg" (NOUN) should appear as lemmas
|
||||
assert "brief" in keywords
|
||||
assert "krieg" in keywords
|
||||
|
||||
|
||||
def test_keywords_excludes_stopwords(nlp_de):
|
||||
from extractor import extract_keywords
|
||||
doc = nlp_de("Briefe aus dem Krieg", disable=["ner"])
|
||||
keywords = extract_keywords(doc, [])
|
||||
# "dem" is a stopword article — must not appear
|
||||
assert "dem" not in keywords
|
||||
|
||||
|
||||
def test_keywords_excludes_per_ner_spans(nlp_de):
|
||||
from extractor import extract_keywords
|
||||
# Run full NLP for POS tags, then inject a PER span over "Hermann"
|
||||
# "Briefe von Hermann": B=0..6, ' '=6, v=7..10, ' '=10, H=11..18
|
||||
doc = nlp_de("Briefe von Hermann")
|
||||
per_span = doc.char_span(11, 18, label="PER")
|
||||
if per_span:
|
||||
doc.ents = [per_span]
|
||||
keywords = extract_keywords(doc, list(doc.ents))
|
||||
assert "hermann" not in keywords
|
||||
|
||||
|
||||
def test_keywords_excludes_short_lemmas(nlp_de):
|
||||
from extractor import extract_keywords
|
||||
doc = nlp_de("Briefe an ihn", disable=["ner"])
|
||||
keywords = extract_keywords(doc, [])
|
||||
# "ihn" is 3 chars but is a stopword pronoun; "an" is 2 chars
|
||||
assert "an" not in keywords
|
||||
|
||||
|
||||
def test_keywords_deduplicates(nlp_de):
|
||||
from extractor import extract_keywords
|
||||
doc = nlp_de("Brief Brief Krieg", disable=["ner"])
|
||||
keywords = extract_keywords(doc, [])
|
||||
assert keywords.count("brief") == 1
|
||||
|
||||
|
||||
# ── Full extract() pipeline ──────────────────────────────────────────────────
|
||||
|
||||
def test_extract_dates_de():
|
||||
from extractor import extract
|
||||
result = extract("Briefe vor 1920", "de")
|
||||
assert result.dateFrom is None
|
||||
assert result.dateTo == "1920-12-31"
|
||||
assert result.rawQuery == "Briefe vor 1920"
|
||||
assert result.personNames == []
|
||||
assert result.personRole == "any"
|
||||
|
||||
|
||||
def test_extract_keywords_from_topic_de():
|
||||
from extractor import extract
|
||||
result = extract("Briefe aus dem Krieg", "de")
|
||||
assert "krieg" in result.keywords
|
||||
assert result.dateFrom is None
|
||||
assert result.dateTo is None
|
||||
|
||||
|
||||
def test_extract_dates_en():
|
||||
from extractor import extract
|
||||
result = extract("letters before 1920", "en")
|
||||
assert result.dateTo == "1920-12-31"
|
||||
assert result.dateFrom is None
|
||||
|
||||
|
||||
def test_extract_dates_es():
|
||||
from extractor import extract
|
||||
result = extract("cartas antes de 1920", "es")
|
||||
assert result.dateTo == "1920-12-31"
|
||||
assert result.dateFrom is None
|
||||
|
||||
|
||||
def test_extract_rawquery_echoed():
|
||||
from extractor import extract
|
||||
q = "Texte über Weihnachten"
|
||||
result = extract(q, "de")
|
||||
assert result.rawQuery == q
|
||||
|
||||
|
||||
def test_extract_response_fields_are_complete():
|
||||
from extractor import extract
|
||||
result = extract("Briefe 1900", "de")
|
||||
assert isinstance(result.personNames, list)
|
||||
assert result.personRole in ("sender", "receiver", "any")
|
||||
assert isinstance(result.keywords, list)
|
||||
assert result.rawQuery == "Briefe 1900"
|
||||
from extractor import extract, extract_dates, extract_keywords, set_person_matcher
|
||||
from person_matcher import PersonMatcher
|
||||
|
||||
# ── Shared test fixture ───────────────────────────────────────────────────────
|
||||
|
||||
_TEST_PERSONS = [
|
||||
("Clara", "Cram"),
|
||||
("Herbert", "Cram"),
|
||||
("Eugenie", "de Gruyter"),
|
||||
("Walter", "de Gruyter"),
|
||||
("Marie", "Cram"),
|
||||
("Juan", "Cram"),
|
||||
("Hilde", "de Gruyter"),
|
||||
("Hans", "de Gruyter"),
|
||||
("Albert", "de Gruyter"),
|
||||
("Anita", "Wöhler"),
|
||||
("Else", "Bohrmann"),
|
||||
("Lili", "Duvenbeck"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def seeded_matcher():
|
||||
"""Load test persons into the global matcher before any test runs."""
|
||||
m = PersonMatcher()
|
||||
m.load(_TEST_PERSONS)
|
||||
set_person_matcher(m)
|
||||
return m
|
||||
|
||||
|
||||
# ── PersonMatcher unit tests ──────────────────────────────────────────────────
|
||||
|
||||
class TestPersonMatcher:
|
||||
DE_PREPS = frozenset({"von", "vom", "an", "nach", "für"})
|
||||
|
||||
def test_load_populates_names(self, seeded_matcher):
|
||||
assert len(seeded_matcher) > 0
|
||||
|
||||
def test_exact_full_name_match(self, seeded_matcher):
|
||||
hits = seeded_matcher.find_in_query("Briefe von Clara Cram", self.DE_PREPS)
|
||||
assert hits == [("Clara Cram", "von")]
|
||||
|
||||
def test_exact_first_name_only(self, seeded_matcher):
|
||||
hits = seeded_matcher.find_in_query("Briefe von Eugenie", self.DE_PREPS)
|
||||
assert hits == [("Eugenie", "von")]
|
||||
|
||||
def test_exact_first_name_receiver(self, seeded_matcher):
|
||||
hits = seeded_matcher.find_in_query("Briefe an Herbert", self.DE_PREPS)
|
||||
assert hits == [("Herbert", "an")]
|
||||
|
||||
def test_fuzzy_typo(self, seeded_matcher):
|
||||
hits = seeded_matcher.find_in_query("Briefe von Herrbert Cram", self.DE_PREPS)
|
||||
assert len(hits) == 1
|
||||
assert hits[0][1] == "von"
|
||||
|
||||
def test_two_persons_extracted(self, seeded_matcher):
|
||||
hits = seeded_matcher.find_in_query(
|
||||
"Briefe von Clara Cram an Herbert Cram", self.DE_PREPS
|
||||
)
|
||||
assert len(hits) == 2
|
||||
assert hits[0][0] == "Clara Cram"
|
||||
assert hits[0][1] == "von"
|
||||
assert hits[1][0] == "Herbert Cram"
|
||||
assert hits[1][1] == "an"
|
||||
|
||||
def test_no_match_for_place_name(self, seeded_matcher):
|
||||
hits = seeded_matcher.find_in_query("Reise nach Mexiko", self.DE_PREPS)
|
||||
assert hits == []
|
||||
|
||||
def test_no_match_for_topic_word(self, seeded_matcher):
|
||||
hits = seeded_matcher.find_in_query("Briefe aus dem Krieg", self.DE_PREPS)
|
||||
assert hits == []
|
||||
|
||||
def test_first_name_eugenie_regression(self, seeded_matcher):
|
||||
# spaCy NER missed standalone first names
|
||||
hits = seeded_matcher.find_in_query("Briefe von Eugenie", self.DE_PREPS)
|
||||
assert len(hits) == 1
|
||||
|
||||
def test_merged_names_regression(self, seeded_matcher):
|
||||
# spaCy NER merged "Herbert an Eugenie de Gruyter" into one PER span
|
||||
hits = seeded_matcher.find_in_query(
|
||||
"Briefe von Herbert an Eugenie de Gruyter nach 1914", self.DE_PREPS
|
||||
)
|
||||
assert len(hits) == 2
|
||||
names = [h[0] for h in hits]
|
||||
assert "Herbert" in names
|
||||
assert "Eugenie de Gruyter" in names
|
||||
|
||||
def test_english_preps(self, seeded_matcher):
|
||||
en_preps = frozenset({"from", "by", "to", "for"})
|
||||
hits = seeded_matcher.find_in_query(
|
||||
"Letters from Clara Cram to Walter de Gruyter in 1920", en_preps
|
||||
)
|
||||
assert len(hits) == 2
|
||||
assert hits[0][0] == "Clara Cram"
|
||||
assert hits[1][0] == "Walter de Gruyter"
|
||||
|
||||
def test_double_preposition_de(self, seeded_matcher):
|
||||
hits = seeded_matcher.find_in_query(
|
||||
"Briefe von Clara nach Herbert", self.DE_PREPS
|
||||
)
|
||||
assert len(hits) == 2
|
||||
names = [h[0] for h in hits]
|
||||
assert "Clara" in names
|
||||
assert "Herbert" in names
|
||||
|
||||
|
||||
# ── Date extraction tests ─────────────────────────────────────────────────────
|
||||
|
||||
class TestExtractDates:
|
||||
def test_bare_year_gives_range(self):
|
||||
assert extract_dates("Briefe 1920", "de") == ("1920-01-01", "1920-12-31")
|
||||
|
||||
def test_im_jahr(self):
|
||||
assert extract_dates("Schriften im Jahr 1905", "de") == (
|
||||
"1905-01-01", "1905-12-31"
|
||||
)
|
||||
|
||||
def test_vor_year(self):
|
||||
assert extract_dates("Briefe vor 1920", "de") == (None, "1920-12-31")
|
||||
|
||||
def test_nach_year(self):
|
||||
assert extract_dates("Schriften nach 1920", "de") == ("1920-01-01", None)
|
||||
|
||||
def test_zwischen(self):
|
||||
assert extract_dates("Dokumente zwischen 1914 und 1918", "de") == (
|
||||
"1914-01-01", "1918-12-31"
|
||||
)
|
||||
|
||||
def test_before_en(self):
|
||||
assert extract_dates("Letters before 1918", "en") == (None, "1918-12-31")
|
||||
|
||||
def test_after_en(self):
|
||||
assert extract_dates("Letters after 1939", "en") == ("1939-01-01", None)
|
||||
|
||||
def test_between_en(self):
|
||||
assert extract_dates("Letters between 1914 and 1918", "en") == (
|
||||
"1914-01-01", "1918-12-31"
|
||||
)
|
||||
|
||||
def test_antes_de_es(self):
|
||||
assert extract_dates("Cartas antes de 1900", "es") == (None, "1900-12-31")
|
||||
|
||||
def test_entre_es(self):
|
||||
assert extract_dates("entre 1915 y 1920", "es") == (
|
||||
"1915-01-01", "1920-12-31"
|
||||
)
|
||||
|
||||
def test_no_year(self):
|
||||
assert extract_dates("Briefe aus dem Krieg", "de") == (None, None)
|
||||
|
||||
def test_nach_before_person_then_year(self):
|
||||
# "nach Marie 1920" — "nach" belongs to person, not date
|
||||
date_from, date_to = extract_dates("Briefe nach Marie 1920", "de", ["Marie"])
|
||||
assert date_from == "1920-01-01"
|
||||
assert date_to == "1920-12-31"
|
||||
|
||||
def test_bare_year_alone(self):
|
||||
assert extract_dates("1918", "de") == ("1918-01-01", "1918-12-31")
|
||||
|
||||
|
||||
# ── Keyword extraction tests ──────────────────────────────────────────────────
|
||||
|
||||
class TestExtractKeywords:
|
||||
def test_basic_topic_words(self):
|
||||
kws = extract_keywords("Briefe aus dem Krieg", "de", [], [])
|
||||
assert "krieg" in kws
|
||||
|
||||
def test_stopwords_excluded(self):
|
||||
kws = extract_keywords("von der nach dem aus", "de", [], [])
|
||||
for sw in ("von", "der", "nach", "dem", "aus"):
|
||||
assert sw not in kws
|
||||
|
||||
def test_person_spans_excluded(self):
|
||||
kws = extract_keywords(
|
||||
"Briefe von Clara Cram nach Herbert", "de",
|
||||
["Clara Cram", "Herbert"], []
|
||||
)
|
||||
assert "clara" not in kws
|
||||
assert "cram" not in kws
|
||||
assert "herbert" not in kws
|
||||
|
||||
def test_years_excluded(self):
|
||||
kws = extract_keywords("Schriften 1920 über Reise", "de", [], ["1920"])
|
||||
assert "1920" not in kws
|
||||
|
||||
def test_deduplication(self):
|
||||
kws = extract_keywords("Krieg Krieg Krieg", "de", [], [])
|
||||
assert kws.count("krieg") == 1
|
||||
|
||||
def test_en_stopwords(self):
|
||||
kws = extract_keywords("Letters about the war", "en", [], [])
|
||||
assert "the" not in kws
|
||||
assert "war" in kws
|
||||
|
||||
def test_short_words_excluded(self):
|
||||
kws = extract_keywords("ab cd ef xy", "de", [], [])
|
||||
assert all(len(k) >= 3 for k in kws)
|
||||
|
||||
|
||||
# ── Full pipeline integration tests ──────────────────────────────────────────
|
||||
|
||||
class TestExtract:
|
||||
def test_full_sentence_de(self):
|
||||
r = extract("Briefe von Clara Cram an Walter de Gruyter im Jahr 1920", "de")
|
||||
assert "Clara Cram" in r.personNames
|
||||
assert "Walter de Gruyter" in r.personNames
|
||||
assert r.personRole == "any"
|
||||
assert r.dateFrom == "1920-01-01"
|
||||
assert r.dateTo == "1920-12-31"
|
||||
|
||||
def test_sender_role_de(self):
|
||||
r = extract("Briefe von Clara Cram vor 1910", "de")
|
||||
assert r.personNames == ["Clara Cram"]
|
||||
assert r.personRole == "sender"
|
||||
assert r.dateTo == "1910-12-31"
|
||||
assert r.dateFrom is None
|
||||
|
||||
def test_receiver_role_de(self):
|
||||
r = extract("Briefe an Walter de Gruyter", "de")
|
||||
assert r.personNames == ["Walter de Gruyter"]
|
||||
assert r.personRole == "receiver"
|
||||
|
||||
def test_first_name_only_eugenie(self):
|
||||
r = extract("Briefe von Eugenie", "de")
|
||||
assert "Eugenie" in r.personNames
|
||||
assert r.personRole == "sender"
|
||||
|
||||
def test_first_name_only_herbert(self):
|
||||
r = extract("Kriegsbriefe von Herbert", "de")
|
||||
assert "Herbert" in r.personNames
|
||||
|
||||
def test_merged_names_bug_fixed(self):
|
||||
r = extract("Briefe von Herbert an Eugenie de Gruyter nach 1914", "de")
|
||||
assert "Herbert" in r.personNames
|
||||
assert "Eugenie de Gruyter" in r.personNames
|
||||
assert r.dateFrom == "1914-01-01"
|
||||
|
||||
def test_topic_only_krieg(self):
|
||||
r = extract("Briefe aus dem Krieg", "de")
|
||||
assert r.personNames == []
|
||||
assert "krieg" in r.keywords
|
||||
|
||||
def test_topic_only_single_word(self):
|
||||
r = extract("Kriegspost", "de")
|
||||
assert r.personNames == []
|
||||
|
||||
def test_date_range_only(self):
|
||||
r = extract("Dokumente zwischen 1914 und 1918", "de")
|
||||
assert r.personNames == []
|
||||
assert r.dateFrom == "1914-01-01"
|
||||
assert r.dateTo == "1918-12-31"
|
||||
|
||||
def test_colloquial_von(self):
|
||||
r = extract("von Clara", "de")
|
||||
assert r.personNames == ["Clara"]
|
||||
assert r.personRole == "sender"
|
||||
|
||||
def test_colloquial_an(self):
|
||||
r = extract("an Walter", "de")
|
||||
assert r.personNames == ["Walter"]
|
||||
assert r.personRole == "receiver"
|
||||
|
||||
def test_bare_year_alone(self):
|
||||
r = extract("1918", "de")
|
||||
assert r.dateFrom == "1918-01-01"
|
||||
assert r.dateTo == "1918-12-31"
|
||||
assert r.personNames == []
|
||||
|
||||
def test_english_full_sentence(self):
|
||||
r = extract("Letters from Clara Cram to Walter de Gruyter in 1920", "en")
|
||||
assert "Clara Cram" in r.personNames
|
||||
assert "Walter de Gruyter" in r.personNames
|
||||
assert r.dateFrom == "1920-01-01"
|
||||
|
||||
def test_english_receiver_with_date(self):
|
||||
r = extract("Letters to Herbert Cram after 1939", "en")
|
||||
assert "Herbert Cram" in r.personNames
|
||||
assert r.personRole == "receiver"
|
||||
assert r.dateFrom == "1939-01-01"
|
||||
|
||||
def test_english_birthday(self):
|
||||
r = extract("Birthday greetings from Anita Wöhler", "en")
|
||||
assert "Anita Wöhler" in r.personNames
|
||||
assert r.personRole == "sender"
|
||||
|
||||
def test_english_between_dates(self):
|
||||
r = extract("Letters between 1914 and 1918", "en")
|
||||
assert r.dateFrom == "1914-01-01"
|
||||
assert r.dateTo == "1918-12-31"
|
||||
|
||||
def test_spanish_full_sentence(self):
|
||||
r = extract("Cartas de Clara Cram a Walter de Gruyter en 1920", "es")
|
||||
assert "Clara Cram" in r.personNames
|
||||
assert "Walter de Gruyter" in r.personNames
|
||||
assert r.dateFrom == "1920-01-01"
|
||||
|
||||
def test_spanish_before(self):
|
||||
r = extract("Cartas antes de 1900", "es")
|
||||
assert r.dateTo == "1900-12-31"
|
||||
assert r.dateFrom is None
|
||||
|
||||
def test_rawquery_echoed(self):
|
||||
q = "test query"
|
||||
r = extract(q, "de")
|
||||
assert r.rawQuery == q
|
||||
|
||||
def test_false_positive_compound_noun_regression(self):
|
||||
# spaCy tagged "Geburtstagsglückwünsche" as a PER entity
|
||||
r = extract("Geburtstagsglückwünsche", "de")
|
||||
assert r.personNames == []
|
||||
|
||||
def test_question_phrasing(self):
|
||||
r = extract("Wer hat an Herbert Cram 1918 geschrieben?", "de")
|
||||
assert "Herbert Cram" in r.personNames
|
||||
assert r.personRole == "receiver"
|
||||
assert r.dateFrom == "1918-01-01"
|
||||
|
||||
def test_lowercase_query(self):
|
||||
r = extract("briefe von clara cram an herbert 1920", "de")
|
||||
# Should still find persons despite lowercase
|
||||
assert len(r.personNames) >= 1
|
||||
|
||||
def test_empty_matcher_returns_no_persons(self):
|
||||
# Temporarily use an empty matcher
|
||||
from extractor import set_person_matcher
|
||||
empty = PersonMatcher()
|
||||
set_person_matcher(empty)
|
||||
r = extract("Briefe von Clara Cram", "de")
|
||||
assert r.personNames == []
|
||||
# Restore seeded matcher
|
||||
m = PersonMatcher()
|
||||
m.load(_TEST_PERSONS)
|
||||
set_person_matcher(m)
|
||||
|
||||
Reference in New Issue
Block a user