marcel/familienarchiv
1
0
Fork 0
Code Issues 114 Pull Requests 3 Actions 2 Packages Projects Releases Wiki Activity

feat(ocr): per-sender specialized Kurrent models with automatic active-learning retraining #263

Merged
marcel merged 32 commits from feat/issue-253-sender-models into main 2026-04-18 12:30:57 +02:00
Conversation 41 Commits 32 Files Changed 42 +42 -17
3 changed files with 42 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit bd23a76330 - Show all commits

1
backend/src/test/java/org/raddatz/familienarchiv/controller/OcrControllerTest.java
View File

@@ -43,6 +43,7 @@ class OcrControllerTest {
@MockitoBean OcrBatchService ocrBatchService;
@MockitoBean OcrProgressService ocrProgressService;
@MockitoBean UserService userService;
@MockitoBean PersonService personService;
@MockitoBean CustomUserDetailsService customUserDetailsService;
@MockitoBean TrainingDataExportService trainingDataExportService;
@MockitoBean SegmentationTrainingExportService segmentationTrainingExportService;

52
backend/src/test/java/org/raddatz/familienarchiv/service/OcrAsyncRunnerTest.java
View File

@@ -31,6 +31,7 @@ class OcrAsyncRunnerTest {
@Mock OcrJobRepository ocrJobRepository;
@Mock OcrJobDocumentRepository ocrJobDocumentRepository;
@Mock OcrProgressService ocrProgressService;
@Mock SenderModelService senderModelService;
@InjectMocks OcrAsyncRunner ocrAsyncRunner;
@@ -42,7 +43,12 @@ class OcrAsyncRunnerTest {
.fileHash("hash").scriptType(ScriptType.TYPEWRITER).build();
when(fileService.generatePresignedUrl(any())).thenReturn("http://presigned");
when(ocrClient.extractBlocks(any(), any())).thenReturn(List.of());
doAnswer(inv -> {
Consumer<OcrStreamEvent> handler = inv.getArgument(4);
handler.accept(new OcrStreamEvent.Start(0));
handler.accept(new OcrStreamEvent.Done(0, 0));
return null;
}).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
ocrAsyncRunner.processDocument(docId, doc, userId);
@@ -59,9 +65,15 @@ class OcrAsyncRunnerTest {
when(fileService.generatePresignedUrl(any())).thenReturn("http://presigned");
when(ocrClient.extractBlocks(any(), any())).thenReturn(List.of(
new OcrBlockResult(0, 0.1, 0.1, 0.8, 0.04, null, "Line 1", null),
new OcrBlockResult(0, 0.1, 0.2, 0.8, 0.04, null, "Line 2", null)));
doAnswer(inv -> {
Consumer<OcrStreamEvent> handler = inv.getArgument(4);
handler.accept(new OcrStreamEvent.Start(1));
handler.accept(new OcrStreamEvent.Page(0, List.of(
new OcrBlockResult(0, 0.1, 0.1, 0.8, 0.04, null, "Line 1", null),
new OcrBlockResult(0, 0.1, 0.2, 0.8, 0.04, null, "Line 2", null))));
handler.accept(new OcrStreamEvent.Done(2, 0));
return null;
}).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
DocumentAnnotation ann = DocumentAnnotation.builder().id(annId).build();
when(annotationService.createOcrAnnotation(any(), any(), any(), any(), any())).thenReturn(ann);
@@ -83,8 +95,14 @@ class OcrAsyncRunnerTest {
when(fileService.generatePresignedUrl(any())).thenReturn("http://presigned");
when(ocrClient.extractBlocks(any(), any())).thenReturn(List.of(
new OcrBlockResult(0, 0.1, 0.1, 0.8, 0.04, null, "Test", null)));
doAnswer(inv -> {
Consumer<OcrStreamEvent> handler = inv.getArgument(4);
handler.accept(new OcrStreamEvent.Start(1));
handler.accept(new OcrStreamEvent.Page(0, List.of(
new OcrBlockResult(0, 0.1, 0.1, 0.8, 0.04, null, "Test", null))));
handler.accept(new OcrStreamEvent.Done(1, 0));
return null;
}).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
DocumentAnnotation ann = DocumentAnnotation.builder().id(annId).build();
when(annotationService.createOcrAnnotation(any(), any(), any(), any(), any())).thenReturn(ann);
@@ -112,12 +130,12 @@ class OcrAsyncRunnerTest {
when(fileService.generatePresignedUrl(any())).thenReturn("http://presigned");
doAnswer(inv -> {
Consumer<OcrStreamEvent> handler = inv.getArgument(3);
Consumer<OcrStreamEvent> handler = inv.getArgument(4);
handler.accept(new OcrStreamEvent.Start(1));
handler.accept(new OcrStreamEvent.Page(0, List.of()));
handler.accept(new OcrStreamEvent.Done(0, 0));
return null;
}).when(ocrClient).streamBlocks(any(), any(), any(), any());
}).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
ocrAsyncRunner.runSingleDocument(jobId, docId, userId);
@@ -142,7 +160,7 @@ class OcrAsyncRunnerTest {
when(documentService.getDocumentById(docId)).thenReturn(doc);
when(fileService.generatePresignedUrl(any())).thenReturn("http://presigned");
doThrow(new RuntimeException("OCR failed")).when(ocrClient).streamBlocks(any(), any(), any(), any());
doThrow(new RuntimeException("OCR failed")).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
ocrAsyncRunner.runSingleDocument(jobId, docId, userId);
@@ -174,7 +192,7 @@ class OcrAsyncRunnerTest {
List<String> progressMessages = new ArrayList<>();
doAnswer(inv -> {
Consumer<OcrStreamEvent> handler = inv.getArgument(3);
Consumer<OcrStreamEvent> handler = inv.getArgument(4);
handler.accept(new OcrStreamEvent.Start(3));
handler.accept(new OcrStreamEvent.Page(0, List.of(
new OcrBlockResult(0, 0.1, 0.1, 0.8, 0.04, null, "L1", null),
@@ -185,7 +203,7 @@ class OcrAsyncRunnerTest {
progressMessages.add(job.getProgressMessage());
handler.accept(new OcrStreamEvent.Done(3, 0));
return null;
}).when(ocrClient).streamBlocks(any(), any(), any(), any());
}).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
ocrAsyncRunner.runSingleDocument(jobId, docId, userId);
@@ -215,14 +233,14 @@ class OcrAsyncRunnerTest {
when(fileService.generatePresignedUrl(any())).thenReturn("http://presigned");
doAnswer(inv -> {
Consumer<OcrStreamEvent> handler = inv.getArgument(3);
Consumer<OcrStreamEvent> handler = inv.getArgument(4);
handler.accept(new OcrStreamEvent.Start(3));
handler.accept(new OcrStreamEvent.Page(0, List.of()));
handler.accept(new OcrStreamEvent.Error(1, "failed"));
handler.accept(new OcrStreamEvent.Page(2, List.of()));
handler.accept(new OcrStreamEvent.Done(0, 1));
return null;
}).when(ocrClient).streamBlocks(any(), any(), any(), any());
}).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
ocrAsyncRunner.runSingleDocument(jobId, docId, userId);
@@ -251,14 +269,14 @@ class OcrAsyncRunnerTest {
List<String> progressMessages = new ArrayList<>();
doAnswer(inv -> {
Consumer<OcrStreamEvent> handler = inv.getArgument(3);
Consumer<OcrStreamEvent> handler = inv.getArgument(4);
handler.accept(new OcrStreamEvent.Start(5));
handler.accept(new OcrStreamEvent.Preprocessing(1));
progressMessages.add(job.getProgressMessage());
handler.accept(new OcrStreamEvent.Page(1, List.of()));
handler.accept(new OcrStreamEvent.Done(0, 0));
return null;
}).when(ocrClient).streamBlocks(any(), any(), any(), any());
}).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
ocrAsyncRunner.runSingleDocument(jobId, docId, userId);
@@ -287,13 +305,13 @@ class OcrAsyncRunnerTest {
when(fileService.generatePresignedUrl(any())).thenReturn("http://presigned");
doAnswer(inv -> {
Consumer<OcrStreamEvent> handler = inv.getArgument(3);
Consumer<OcrStreamEvent> handler = inv.getArgument(4);
handler.accept(new OcrStreamEvent.Start(2));
handler.accept(new OcrStreamEvent.Error(0, "some python traceback details"));
handler.accept(new OcrStreamEvent.Page(1, List.of()));
handler.accept(new OcrStreamEvent.Done(0, 1));
return null;
}).when(ocrClient).streamBlocks(any(), any(), any(), any());
}).when(ocrClient).streamBlocks(any(), any(), any(), any(), any());
ocrAsyncRunner.runSingleDocument(jobId, docId, userId);

6
backend/src/test/java/org/raddatz/familienarchiv/service/TranscriptionServiceTest.java
View File

@@ -13,6 +13,7 @@ import org.raddatz.familienarchiv.exception.DomainException;
import org.raddatz.familienarchiv.model.BlockSource;
import org.raddatz.familienarchiv.model.Document;
import org.raddatz.familienarchiv.model.DocumentAnnotation;
import org.raddatz.familienarchiv.model.ScriptType;
import org.raddatz.familienarchiv.model.TranscriptionBlock;
import org.raddatz.familienarchiv.model.TranscriptionBlockVersion;
import org.raddatz.familienarchiv.repository.AnnotationRepository;
@@ -39,6 +40,7 @@ class TranscriptionServiceTest {
@Mock AnnotationRepository annotationRepository;
@Mock AnnotationService annotationService;
@Mock DocumentService documentService;
@Mock SenderModelService senderModelService;
@InjectMocks TranscriptionService transcriptionService;
// ─── getBlock ────────────────────────────────────────────────────────────────
@@ -156,6 +158,8 @@ class TranscriptionServiceTest {
.id(blockId).documentId(docId).text("old").build();
when(blockRepository.findByIdAndDocumentId(blockId, docId)).thenReturn(Optional.of(block));
when(blockRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
when(documentService.getDocumentById(any())).thenReturn(
Document.builder().scriptType(ScriptType.TYPEWRITER).build());
UpdateTranscriptionBlockDTO dto = new UpdateTranscriptionBlockDTO("new text", null);
@@ -175,6 +179,8 @@ class TranscriptionServiceTest {
.id(blockId).documentId(docId).text("text").label("old label").build();
when(blockRepository.findByIdAndDocumentId(blockId, docId)).thenReturn(Optional.of(block));
when(blockRepository.save(any())).thenAnswer(inv -> inv.getArgument(0));
when(documentService.getDocumentById(any())).thenReturn(
Document.builder().scriptType(ScriptType.TYPEWRITER).build());
UpdateTranscriptionBlockDTO dto = new UpdateTranscriptionBlockDTO("text", "Anrede");