Compare commits
58 Commits
feat/issue
...
80d77a53e9
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
80d77a53e9 | ||
|
|
a45652466e | ||
|
|
49a17b581b | ||
|
|
53c8d6e9f0 | ||
|
|
279b4f1098 | ||
|
|
15114c2d92 | ||
|
|
35017d91c4 | ||
|
|
5b367a53a1 | ||
|
|
cb91ed340d | ||
|
|
2e0eb40aec | ||
|
|
d9e01ef1ff | ||
|
|
2e0f85c360 | ||
|
|
a1035171c2 | ||
|
|
8e9e3bba06 | ||
|
|
627fc44d99 | ||
|
|
6583226d79 | ||
|
|
41b205becc | ||
|
|
f22dcaecb7 | ||
|
|
1109ab917b | ||
|
|
769984608b | ||
|
|
c282f38170 | ||
|
|
3ea7f0b5b2 | ||
|
|
bcba4dab80 | ||
|
|
a4a3e3b105 | ||
|
|
cac00ed711 | ||
|
|
637829cebc | ||
|
|
4e636b3253 | ||
|
|
ab2708e63b | ||
|
|
ed8e9576e4 | ||
|
|
0958df7768 | ||
|
|
f4ffd8acee | ||
|
|
0801da8df0 | ||
|
|
e0e1578bdd | ||
|
|
2df71beb7e | ||
|
|
2dbb3c37b4 | ||
|
|
67368b4413 | ||
|
|
ddf6cf4cbc | ||
|
|
df952861c4 | ||
|
|
22a5ee816a | ||
|
|
0179e93a4b | ||
|
|
0fc0cbcffd | ||
|
|
549cb15845 | ||
|
|
74ddf16b01 | ||
|
|
ebaedb1af0 | ||
|
|
e75ac8ec45 | ||
|
|
525f091b3a | ||
|
|
d6abf990c7 | ||
|
|
77d59c5d83 | ||
|
|
6c2b9af10b | ||
|
|
2e3744d9ef | ||
|
|
131ed336bc | ||
|
|
3fa3460dbf | ||
|
|
79edb94558 | ||
|
|
52d8dc2b20 | ||
|
|
696b71da5a | ||
|
|
f3e3545d06 | ||
|
|
4bb6685edb | ||
|
|
18c93d4eaa |
@@ -39,6 +39,12 @@ PORT_PROMETHEUS=9090
|
|||||||
# Grafana admin password — change this before exposing Grafana beyond localhost
|
# Grafana admin password — change this before exposing Grafana beyond localhost
|
||||||
GRAFANA_ADMIN_PASSWORD=changeme
|
GRAFANA_ADMIN_PASSWORD=changeme
|
||||||
|
|
||||||
|
# Password for the read-only grafana_reader PostgreSQL role used by the PO
|
||||||
|
# Overview dashboard. Consumed by Flyway V68 (to set the role's password) and
|
||||||
|
# by Grafana's PostgreSQL datasource (to connect). REQUIRED in production —
|
||||||
|
# generate with: openssl rand -hex 32
|
||||||
|
GRAFANA_DB_PASSWORD=changeme-generate-with-openssl-rand-hex-32
|
||||||
|
|
||||||
# GlitchTip domain — production: use https://glitchtip.archiv.raddatz.cloud (must match Caddy vhost)
|
# GlitchTip domain — production: use https://glitchtip.archiv.raddatz.cloud (must match Caddy vhost)
|
||||||
GLITCHTIP_DOMAIN=http://localhost:3002
|
GLITCHTIP_DOMAIN=http://localhost:3002
|
||||||
|
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ name: nightly
|
|||||||
# STAGING_APP_ADMIN_USERNAME
|
# STAGING_APP_ADMIN_USERNAME
|
||||||
# STAGING_APP_ADMIN_PASSWORD
|
# STAGING_APP_ADMIN_PASSWORD
|
||||||
# GRAFANA_ADMIN_PASSWORD
|
# GRAFANA_ADMIN_PASSWORD
|
||||||
|
# GRAFANA_DB_PASSWORD (read-only grafana_reader DB role, issue #651)
|
||||||
# GLITCHTIP_SECRET_KEY
|
# GLITCHTIP_SECRET_KEY
|
||||||
# SENTRY_DSN (set after GlitchTip first-run; empty = Sentry disabled)
|
# SENTRY_DSN (set after GlitchTip first-run; empty = Sentry disabled)
|
||||||
|
|
||||||
@@ -80,6 +81,7 @@ jobs:
|
|||||||
POSTGRES_USER=archiv
|
POSTGRES_USER=archiv
|
||||||
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
|
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
|
||||||
VITE_SENTRY_DSN=${{ secrets.VITE_SENTRY_DSN }}
|
VITE_SENTRY_DSN=${{ secrets.VITE_SENTRY_DSN }}
|
||||||
|
GRAFANA_DB_PASSWORD=${{ secrets.GRAFANA_DB_PASSWORD }}
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
- name: Verify backend /import:ro mount is wired
|
- name: Verify backend /import:ro mount is wired
|
||||||
@@ -143,6 +145,7 @@ jobs:
|
|||||||
cp docker-compose.observability.yml /opt/familienarchiv/
|
cp docker-compose.observability.yml /opt/familienarchiv/
|
||||||
cat > /opt/familienarchiv/obs-secrets.env <<'EOF'
|
cat > /opt/familienarchiv/obs-secrets.env <<'EOF'
|
||||||
GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
||||||
|
GRAFANA_DB_PASSWORD=${{ secrets.GRAFANA_DB_PASSWORD }}
|
||||||
GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
|
GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
|
||||||
POSTGRES_PASSWORD=${{ secrets.STAGING_POSTGRES_PASSWORD }}
|
POSTGRES_PASSWORD=${{ secrets.STAGING_POSTGRES_PASSWORD }}
|
||||||
POSTGRES_HOST=archiv-staging-db-1
|
POSTGRES_HOST=archiv-staging-db-1
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ name: release
|
|||||||
# MAIL_USERNAME
|
# MAIL_USERNAME
|
||||||
# MAIL_PASSWORD
|
# MAIL_PASSWORD
|
||||||
# GRAFANA_ADMIN_PASSWORD
|
# GRAFANA_ADMIN_PASSWORD
|
||||||
|
# GRAFANA_DB_PASSWORD (read-only grafana_reader DB role, issue #651)
|
||||||
# GLITCHTIP_SECRET_KEY
|
# GLITCHTIP_SECRET_KEY
|
||||||
# SENTRY_DSN (set after GlitchTip first-run; empty = Sentry disabled)
|
# SENTRY_DSN (set after GlitchTip first-run; empty = Sentry disabled)
|
||||||
|
|
||||||
@@ -77,6 +78,7 @@ jobs:
|
|||||||
IMPORT_HOST_DIR=/srv/familienarchiv-production/import
|
IMPORT_HOST_DIR=/srv/familienarchiv-production/import
|
||||||
POSTGRES_USER=archiv
|
POSTGRES_USER=archiv
|
||||||
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
|
SENTRY_DSN=${{ secrets.SENTRY_DSN }}
|
||||||
|
GRAFANA_DB_PASSWORD=${{ secrets.GRAFANA_DB_PASSWORD }}
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
- name: Build images
|
- name: Build images
|
||||||
@@ -110,6 +112,7 @@ jobs:
|
|||||||
cp docker-compose.observability.yml /opt/familienarchiv/
|
cp docker-compose.observability.yml /opt/familienarchiv/
|
||||||
cat > /opt/familienarchiv/obs-secrets.env <<'EOF'
|
cat > /opt/familienarchiv/obs-secrets.env <<'EOF'
|
||||||
GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
GRAFANA_ADMIN_PASSWORD=${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
||||||
|
GRAFANA_DB_PASSWORD=${{ secrets.GRAFANA_DB_PASSWORD }}
|
||||||
GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
|
GLITCHTIP_SECRET_KEY=${{ secrets.GLITCHTIP_SECRET_KEY }}
|
||||||
POSTGRES_PASSWORD=${{ secrets.PROD_POSTGRES_PASSWORD }}
|
POSTGRES_PASSWORD=${{ secrets.PROD_POSTGRES_PASSWORD }}
|
||||||
POSTGRES_HOST=archiv-production-db-1
|
POSTGRES_HOST=archiv-production-db-1
|
||||||
|
|||||||
@@ -197,6 +197,7 @@ frontend/src/routes/
|
|||||||
├── aktivitaeten/ Unified activity feed (Chronik)
|
├── aktivitaeten/ Unified activity feed (Chronik)
|
||||||
├── geschichten/ Stories — list, [id], [id]/edit, new
|
├── geschichten/ Stories — list, [id], [id]/edit, new
|
||||||
├── stammbaum/ Family tree (Stammbaum)
|
├── stammbaum/ Family tree (Stammbaum)
|
||||||
|
├── themen/ Topics directory — browsable tag index
|
||||||
├── enrich/ Enrichment workflow — [id], done
|
├── enrich/ Enrichment workflow — [id], done
|
||||||
├── admin/ User, group, tag, OCR, system management
|
├── admin/ User, group, tag, OCR, system management
|
||||||
├── hilfe/transkription/ Transcription help page
|
├── hilfe/transkription/ Transcription help page
|
||||||
|
|||||||
@@ -5,8 +5,10 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.flywaydb.core.Flyway;
|
import org.flywaydb.core.Flyway;
|
||||||
import org.springframework.context.annotation.Bean;
|
import org.springframework.context.annotation.Bean;
|
||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
import org.springframework.core.env.Environment;
|
||||||
|
|
||||||
import javax.sql.DataSource;
|
import javax.sql.DataSource;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
@Configuration
|
@Configuration
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@@ -14,6 +16,7 @@ import javax.sql.DataSource;
|
|||||||
public class FlywayConfig {
|
public class FlywayConfig {
|
||||||
|
|
||||||
private final DataSource dataSource;
|
private final DataSource dataSource;
|
||||||
|
private final Environment environment;
|
||||||
|
|
||||||
@Bean(name = "flyway")
|
@Bean(name = "flyway")
|
||||||
public Flyway flyway() {
|
public Flyway flyway() {
|
||||||
@@ -21,6 +24,7 @@ public class FlywayConfig {
|
|||||||
Flyway flyway = Flyway.configure()
|
Flyway flyway = Flyway.configure()
|
||||||
.dataSource(dataSource)
|
.dataSource(dataSource)
|
||||||
.locations("classpath:db/migration")
|
.locations("classpath:db/migration")
|
||||||
|
.placeholders(Map.of("grafanaDbPassword", resolveGrafanaDbPassword()))
|
||||||
.baselineOnMigrate(true)
|
.baselineOnMigrate(true)
|
||||||
.baselineVersion("4")
|
.baselineVersion("4")
|
||||||
.load();
|
.load();
|
||||||
@@ -28,4 +32,22 @@ public class FlywayConfig {
|
|||||||
log.info("Flyway: {} migration(s) applied.", result.migrationsExecuted);
|
log.info("Flyway: {} migration(s) applied.", result.migrationsExecuted);
|
||||||
return flyway;
|
return flyway;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fail-closed: refuse to boot when GRAFANA_DB_PASSWORD is unset. The
|
||||||
|
// grafana_reader role's password is (re)set on every boot by
|
||||||
|
// R__grafana_reader_password.sql, so a missing env var means we'd either
|
||||||
|
// skip the rotation silently or — with a hardcoded fallback — publish a
|
||||||
|
// well-known credential for a role with SELECT on audit_log, documents,
|
||||||
|
// and transcription_blocks. Same shape as UserDataInitializer's refusal
|
||||||
|
// to seed default admin credentials outside dev/test/e2e.
|
||||||
|
String resolveGrafanaDbPassword() {
|
||||||
|
String value = environment.getProperty("GRAFANA_DB_PASSWORD");
|
||||||
|
if (value == null || value.isBlank()) {
|
||||||
|
throw new IllegalStateException(
|
||||||
|
"GRAFANA_DB_PASSWORD is required: it is consumed by "
|
||||||
|
+ "R__grafana_reader_password.sql to (re)set the grafana_reader "
|
||||||
|
+ "role's password on every boot. Generate with: openssl rand -hex 32");
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,8 +31,7 @@ import java.util.UUID;
|
|||||||
@NamedEntityGraph(name = "Document.list", attributeNodes = {
|
@NamedEntityGraph(name = "Document.list", attributeNodes = {
|
||||||
@NamedAttributeNode("sender"),
|
@NamedAttributeNode("sender"),
|
||||||
@NamedAttributeNode("receivers"),
|
@NamedAttributeNode("receivers"),
|
||||||
@NamedAttributeNode("tags"),
|
@NamedAttributeNode("tags")
|
||||||
@NamedAttributeNode("trainingLabels")
|
|
||||||
})
|
})
|
||||||
@Entity
|
@Entity
|
||||||
@Table(name = "documents")
|
@Table(name = "documents")
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package org.raddatz.familienarchiv.document;
|
||||||
|
|
||||||
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
|
import org.raddatz.familienarchiv.audit.ActivityActorDTO;
|
||||||
|
import org.raddatz.familienarchiv.person.Person;
|
||||||
|
import org.raddatz.familienarchiv.tag.Tag;
|
||||||
|
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
public record DocumentListItem(
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
UUID id,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
String title,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
String originalFilename,
|
||||||
|
String thumbnailUrl,
|
||||||
|
LocalDate documentDate,
|
||||||
|
Person sender,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
List<Person> receivers,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
List<Tag> tags,
|
||||||
|
String archiveBox,
|
||||||
|
String archiveFolder,
|
||||||
|
String location,
|
||||||
|
String summary,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
int completionPercentage,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
List<ActivityActorDTO> contributors,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
SearchMatchData matchData,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
LocalDateTime createdAt,
|
||||||
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
|
LocalDateTime updatedAt
|
||||||
|
) {}
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
package org.raddatz.familienarchiv.document;
|
|
||||||
|
|
||||||
import io.swagger.v3.oas.annotations.media.Schema;
|
|
||||||
import org.raddatz.familienarchiv.audit.ActivityActorDTO;
|
|
||||||
import org.raddatz.familienarchiv.document.Document;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public record DocumentSearchItem(
|
|
||||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
|
||||||
Document document,
|
|
||||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
|
||||||
SearchMatchData matchData,
|
|
||||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
|
||||||
int completionPercentage,
|
|
||||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
|
||||||
List<ActivityActorDTO> contributors
|
|
||||||
) {}
|
|
||||||
@@ -7,7 +7,7 @@ import java.util.List;
|
|||||||
|
|
||||||
public record DocumentSearchResult(
|
public record DocumentSearchResult(
|
||||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
List<DocumentSearchItem> items,
|
List<DocumentListItem> items,
|
||||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
long totalElements,
|
long totalElements,
|
||||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
@Schema(requiredMode = Schema.RequiredMode.REQUIRED)
|
||||||
@@ -21,16 +21,16 @@ public record DocumentSearchResult(
|
|||||||
* Single-page convenience factory used by empty-result shortcuts and by tests that
|
* Single-page convenience factory used by empty-result shortcuts and by tests that
|
||||||
* don't care about paging. Treats the whole list as page 0 of itself.
|
* don't care about paging. Treats the whole list as page 0 of itself.
|
||||||
*/
|
*/
|
||||||
public static DocumentSearchResult of(List<DocumentSearchItem> items) {
|
public static DocumentSearchResult of(List<DocumentListItem> items) {
|
||||||
int size = items.size();
|
int size = items.size();
|
||||||
return new DocumentSearchResult(items, size, 0, size, size == 0 ? 0 : 1);
|
return new DocumentSearchResult(items, size, 0, size, size == 0 ? 0 : 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Paged factory used by the service when it has a real Pageable + full match count
|
* Paged factory used by the service when it has a real Pageable + full match count
|
||||||
* (e.g. from Spring's Page<T> or from an in-memory sort-then-slice).
|
* (e.g. from Spring's Page<T> or from an in-memory sort-then-slice).
|
||||||
*/
|
*/
|
||||||
public static DocumentSearchResult paged(List<DocumentSearchItem> slice, Pageable pageable, long totalElements) {
|
public static DocumentSearchResult paged(List<DocumentListItem> slice, Pageable pageable, long totalElements) {
|
||||||
int pageSize = pageable.getPageSize();
|
int pageSize = pageable.getPageSize();
|
||||||
int totalPages = pageSize == 0 ? 0 : (int) ((totalElements + pageSize - 1) / pageSize);
|
int totalPages = pageSize == 0 ? 0 : (int) ((totalElements + pageSize - 1) / pageSize);
|
||||||
return new DocumentSearchResult(slice, totalElements, pageable.getPageNumber(), pageSize, totalPages);
|
return new DocumentSearchResult(slice, totalElements, pageable.getPageNumber(), pageSize, totalPages);
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import org.raddatz.familienarchiv.audit.AuditService;
|
|||||||
import org.raddatz.familienarchiv.document.DocumentBatchMetadataDTO;
|
import org.raddatz.familienarchiv.document.DocumentBatchMetadataDTO;
|
||||||
import org.raddatz.familienarchiv.document.DocumentBatchSummary;
|
import org.raddatz.familienarchiv.document.DocumentBatchSummary;
|
||||||
import org.raddatz.familienarchiv.document.DocumentBulkEditDTO;
|
import org.raddatz.familienarchiv.document.DocumentBulkEditDTO;
|
||||||
import org.raddatz.familienarchiv.document.DocumentSearchItem;
|
|
||||||
import org.raddatz.familienarchiv.document.DocumentSearchResult;
|
import org.raddatz.familienarchiv.document.DocumentSearchResult;
|
||||||
import org.raddatz.familienarchiv.document.DocumentSort;
|
import org.raddatz.familienarchiv.document.DocumentSort;
|
||||||
import org.raddatz.familienarchiv.document.DocumentUpdateDTO;
|
import org.raddatz.familienarchiv.document.DocumentUpdateDTO;
|
||||||
@@ -736,7 +735,7 @@ public class DocumentService {
|
|||||||
return DocumentSearchResult.paged(enrichItems(slice, text), pageable, totalElements);
|
return DocumentSearchResult.paged(enrichItems(slice, text), pageable, totalElements);
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<DocumentSearchItem> enrichItems(List<Document> documents, String text) {
|
private List<DocumentListItem> enrichItems(List<Document> documents, String text) {
|
||||||
List<Document> colorResolved = resolveDocumentTagColors(documents);
|
List<Document> colorResolved = resolveDocumentTagColors(documents);
|
||||||
Map<UUID, SearchMatchData> matchData = enrichWithMatchData(colorResolved, text);
|
Map<UUID, SearchMatchData> matchData = enrichWithMatchData(colorResolved, text);
|
||||||
|
|
||||||
@@ -744,7 +743,7 @@ public class DocumentService {
|
|||||||
Map<UUID, Integer> completionByDoc = fetchCompletionPercentages(docIds);
|
Map<UUID, Integer> completionByDoc = fetchCompletionPercentages(docIds);
|
||||||
Map<UUID, List<ActivityActorDTO>> contributorsByDoc = auditLogQueryService.findRecentContributorsPerDocument(docIds);
|
Map<UUID, List<ActivityActorDTO>> contributorsByDoc = auditLogQueryService.findRecentContributorsPerDocument(docIds);
|
||||||
|
|
||||||
return colorResolved.stream().map(doc -> new DocumentSearchItem(
|
return colorResolved.stream().map(doc -> toListItem(
|
||||||
doc,
|
doc,
|
||||||
matchData.getOrDefault(doc.getId(), SearchMatchData.empty()),
|
matchData.getOrDefault(doc.getId(), SearchMatchData.empty()),
|
||||||
completionByDoc.getOrDefault(doc.getId(), 0),
|
completionByDoc.getOrDefault(doc.getId(), 0),
|
||||||
@@ -752,6 +751,28 @@ public class DocumentService {
|
|||||||
)).toList();
|
)).toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private DocumentListItem toListItem(Document doc, SearchMatchData match, int completionPct, List<ActivityActorDTO> contributors) {
|
||||||
|
return new DocumentListItem(
|
||||||
|
doc.getId(),
|
||||||
|
doc.getTitle(),
|
||||||
|
doc.getOriginalFilename(),
|
||||||
|
doc.getThumbnailUrl(),
|
||||||
|
doc.getDocumentDate(),
|
||||||
|
doc.getSender(),
|
||||||
|
List.copyOf(doc.getReceivers()),
|
||||||
|
List.copyOf(doc.getTags()),
|
||||||
|
doc.getArchiveBox(),
|
||||||
|
doc.getArchiveFolder(),
|
||||||
|
doc.getLocation(),
|
||||||
|
doc.getSummary(),
|
||||||
|
completionPct,
|
||||||
|
contributors,
|
||||||
|
match,
|
||||||
|
doc.getCreatedAt(),
|
||||||
|
doc.getUpdatedAt()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
private Map<UUID, Integer> fetchCompletionPercentages(List<UUID> docIds) {
|
private Map<UUID, Integer> fetchCompletionPercentages(List<UUID> docIds) {
|
||||||
return transcriptionBlockQueryService.getCompletionStats(docIds);
|
return transcriptionBlockQueryService.getCompletionStats(docIds);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
-- Repeatable migration: sets the grafana_reader role's password from the
|
||||||
|
-- ${grafanaDbPassword} placeholder (resolved by FlywayConfig from the
|
||||||
|
-- GRAFANA_DB_PASSWORD environment variable). Flyway computes the checksum on
|
||||||
|
-- the resolved migration content, so any change to GRAFANA_DB_PASSWORD changes
|
||||||
|
-- the checksum and re-applies this migration on the next boot. That makes
|
||||||
|
-- password rotation a "change env var + restart" operation — no manual psql.
|
||||||
|
--
|
||||||
|
-- V68 created the role itself (without a usable password). This file owns the
|
||||||
|
-- password lifecycle; nothing else writes it.
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
EXECUTE format('ALTER ROLE grafana_reader WITH PASSWORD %L', '${grafanaDbPassword}');
|
||||||
|
END
|
||||||
|
$$;
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
-- Read-only role used by the Grafana PostgreSQL datasource for the PO Overview
|
||||||
|
-- dashboard (issue #651). The role is created here without a usable password
|
||||||
|
-- (LOGIN-capable but no password set); R__grafana_reader_password.sql sets the
|
||||||
|
-- password from GRAFANA_DB_PASSWORD on every boot, so rotation is just "bump
|
||||||
|
-- the env var and restart the backend" — see docs/adr/024-* and the rotation
|
||||||
|
-- runbook in docs/DEPLOYMENT.md.
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'grafana_reader') THEN
|
||||||
|
CREATE ROLE grafana_reader WITH LOGIN;
|
||||||
|
END IF;
|
||||||
|
END
|
||||||
|
$$;
|
||||||
|
|
||||||
|
GRANT CONNECT ON DATABASE ${flyway:database} TO grafana_reader;
|
||||||
|
GRANT USAGE ON SCHEMA public TO grafana_reader;
|
||||||
|
GRANT SELECT ON audit_log, documents, transcription_blocks TO grafana_reader;
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package org.raddatz.familienarchiv.config;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.mock.env.MockEnvironment;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||||
|
|
||||||
|
class FlywayConfigTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void resolveGrafanaDbPassword_throws_when_env_unset() {
|
||||||
|
FlywayConfig config = new FlywayConfig(null, new MockEnvironment());
|
||||||
|
|
||||||
|
assertThatThrownBy(config::resolveGrafanaDbPassword)
|
||||||
|
.isInstanceOf(IllegalStateException.class)
|
||||||
|
.hasMessageContaining("GRAFANA_DB_PASSWORD is required");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void resolveGrafanaDbPassword_throws_when_env_blank() {
|
||||||
|
MockEnvironment env = new MockEnvironment().withProperty("GRAFANA_DB_PASSWORD", " ");
|
||||||
|
FlywayConfig config = new FlywayConfig(null, env);
|
||||||
|
|
||||||
|
assertThatThrownBy(config::resolveGrafanaDbPassword)
|
||||||
|
.isInstanceOf(IllegalStateException.class)
|
||||||
|
.hasMessageContaining("GRAFANA_DB_PASSWORD is required");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void resolveGrafanaDbPassword_returns_value_when_env_set() {
|
||||||
|
MockEnvironment env = new MockEnvironment().withProperty("GRAFANA_DB_PASSWORD", "abc");
|
||||||
|
FlywayConfig config = new FlywayConfig(null, env);
|
||||||
|
|
||||||
|
assertThat(config.resolveGrafanaDbPassword()).isEqualTo("abc");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package org.raddatz.familienarchiv.config;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.params.ParameterizedTest;
|
||||||
|
import org.junit.jupiter.params.provider.ValueSource;
|
||||||
|
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
|
||||||
|
import org.springframework.boot.jdbc.test.autoconfigure.AutoConfigureTestDatabase;
|
||||||
|
import org.springframework.context.annotation.Import;
|
||||||
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|
||||||
|
// GRAFANA_DB_PASSWORD is supplied via the global test default in
|
||||||
|
// src/test/resources/application.properties — FlywayConfig fails closed
|
||||||
|
// when it is unset, so all tests that load the migration path need it.
|
||||||
|
@DataJpaTest
|
||||||
|
@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE)
|
||||||
|
@Import({PostgresContainerConfig.class, FlywayConfig.class})
|
||||||
|
class GrafanaReaderRoleIntegrationTest {
|
||||||
|
|
||||||
|
@Autowired JdbcTemplate jdbc;
|
||||||
|
|
||||||
|
// --- positive grants (SELECT on the three explicitly granted tables) ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void grafana_reader_has_select_on_audit_log() {
|
||||||
|
assertThat(hasPrivilege("audit_log", "SELECT")).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void grafana_reader_has_select_on_documents() {
|
||||||
|
assertThat(hasPrivilege("documents", "SELECT")).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void grafana_reader_has_select_on_transcription_blocks() {
|
||||||
|
assertThat(hasPrivilege("transcription_blocks", "SELECT")).isTrue();
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- write-deny on the granted tables: SELECT-only means SELECT-only.
|
||||||
|
// A future migration that GRANTs INSERT/UPDATE/DELETE on any of these
|
||||||
|
// would fail these tests, even though the original positive grants still
|
||||||
|
// pass. Locks the boundary in both directions.
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void grafana_reader_has_no_INSERT_on_documents() {
|
||||||
|
assertThat(hasPrivilege("documents", "INSERT")).isFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void grafana_reader_has_no_UPDATE_on_audit_log() {
|
||||||
|
assertThat(hasPrivilege("audit_log", "UPDATE")).isFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void grafana_reader_has_no_DELETE_on_transcription_blocks() {
|
||||||
|
assertThat(hasPrivilege("transcription_blocks", "DELETE")).isFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- negative grants: PII / sensitive tables MUST NOT be readable.
|
||||||
|
// The parameterized form catches the "someone widened the grant to
|
||||||
|
// ALL TABLES IN SCHEMA public" footgun — three specific positive grants
|
||||||
|
// would still pass while this sweep turns red.
|
||||||
|
|
||||||
|
@ParameterizedTest
|
||||||
|
@ValueSource(strings = {
|
||||||
|
"app_users",
|
||||||
|
"user_groups",
|
||||||
|
"persons",
|
||||||
|
"notifications",
|
||||||
|
"document_comments",
|
||||||
|
"document_annotations",
|
||||||
|
"geschichten"
|
||||||
|
})
|
||||||
|
void grafana_reader_has_no_SELECT_on_protected_table(String table) {
|
||||||
|
assertThat(hasPrivilege(table, "SELECT")).isFalse();
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean hasPrivilege(String table, String privilege) {
|
||||||
|
Boolean result = jdbc.queryForObject(
|
||||||
|
"SELECT has_table_privilege('grafana_reader', ?, ?)",
|
||||||
|
Boolean.class,
|
||||||
|
table,
|
||||||
|
privilege);
|
||||||
|
return Boolean.TRUE.equals(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,7 +27,6 @@ import org.springframework.security.test.context.support.WithMockUser;
|
|||||||
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||||
import org.springframework.test.web.servlet.MockMvc;
|
import org.springframework.test.web.servlet.MockMvc;
|
||||||
|
|
||||||
import org.raddatz.familienarchiv.document.DocumentSearchItem;
|
|
||||||
import org.raddatz.familienarchiv.document.SearchMatchData;
|
import org.raddatz.familienarchiv.document.SearchMatchData;
|
||||||
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
@@ -130,16 +129,14 @@ class DocumentControllerTest {
|
|||||||
@WithMockUser
|
@WithMockUser
|
||||||
void search_responseBodyItemsContainMatchData() throws Exception {
|
void search_responseBodyItemsContainMatchData() throws Exception {
|
||||||
UUID docId = UUID.randomUUID();
|
UUID docId = UUID.randomUUID();
|
||||||
Document doc = Document.builder()
|
|
||||||
.id(docId)
|
|
||||||
.title("Brief an Anna")
|
|
||||||
.originalFilename("brief.pdf")
|
|
||||||
.status(DocumentStatus.UPLOADED)
|
|
||||||
.build();
|
|
||||||
var matchData = new SearchMatchData(
|
var matchData = new SearchMatchData(
|
||||||
"Er schrieb einen langen Brief", List.of(), false, List.of(), List.of(), List.of(), null, List.of());
|
"Er schrieb einen langen Brief", List.of(), false, List.of(), List.of(), List.of(), null, List.of());
|
||||||
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||||
.thenReturn(DocumentSearchResult.of(List.of(new DocumentSearchItem(doc, matchData, 0, List.of()))));
|
.thenReturn(DocumentSearchResult.of(List.of(new DocumentListItem(
|
||||||
|
docId, "Brief an Anna", "brief.pdf", null, null, null,
|
||||||
|
List.of(), List.of(), null, null, null, null,
|
||||||
|
0, List.of(), matchData,
|
||||||
|
LocalDateTime.of(2026, 1, 15, 10, 0), LocalDateTime.of(2026, 1, 15, 10, 0)))));
|
||||||
|
|
||||||
mockMvc.perform(get("/api/documents/search").param("q", "Brief"))
|
mockMvc.perform(get("/api/documents/search").param("q", "Brief"))
|
||||||
.andExpect(status().isOk())
|
.andExpect(status().isOk())
|
||||||
@@ -148,6 +145,28 @@ class DocumentControllerTest {
|
|||||||
.value("Er schrieb einen langen Brief"));
|
.value("Er schrieb einen langen Brief"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@WithMockUser
|
||||||
|
void search_returns_flat_item_with_id_and_without_sensitive_fields() throws Exception {
|
||||||
|
UUID docId = UUID.randomUUID();
|
||||||
|
var matchData = new SearchMatchData(null, List.of(), false, List.of(), List.of(), List.of(), null, List.of());
|
||||||
|
when(documentService.searchDocuments(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||||
|
.thenReturn(DocumentSearchResult.of(List.of(new DocumentListItem(
|
||||||
|
docId, "Brief an Anna", "brief.pdf", null, null, null,
|
||||||
|
List.of(), List.of(), null, null, null, null,
|
||||||
|
0, List.of(), matchData,
|
||||||
|
LocalDateTime.of(2026, 1, 15, 10, 0), LocalDateTime.of(2026, 1, 15, 10, 0)))));
|
||||||
|
|
||||||
|
mockMvc.perform(get("/api/documents/search"))
|
||||||
|
.andExpect(status().isOk())
|
||||||
|
// flat id field present at top of item (not nested under $.items[0].document.id)
|
||||||
|
.andExpect(jsonPath("$.items[0].id").value(docId.toString()))
|
||||||
|
// sensitive storage fields must never appear in list response
|
||||||
|
.andExpect(jsonPath("$.items[0].transcription").doesNotExist())
|
||||||
|
.andExpect(jsonPath("$.items[0].filePath").doesNotExist())
|
||||||
|
.andExpect(jsonPath("$.items[0].fileHash").doesNotExist());
|
||||||
|
}
|
||||||
|
|
||||||
// ─── /api/documents/search pagination ─────────────────────────────────────
|
// ─── /api/documents/search pagination ─────────────────────────────────────
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
@@ -127,7 +127,7 @@ class DocumentLazyLoadingTest {
|
|||||||
PageRequest.of(0, 20));
|
PageRequest.of(0, 20));
|
||||||
assertThat(result.totalElements()).isGreaterThan(0);
|
assertThat(result.totalElements()).isGreaterThan(0);
|
||||||
assertThatCode(() ->
|
assertThatCode(() ->
|
||||||
result.items().forEach(i -> i.document().getSender().getLastName()))
|
result.items().forEach(i -> { if (i.sender() != null) i.sender().getLastName(); }))
|
||||||
.doesNotThrowAnyException();
|
.doesNotThrowAnyException();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package org.raddatz.familienarchiv.document;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.AfterEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.raddatz.familienarchiv.PostgresContainerConfig;
|
||||||
|
import org.raddatz.familienarchiv.audit.AuditLogQueryService;
|
||||||
|
import org.raddatz.familienarchiv.ocr.TrainingLabel;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.boot.test.context.SpringBootTest;
|
||||||
|
import org.springframework.context.annotation.Import;
|
||||||
|
import org.springframework.data.domain.PageRequest;
|
||||||
|
import org.springframework.test.context.ActiveProfiles;
|
||||||
|
import org.springframework.test.context.bean.override.mockito.MockitoBean;
|
||||||
|
import software.amazon.awssdk.services.s3.S3Client;
|
||||||
|
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
import static org.assertj.core.api.Assertions.assertThatCode;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* AC #2: Document with trainingLabels does not cause LazyInitializationException in search.
|
||||||
|
* AC #3: Detail API still returns trainingLabels after the Document.list graph change.
|
||||||
|
*/
|
||||||
|
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE)
|
||||||
|
@ActiveProfiles("test")
|
||||||
|
@Import(PostgresContainerConfig.class)
|
||||||
|
class DocumentListItemIntegrationTest {
|
||||||
|
|
||||||
|
@MockitoBean
|
||||||
|
S3Client s3Client;
|
||||||
|
|
||||||
|
@MockitoBean
|
||||||
|
AuditLogQueryService auditLogQueryService;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
DocumentRepository documentRepository;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
DocumentService documentService;
|
||||||
|
|
||||||
|
@AfterEach
|
||||||
|
void cleanup() {
|
||||||
|
documentRepository.deleteAll();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void search_doesNotThrow_whenDocumentHasTrainingLabels() {
|
||||||
|
documentRepository.save(Document.builder()
|
||||||
|
.title("Kurrent Brief")
|
||||||
|
.originalFilename("kurrent.pdf")
|
||||||
|
.status(DocumentStatus.UPLOADED)
|
||||||
|
.trainingLabels(new HashSet<>(Set.of(TrainingLabel.KURRENT_RECOGNITION)))
|
||||||
|
.build());
|
||||||
|
|
||||||
|
assertThatCode(() -> documentService.searchDocuments(
|
||||||
|
null, null, null, null, null, null, null, null,
|
||||||
|
DocumentSort.DATE, "DESC", null,
|
||||||
|
PageRequest.of(0, 50)))
|
||||||
|
.doesNotThrowAnyException();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void search_returns_list_item_without_sensitive_fields_when_document_has_training_labels() {
|
||||||
|
documentRepository.save(Document.builder()
|
||||||
|
.title("Kurrent Brief")
|
||||||
|
.originalFilename("kurrent2.pdf")
|
||||||
|
.status(DocumentStatus.UPLOADED)
|
||||||
|
.trainingLabels(new HashSet<>(Set.of(TrainingLabel.KURRENT_RECOGNITION)))
|
||||||
|
.build());
|
||||||
|
|
||||||
|
DocumentSearchResult result = documentService.searchDocuments(
|
||||||
|
null, null, null, null, null, null, null, null,
|
||||||
|
DocumentSort.DATE, "DESC", null,
|
||||||
|
PageRequest.of(0, 50));
|
||||||
|
|
||||||
|
assertThat(result.totalElements()).isGreaterThan(0);
|
||||||
|
DocumentListItem item = result.items().get(0);
|
||||||
|
assertThat(item.id()).isNotNull();
|
||||||
|
assertThat(item.title()).isEqualTo("Kurrent Brief");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void detail_stillReturnsTrainingLabels() {
|
||||||
|
Document saved = documentRepository.save(Document.builder()
|
||||||
|
.title("Detail Test")
|
||||||
|
.originalFilename("detail_test.pdf")
|
||||||
|
.status(DocumentStatus.UPLOADED)
|
||||||
|
.trainingLabels(new HashSet<>(Set.of(TrainingLabel.KURRENT_RECOGNITION)))
|
||||||
|
.build());
|
||||||
|
|
||||||
|
// Document.full entity graph (used by getDocumentById) must still load trainingLabels
|
||||||
|
Document loaded = documentService.getDocumentById(saved.getId());
|
||||||
|
|
||||||
|
assertThat(loaded.getTrainingLabels()).containsExactly(TrainingLabel.KURRENT_RECOGNITION);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -125,10 +125,10 @@ class DocumentSearchPagedIntegrationTest {
|
|||||||
|
|
||||||
// No document id should appear on both pages — slicing must be exclusive.
|
// No document id should appear on both pages — slicing must be exclusive.
|
||||||
var idsOnPage0 = page0.items().stream()
|
var idsOnPage0 = page0.items().stream()
|
||||||
.map(item -> item.document().getId())
|
.map(item -> item.id())
|
||||||
.toList();
|
.toList();
|
||||||
var idsOnPage1 = page1.items().stream()
|
var idsOnPage1 = page1.items().stream()
|
||||||
.map(item -> item.document().getId())
|
.map(item -> item.id())
|
||||||
.toList();
|
.toList();
|
||||||
for (UUID id : idsOnPage0) {
|
for (UUID id : idsOnPage0) {
|
||||||
assertThat(idsOnPage1).doesNotContain(id);
|
assertThat(idsOnPage1).doesNotContain(id);
|
||||||
|
|||||||
@@ -3,10 +3,9 @@ package org.raddatz.familienarchiv.document;
|
|||||||
import io.swagger.v3.oas.annotations.media.Schema;
|
import io.swagger.v3.oas.annotations.media.Schema;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.raddatz.familienarchiv.audit.ActivityActorDTO;
|
import org.raddatz.familienarchiv.audit.ActivityActorDTO;
|
||||||
import org.raddatz.familienarchiv.document.Document;
|
|
||||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
|
||||||
import org.springframework.data.domain.PageRequest;
|
import org.springframework.data.domain.PageRequest;
|
||||||
|
|
||||||
|
import java.time.LocalDateTime;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
|
||||||
@@ -14,14 +13,12 @@ import static org.assertj.core.api.Assertions.assertThat;
|
|||||||
|
|
||||||
class DocumentSearchResultTest {
|
class DocumentSearchResultTest {
|
||||||
|
|
||||||
private DocumentSearchItem item(UUID docId) {
|
private DocumentListItem item(UUID docId) {
|
||||||
Document doc = Document.builder()
|
return new DocumentListItem(
|
||||||
.id(docId)
|
docId, "Test", "test.pdf", null, null, null,
|
||||||
.title("Test")
|
List.of(), List.of(), null, null, null, null,
|
||||||
.originalFilename("test.pdf")
|
0, List.of(), SearchMatchData.empty(),
|
||||||
.status(DocumentStatus.UPLOADED)
|
LocalDateTime.of(2026, 1, 15, 10, 0), LocalDateTime.of(2026, 1, 15, 10, 0));
|
||||||
.build();
|
|
||||||
return new DocumentSearchItem(doc, SearchMatchData.empty(), 0, List.of());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -45,7 +42,7 @@ class DocumentSearchResultTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void paged_factory_populates_paging_fields_from_pageable_and_total() {
|
void paged_factory_populates_paging_fields_from_pageable_and_total() {
|
||||||
List<DocumentSearchItem> slice = List.of(item(UUID.randomUUID()), item(UUID.randomUUID()));
|
List<DocumentListItem> slice = List.of(item(UUID.randomUUID()), item(UUID.randomUUID()));
|
||||||
|
|
||||||
DocumentSearchResult result = DocumentSearchResult.paged(slice, PageRequest.of(1, 50), 120L);
|
DocumentSearchResult result = DocumentSearchResult.paged(slice, PageRequest.of(1, 50), 120L);
|
||||||
|
|
||||||
@@ -68,9 +65,11 @@ class DocumentSearchResultTest {
|
|||||||
void of_exposes_items_with_completion_and_contributors() {
|
void of_exposes_items_with_completion_and_contributors() {
|
||||||
UUID id = UUID.randomUUID();
|
UUID id = UUID.randomUUID();
|
||||||
ActivityActorDTO actor = new ActivityActorDTO("AB", "#f00", "Anna Braun");
|
ActivityActorDTO actor = new ActivityActorDTO("AB", "#f00", "Anna Braun");
|
||||||
Document doc = Document.builder().id(id).title("T").originalFilename("t.pdf")
|
DocumentListItem item = new DocumentListItem(
|
||||||
.status(DocumentStatus.UPLOADED).build();
|
id, "T", "t.pdf", null, null, null,
|
||||||
DocumentSearchItem item = new DocumentSearchItem(doc, SearchMatchData.empty(), 75, List.of(actor));
|
List.of(), List.of(), null, null, null, null,
|
||||||
|
75, List.of(actor), SearchMatchData.empty(),
|
||||||
|
LocalDateTime.of(2026, 1, 15, 10, 0), LocalDateTime.of(2026, 1, 15, 10, 0));
|
||||||
|
|
||||||
DocumentSearchResult result = DocumentSearchResult.of(List.of(item));
|
DocumentSearchResult result = DocumentSearchResult.of(List.of(item));
|
||||||
|
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ class DocumentServiceSortTest {
|
|||||||
"Brief", null, null, null, null, null, null, null, DocumentSort.DATE, "DESC", null, PAGE);
|
"Brief", null, null, null, null, null, null, null, DocumentSort.DATE, "DESC", null, PAGE);
|
||||||
|
|
||||||
assertThat(result.items()).hasSize(2);
|
assertThat(result.items()).hasSize(2);
|
||||||
assertThat(result.items().get(0).document().getId()).isEqualTo(id2); // newer first
|
assertThat(result.items().get(0).id()).isEqualTo(id2); // newer first
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── RELEVANCE sort — pure text (no filters) ──────────────────────────────
|
// ─── RELEVANCE sort — pure text (no filters) ──────────────────────────────
|
||||||
@@ -104,7 +104,7 @@ class DocumentServiceSortTest {
|
|||||||
DocumentSearchResult result = documentService.searchDocuments(
|
DocumentSearchResult result = documentService.searchDocuments(
|
||||||
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, PAGE);
|
"Brief", null, null, null, null, null, null, null, DocumentSort.RELEVANCE, null, null, PAGE);
|
||||||
|
|
||||||
assertThat(result.items().get(0).document().getId()).isEqualTo(id1);
|
assertThat(result.items().get(0).id()).isEqualTo(id1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -121,7 +121,7 @@ class DocumentServiceSortTest {
|
|||||||
DocumentSearchResult result = documentService.searchDocuments(
|
DocumentSearchResult result = documentService.searchDocuments(
|
||||||
"Brief", null, null, null, null, null, null, null, null, null, null, PAGE);
|
"Brief", null, null, null, null, null, null, null, null, null, null, PAGE);
|
||||||
|
|
||||||
assertThat(result.items().get(0).document().getId()).isEqualTo(id1);
|
assertThat(result.items().get(0).id()).isEqualTo(id1);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── RELEVANCE sort — overflow guard ─────────────────────────────────────
|
// ─── RELEVANCE sort — overflow guard ─────────────────────────────────────
|
||||||
@@ -156,7 +156,7 @@ class DocumentServiceSortTest {
|
|||||||
DocumentSort.RELEVANCE, null, null, PAGE);
|
DocumentSort.RELEVANCE, null, null, PAGE);
|
||||||
|
|
||||||
assertThat(result.items()).hasSize(1);
|
assertThat(result.items()).hasSize(1);
|
||||||
assertThat(result.items().get(0).document().getId()).isEqualTo(uuidId);
|
assertThat(result.items().get(0).id()).isEqualTo(uuidId);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── RELEVANCE sort — text + active filter ────────────────────────────────
|
// ─── RELEVANCE sort — text + active filter ────────────────────────────────
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import org.raddatz.familienarchiv.audit.AuditLogQueryService;
|
|||||||
import org.raddatz.familienarchiv.audit.AuditService;
|
import org.raddatz.familienarchiv.audit.AuditService;
|
||||||
import org.raddatz.familienarchiv.document.annotation.AnnotationService;
|
import org.raddatz.familienarchiv.document.annotation.AnnotationService;
|
||||||
import org.raddatz.familienarchiv.document.transcription.TranscriptionBlockQueryService;
|
import org.raddatz.familienarchiv.document.transcription.TranscriptionBlockQueryService;
|
||||||
import org.raddatz.familienarchiv.document.DocumentSearchItem;
|
import org.raddatz.familienarchiv.document.DocumentListItem;
|
||||||
import org.raddatz.familienarchiv.document.DocumentSearchResult;
|
import org.raddatz.familienarchiv.document.DocumentSearchResult;
|
||||||
import org.raddatz.familienarchiv.document.DocumentSort;
|
import org.raddatz.familienarchiv.document.DocumentSort;
|
||||||
import org.raddatz.familienarchiv.document.DocumentUpdateDTO;
|
import org.raddatz.familienarchiv.document.DocumentUpdateDTO;
|
||||||
@@ -1444,7 +1444,7 @@ class DocumentServiceTest {
|
|||||||
assertThat(result.totalPages()).isEqualTo(3);
|
assertThat(result.totalPages()).isEqualTo(3);
|
||||||
assertThat(result.items()).hasSize(50);
|
assertThat(result.items()).hasSize(50);
|
||||||
// Page 1 (offset 50) under ascending sender sort should start at L050
|
// Page 1 (offset 50) under ascending sender sort should start at L050
|
||||||
assertThat(result.items().get(0).document().getSender().getLastName()).isEqualTo("L050");
|
assertThat(result.items().get(0).sender().getLastName()).isEqualTo("L050");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@@ -1565,7 +1565,7 @@ class DocumentServiceTest {
|
|||||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, UNPAGED);
|
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, UNPAGED);
|
||||||
|
|
||||||
assertThat(result.items()).hasSize(2);
|
assertThat(result.items()).hasSize(2);
|
||||||
assertThat(result.items()).extracting(item -> item.document().getTitle()).containsExactly("Has Sender", "No Sender");
|
assertThat(result.items()).extracting(DocumentListItem::title).containsExactly("Has Sender", "No Sender");
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── searchDocuments — RECEIVER sort, empty receivers ───────────────────────
|
// ─── searchDocuments — RECEIVER sort, empty receivers ───────────────────────
|
||||||
@@ -1584,7 +1584,7 @@ class DocumentServiceTest {
|
|||||||
DocumentSearchResult result = documentService.searchDocuments(
|
DocumentSearchResult result = documentService.searchDocuments(
|
||||||
null, null, null, null, null, null, null, null, DocumentSort.RECEIVER, "asc", null, UNPAGED);
|
null, null, null, null, null, null, null, null, DocumentSort.RECEIVER, "asc", null, UNPAGED);
|
||||||
|
|
||||||
assertThat(result.items()).extracting(item -> item.document().getTitle())
|
assertThat(result.items()).extracting(DocumentListItem::title)
|
||||||
.containsExactly("Has Receiver", "No Receivers");
|
.containsExactly("Has Receiver", "No Receivers");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1607,7 +1607,7 @@ class DocumentServiceTest {
|
|||||||
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, UNPAGED);
|
null, null, null, null, null, null, null, null, DocumentSort.SENDER, "asc", null, UNPAGED);
|
||||||
|
|
||||||
// null lastName should sort to end (treated as empty), not before "smith" (as "null")
|
// null lastName should sort to end (treated as empty), not before "smith" (as "null")
|
||||||
assertThat(result.items()).extracting(item -> item.document().getTitle())
|
assertThat(result.items()).extracting(DocumentListItem::title)
|
||||||
.containsExactly("smith doc", "Null lastname doc");
|
.containsExactly("smith doc", "Null lastname doc");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,2 +1,8 @@
|
|||||||
logging.level.root=WARN
|
logging.level.root=WARN
|
||||||
logging.level.org.raddatz=INFO
|
logging.level.org.raddatz=INFO
|
||||||
|
|
||||||
|
# Default test value so FlywayConfig's fail-closed check passes without each
|
||||||
|
# test having to set GRAFANA_DB_PASSWORD explicitly. The actual value is
|
||||||
|
# irrelevant in tests — Flyway only uses it to set the grafana_reader role's
|
||||||
|
# password, which no test connects with.
|
||||||
|
GRAFANA_DB_PASSWORD=test-grafana-reader-password
|
||||||
|
|||||||
@@ -147,6 +147,9 @@ services:
|
|||||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-changeme}
|
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-changeme}
|
||||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||||
GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL:-http://localhost:3003}
|
GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL:-http://localhost:3003}
|
||||||
|
# Read-only password for the grafana_reader PostgreSQL role; interpolated
|
||||||
|
# into the provisioned PostgreSQL datasource (see datasources.yml).
|
||||||
|
GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD}
|
||||||
volumes:
|
volumes:
|
||||||
- grafana_data:/var/lib/grafana
|
- grafana_data:/var/lib/grafana
|
||||||
- ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro
|
- ./infra/observability/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
@@ -165,6 +168,7 @@ services:
|
|||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
networks:
|
networks:
|
||||||
- obs-net
|
- obs-net
|
||||||
|
- archiv-net # PO Overview dashboard queries archive-db via the grafana_reader role
|
||||||
|
|
||||||
# --- Error Tracking: GlitchTip ---
|
# --- Error Tracking: GlitchTip ---
|
||||||
|
|
||||||
|
|||||||
@@ -227,6 +227,9 @@ services:
|
|||||||
SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/archiv
|
SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/archiv
|
||||||
SPRING_DATASOURCE_USERNAME: archiv
|
SPRING_DATASOURCE_USERNAME: archiv
|
||||||
SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD}
|
SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD}
|
||||||
|
# Consumed by Flyway V68 via the ${grafanaDbPassword} placeholder to set
|
||||||
|
# the read-only grafana_reader role's password.
|
||||||
|
GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD}
|
||||||
# Application uses the bucket-scoped service account, not MinIO root.
|
# Application uses the bucket-scoped service account, not MinIO root.
|
||||||
S3_ENDPOINT: http://minio:9000
|
S3_ENDPOINT: http://minio:9000
|
||||||
S3_ACCESS_KEY: archiv-app
|
S3_ACCESS_KEY: archiv-app
|
||||||
|
|||||||
@@ -163,6 +163,9 @@ services:
|
|||||||
SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${POSTGRES_DB}
|
SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${POSTGRES_DB}
|
||||||
SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER}
|
SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER}
|
||||||
SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD}
|
SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD}
|
||||||
|
# Consumed by Flyway V68 via the ${grafanaDbPassword} placeholder to set
|
||||||
|
# the read-only grafana_reader role's password.
|
||||||
|
GRAFANA_DB_PASSWORD: ${GRAFANA_DB_PASSWORD}
|
||||||
S3_ENDPOINT: http://minio:9000
|
S3_ENDPOINT: http://minio:9000
|
||||||
S3_ACCESS_KEY: ${MINIO_ROOT_USER}
|
S3_ACCESS_KEY: ${MINIO_ROOT_USER}
|
||||||
S3_SECRET_KEY: ${MINIO_ROOT_PASSWORD}
|
S3_SECRET_KEY: ${MINIO_ROOT_PASSWORD}
|
||||||
|
|||||||
@@ -152,6 +152,7 @@ All vars are set in `.env` at the repo root (copy from `.env.example`). The back
|
|||||||
| `PORT_GRAFANA` | Host port for the Grafana UI (bound to `127.0.0.1` only) | `3003` | — | — |
|
| `PORT_GRAFANA` | Host port for the Grafana UI (bound to `127.0.0.1` only) | `3003` | — | — |
|
||||||
| `POSTGRES_HOST` | PostgreSQL hostname for GlitchTip's db-init job and workers. Override when only the staging stack is running and `archive-db` is not resolvable by that name. | `archive-db` | — | — |
|
| `POSTGRES_HOST` | PostgreSQL hostname for GlitchTip's db-init job and workers. Override when only the staging stack is running and `archive-db` is not resolvable by that name. | `archive-db` | — | — |
|
||||||
| `GRAFANA_ADMIN_PASSWORD` | Grafana `admin` user password | `changeme` | YES (prod) | YES |
|
| `GRAFANA_ADMIN_PASSWORD` | Grafana `admin` user password | `changeme` | YES (prod) | YES |
|
||||||
|
| `GRAFANA_DB_PASSWORD` | Password for the read-only `grafana_reader` PostgreSQL role used by the PO Overview dashboard (issue #651). Consumed by Flyway V68 and the Grafana PostgreSQL datasource. Generate with `openssl rand -hex 32`. | — | YES (prod) | YES |
|
||||||
| `PORT_GLITCHTIP` | Host port for the GlitchTip UI (bound to `127.0.0.1` only) | `3002` | — | — |
|
| `PORT_GLITCHTIP` | Host port for the GlitchTip UI (bound to `127.0.0.1` only) | `3002` | — | — |
|
||||||
| `GLITCHTIP_DOMAIN` | Public-facing base URL for GlitchTip (used in email links and CORS) | `http://localhost:3002` | YES (prod) | — |
|
| `GLITCHTIP_DOMAIN` | Public-facing base URL for GlitchTip (used in email links and CORS) | `http://localhost:3002` | YES (prod) | — |
|
||||||
| `GLITCHTIP_SECRET_KEY` | Django secret key for GlitchTip — generate with `python3 -c "import secrets; print(secrets.token_hex(32))"` | — | YES | YES |
|
| `GLITCHTIP_SECRET_KEY` | Django secret key for GlitchTip — generate with `python3 -c "import secrets; print(secrets.token_hex(32))"` | — | YES | YES |
|
||||||
@@ -256,6 +257,7 @@ git.raddatz.cloud A <server IP>
|
|||||||
| `MAIL_USERNAME` | release.yml | SMTP user |
|
| `MAIL_USERNAME` | release.yml | SMTP user |
|
||||||
| `MAIL_PASSWORD` | release.yml | SMTP password |
|
| `MAIL_PASSWORD` | release.yml | SMTP password |
|
||||||
| `GRAFANA_ADMIN_PASSWORD` | both | Grafana `admin` login — generate a strong password |
|
| `GRAFANA_ADMIN_PASSWORD` | both | Grafana `admin` login — generate a strong password |
|
||||||
|
| `GRAFANA_DB_PASSWORD` | both | Read-only `grafana_reader` role password — `openssl rand -hex 32` |
|
||||||
| `GLITCHTIP_SECRET_KEY` | both | Django secret key — `openssl rand -hex 32` |
|
| `GLITCHTIP_SECRET_KEY` | both | Django secret key — `openssl rand -hex 32` |
|
||||||
| `SENTRY_DSN` | both | GlitchTip project DSN — set after first-run (§4); leave empty to keep Sentry disabled |
|
| `SENTRY_DSN` | both | GlitchTip project DSN — set after first-run (§4); leave empty to keep Sentry disabled |
|
||||||
| `VITE_SENTRY_DSN` | both | GlitchTip frontend project DSN — set after first-run (§4); leave empty to keep Sentry disabled |
|
| `VITE_SENTRY_DSN` | both | GlitchTip frontend project DSN — set after first-run (§4); leave empty to keep Sentry disabled |
|
||||||
@@ -357,6 +359,7 @@ Both files are passed explicitly via `--env-file` to the compose command, so the
|
|||||||
| Gitea secret | Notes |
|
| Gitea secret | Notes |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `GRAFANA_ADMIN_PASSWORD` | Strong unique password; shared by nightly and release |
|
| `GRAFANA_ADMIN_PASSWORD` | Strong unique password; shared by nightly and release |
|
||||||
|
| `GRAFANA_DB_PASSWORD` | `openssl rand -hex 32`; shared by nightly and release — read-only DB role for the PO Overview dashboard |
|
||||||
| `GLITCHTIP_SECRET_KEY` | `openssl rand -hex 32`; shared by nightly and release |
|
| `GLITCHTIP_SECRET_KEY` | `openssl rand -hex 32`; shared by nightly and release |
|
||||||
| `STAGING_POSTGRES_PASSWORD` / `PROD_POSTGRES_PASSWORD` | Must match the running PostgreSQL container |
|
| `STAGING_POSTGRES_PASSWORD` / `PROD_POSTGRES_PASSWORD` | Must match the running PostgreSQL container |
|
||||||
|
|
||||||
@@ -427,6 +430,31 @@ docker exec obs-loki wget -qO- \
|
|||||||
|
|
||||||
Prometheus port `9090` and Grafana port `3003` (default; configurable via `PORT_GRAFANA`) are bound to `127.0.0.1` on the host. No other observability ports are host-bound.
|
Prometheus port `9090` and Grafana port `3003` (default; configurable via `PORT_GRAFANA`) are bound to `127.0.0.1` on the host. No other observability ports are host-bound.
|
||||||
|
|
||||||
|
##### Rotate the `grafana_reader` DB password
|
||||||
|
|
||||||
|
The PO Overview dashboard reads `audit_log`, `documents`, and `transcription_blocks` through the SELECT-only `grafana_reader` PostgreSQL role (issue #651, ADR-024). The role's password is owned by `R__grafana_reader_password.sql` — a Flyway *repeatable* migration that re-runs whenever the resolved `${grafanaDbPassword}` placeholder changes. That makes rotation a two-restart operation, no manual `psql` required.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Generate a new value
|
||||||
|
openssl rand -hex 32
|
||||||
|
|
||||||
|
# 2. Update both sides:
|
||||||
|
# - Gitea secret GRAFANA_DB_PASSWORD (nightly + release workflows pick it up)
|
||||||
|
# - Local .env on the server / dev machine
|
||||||
|
|
||||||
|
# 3. Restart the backend. Flyway sees that R__'s resolved checksum changed and
|
||||||
|
# re-applies it, issuing ALTER ROLE grafana_reader WITH PASSWORD '<new>'.
|
||||||
|
docker compose restart backend
|
||||||
|
|
||||||
|
# 4. Restart obs-grafana so the provisioned datasource picks up the new env value.
|
||||||
|
docker compose -f docker-compose.observability.yml restart obs-grafana
|
||||||
|
|
||||||
|
# 5. Verify the dashboard loads — PO Overview's Postgres panels should populate
|
||||||
|
# instead of "Data source error".
|
||||||
|
```
|
||||||
|
|
||||||
|
If `GRAFANA_DB_PASSWORD` is unset, the backend **refuses to start** (`IllegalStateException`). That is deliberate — see `FlywayConfig.resolveGrafanaDbPassword()` and the rationale in ADR-024.
|
||||||
|
|
||||||
#### GlitchTip
|
#### GlitchTip
|
||||||
|
|
||||||
| Item | Value |
|
| Item | Value |
|
||||||
|
|||||||
@@ -80,6 +80,14 @@ _See also [DocumentStatus lifecycle](#documentstatus-lifecycle)._
|
|||||||
|
|
||||||
**Sütterlin** — A specific standardized style of Kurrent taught in German schools from 1915 to 1941.
|
**Sütterlin** — A specific standardized style of Kurrent taught in German schools from 1915 to 1941.
|
||||||
|
|
||||||
|
**Illegible word** — a word whose recognition confidence falls below the configured threshold; replaced with the literal token `[unleserlich]` in the rendered block text and counted in the `ocr_illegible_words_total` Prometheus counter.
|
||||||
|
|
||||||
|
**Models-ready gauge** — the `ocr_models_ready` Prometheus gauge, flipped from `0` to `1` once the FastAPI lifespan startup has finished loading the Kraken model and the spell-checker. Used both for the `/health` endpoint and as the supervised signal for the `ocr_models_ready < 1 for 2m` alert.
|
||||||
|
|
||||||
|
**Recognition model accuracy** — the accuracy reported by `ketos train` for the recognition (text-line) model, exposed as `ocr_model_accuracy{kind="recognition"}`. Sourced from `_parse_best_checkpoint` on the highest-scoring checkpoint after training.
|
||||||
|
|
||||||
|
**Segmentation model accuracy** — the accuracy reported by `ketos segtrain` for the baseline layout analysis (`blla`) model, exposed as `ocr_model_accuracy{kind="segmentation"}`. Distinct from recognition accuracy because the two models are trained and improved independently.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Other Domain Terms
|
## Other Domain Terms
|
||||||
|
|||||||
@@ -118,11 +118,14 @@ To find a trace for a specific request in staging/production, either increase th
|
|||||||
|
|
||||||
## Metrics (Prometheus → Grafana)
|
## Metrics (Prometheus → Grafana)
|
||||||
|
|
||||||
Prometheus scrapes the backend management endpoint every 15 s:
|
Prometheus scrapes two targets every 15 s:
|
||||||
|
|
||||||
```
|
```
|
||||||
Target: backend:8081/actuator/prometheus
|
Target: backend:8081/actuator/prometheus
|
||||||
Labels: job="spring-boot", application="Familienarchiv"
|
Labels: job="spring-boot", application="Familienarchiv"
|
||||||
|
|
||||||
|
Target: ocr:8000/metrics
|
||||||
|
Labels: job="ocr-service"
|
||||||
```
|
```
|
||||||
|
|
||||||
All Spring Boot metrics carry the `application="Familienarchiv"` tag, which is how the Grafana Spring Boot Observability dashboard (ID 17175) filters to this service.
|
All Spring Boot metrics carry the `application="Familienarchiv"` tag, which is how the Grafana Spring Boot Observability dashboard (ID 17175) filters to this service.
|
||||||
@@ -146,6 +149,70 @@ jvm_memory_used_bytes{area="heap", application="Familienarchiv"}
|
|||||||
hikaricp_connections_active
|
hikaricp_connections_active
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### OCR-service custom metrics
|
||||||
|
|
||||||
|
Exposed at `ocr:8000/metrics` by `prometheus-fastapi-instrumentator`. The
|
||||||
|
`http_*` metrics describe the FastAPI request layer; the `ocr_*` series are
|
||||||
|
domain-specific. **Never label these with PII or document content** — labels
|
||||||
|
have unbounded cardinality risk and are visible to anyone with Grafana access.
|
||||||
|
|
||||||
|
| Metric | Type | Labels | Unit | What it tracks |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `ocr_jobs_total` | Counter | `engine` (`surya`/`kraken`), `script_type` | jobs | OCR jobs that started after a successful PDF download |
|
||||||
|
| `ocr_pages_total` | Counter | `engine` | pages | Successfully OCR'd pages in the streaming generator |
|
||||||
|
| `ocr_skipped_pages_total` | Counter | — | pages | Pages skipped because the engine raised on them |
|
||||||
|
| `ocr_words_total` | Counter | — | words | Recognized words summed across every block |
|
||||||
|
| `ocr_illegible_words_total` | Counter | — | words | Words below the confidence threshold (rendered as `[unleserlich]`) |
|
||||||
|
| `ocr_processing_seconds` | Histogram | `engine` | seconds | Per-page (stream) or per-document (`/ocr`) engine time, excluding preprocessing |
|
||||||
|
| `ocr_training_runs_total` | Counter | `kind` (`recognition`/`segmentation`), `outcome` (`success`/`error`) | runs | Completed training runs |
|
||||||
|
| `ocr_model_accuracy` | Gauge | `kind` | ratio (0–1) | Latest accuracy reported by a successful training run |
|
||||||
|
| `ocr_models_ready` | Gauge | — | 0\|1 | 1 once the lifespan startup has finished loading models |
|
||||||
|
|
||||||
|
Canonical example queries (the same ones referenced in issue #652):
|
||||||
|
|
||||||
|
```promql
|
||||||
|
# OCR throughput by engine
|
||||||
|
sum by (engine) (rate(ocr_pages_total[5m]))
|
||||||
|
|
||||||
|
# Share of words rendered as [unleserlich]
|
||||||
|
sum(rate(ocr_illegible_words_total[5m]))
|
||||||
|
/ sum(rate(ocr_words_total[5m]))
|
||||||
|
|
||||||
|
# p95 page processing time per engine
|
||||||
|
histogram_quantile(0.95, sum by (engine, le) (
|
||||||
|
rate(ocr_processing_seconds_bucket[5m])
|
||||||
|
))
|
||||||
|
|
||||||
|
# Training error rate
|
||||||
|
sum(rate(ocr_training_runs_total{outcome="error"}[1h]))
|
||||||
|
/ sum(rate(ocr_training_runs_total[1h]))
|
||||||
|
|
||||||
|
# Latest recognition vs segmentation accuracy
|
||||||
|
ocr_model_accuracy
|
||||||
|
```
|
||||||
|
|
||||||
|
### Internal-only endpoints
|
||||||
|
|
||||||
|
`/metrics` is exposed by the OCR service over plain HTTP without
|
||||||
|
authentication. The container is reachable only on the internal Docker
|
||||||
|
network — Caddy never proxies to it directly. If the service is ever
|
||||||
|
exposed (e.g. a `ports:` mapping is added), block the endpoint at the
|
||||||
|
reverse proxy:
|
||||||
|
|
||||||
|
```caddy
|
||||||
|
ocr.example.com {
|
||||||
|
@internal_only path /metrics /health
|
||||||
|
respond @internal_only 404
|
||||||
|
reverse_proxy ocr:8000
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The `MetricsPathFilter` in `ocr-service/main.py` suppresses uvicorn's
|
||||||
|
**stdout** access log lines for `/metrics` and `/health` so the container
|
||||||
|
console stays focused on real OCR traffic. Promtail/Loki still receive
|
||||||
|
access lines from any other source. Treat the filter as console
|
||||||
|
noise-control, not an audit-suppression mechanism.
|
||||||
|
|
||||||
## Errors (GlitchTip)
|
## Errors (GlitchTip)
|
||||||
|
|
||||||
GlitchTip receives errors from both the backend (via Sentry Java SDK) and the frontend (via Sentry JavaScript SDK). It groups events by fingerprint, tracks first/last seen times, and links to the release that introduced the error.
|
GlitchTip receives errors from both the backend (via Sentry Java SDK) and the frontend (via Sentry JavaScript SDK). It groups events by fingerprint, tracks first/last seen times, and links to the release that introduced the error.
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
# ADR-023: Prometheus Instrumentator and Metrics Registry Injection
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Until issue #652 the OCR service exposed no `/metrics` endpoint. The
|
||||||
|
observability stack already scrapes the Spring Boot backend's actuator
|
||||||
|
endpoint, but it had nothing to scrape on the Python side. Without HTTP-
|
||||||
|
and domain-level metrics from `ocr-service` we cannot answer questions
|
||||||
|
like "what is the share of words rendered as `[unleserlich]`" or
|
||||||
|
"is the training error rate above its budget" from Grafana.
|
||||||
|
|
||||||
|
Two implementation requirements influenced the design:
|
||||||
|
|
||||||
|
1. **Counter / gauge isolation in tests.** `prometheus_client` collectors
|
||||||
|
are module-level singletons keyed by name on the global `REGISTRY`.
|
||||||
|
Re-importing or naively re-instantiating them raises a duplicated-
|
||||||
|
collector error and cross-test state leaks (a `.inc()` in test A is
|
||||||
|
still readable by test B). A test harness needs a way to swap the
|
||||||
|
active container for a fresh per-test instance.
|
||||||
|
|
||||||
|
2. **Minimal blast radius on the request path.** We did not want to
|
||||||
|
hand-instrument every endpoint with FastAPI middleware. The
|
||||||
|
`prometheus-fastapi-instrumentator` library already provides
|
||||||
|
`http_requests_total`, `http_request_duration_seconds`, and the
|
||||||
|
`/metrics` exposition route, all idiomatic Prometheus names.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
- Add `prometheus-fastapi-instrumentator==7.0.0` and pin its transitive
|
||||||
|
dependency `prometheus-client==0.25.0` explicitly in
|
||||||
|
`ocr-service/requirements.txt`.
|
||||||
|
- Mount the instrumentator once at module load:
|
||||||
|
`Instrumentator(excluded_handlers=["/health", "/metrics"]).instrument(app).expose(app)`.
|
||||||
|
This adds `/metrics` and an HTTP-level dashboard surface without
|
||||||
|
changing any endpoint code.
|
||||||
|
- Define every domain metric (`ocr_jobs_total`, `ocr_pages_total`,
|
||||||
|
`ocr_processing_seconds`, …) inside a `build_metrics(registry)`
|
||||||
|
factory in `ocr-service/metrics.py` that returns a frozen `OcrMetrics`
|
||||||
|
dataclass. Production code binds the container to the default
|
||||||
|
`REGISTRY` once: `metrics: OcrMetrics = build_metrics(REGISTRY)`.
|
||||||
|
- Tests use a `fresh_metrics` fixture that builds a new
|
||||||
|
`CollectorRegistry()` per test and monkeypatches `main.metrics` with
|
||||||
|
a container bound to it. The endpoint code keeps reading
|
||||||
|
`metrics.<name>` without knowing whether it is talking to the global
|
||||||
|
registry or a per-test one.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
**Positive**
|
||||||
|
|
||||||
|
- One reusable factory captures the metric definitions; future metrics
|
||||||
|
go in one place.
|
||||||
|
- Tests run with full counter isolation. Cross-test state leakage is
|
||||||
|
impossible because each test sees its own dataclass instance.
|
||||||
|
- The instrumentator gives us `http_*` metrics for free, including a
|
||||||
|
Grafana-ready histogram that pairs with the Spring Boot one.
|
||||||
|
|
||||||
|
**Negative**
|
||||||
|
|
||||||
|
- One extra level of indirection: any test that asserts on metric
|
||||||
|
values must remember to monkeypatch `main.metrics`, not the registry
|
||||||
|
directly. Rebinding through the registry is harmless but useless —
|
||||||
|
the dataclass holds references to the original collectors.
|
||||||
|
- `prometheus-client` is now pinned. Upgrading it requires an explicit
|
||||||
|
bump and re-checking the instrumentator's compatibility range.
|
||||||
|
- `/metrics` is exposed unauthenticated and relies on the Docker
|
||||||
|
internal network for confidentiality. See
|
||||||
|
[docs/OBSERVABILITY.md §Internal-only endpoints](../OBSERVABILITY.md)
|
||||||
|
for the Caddy snippet that must be added if the service ever gets a
|
||||||
|
host-side port mapping.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- **Hand-roll the `/metrics` endpoint.** Rejected: would have meant
|
||||||
|
duplicating what `prometheus-fastapi-instrumentator` ships, plus
|
||||||
|
middleware for the HTTP histograms.
|
||||||
|
- **Skip the factory; pass `registry` as a function argument
|
||||||
|
everywhere.** Rejected: clutters every endpoint signature and breaks
|
||||||
|
the symmetry with the Spring Boot side, which also relies on a
|
||||||
|
process-global Micrometer registry.
|
||||||
|
- **Use a `pytest` autouse fixture that resets `REGISTRY` between
|
||||||
|
tests.** Rejected: `prometheus_client` does not expose a clean
|
||||||
|
"unregister all" hook, and we would be relying on private APIs.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- Issue: [#652](https://git.raddatz.cloud/marcel/familienarchiv/issues/652)
|
||||||
|
- Library: <https://github.com/trallnag/prometheus-fastapi-instrumentator>
|
||||||
|
- Code: `ocr-service/metrics.py`, `ocr-service/main.py`,
|
||||||
|
`ocr-service/test_metrics.py`
|
||||||
123
docs/adr/024-grafana-reads-archive-db-via-bridged-network.md
Normal file
123
docs/adr/024-grafana-reads-archive-db-via-bridged-network.md
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
# ADR-024: Grafana reads archive-db via a bridged network and a SELECT-only role
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Issue #651 (the PO Overview Grafana dashboard) needs aggregates over three
|
||||||
|
tables in the main application database — `audit_log`, `documents`, and
|
||||||
|
`transcription_blocks` — to answer the operator's four weekly questions: is
|
||||||
|
everything working, are people using it, is the archive making progress, is
|
||||||
|
OCR working well.
|
||||||
|
|
||||||
|
Until now, `obs-grafana` and the rest of the observability stack lived on
|
||||||
|
their own Docker network (`obs-net`) and never touched `archiv-net`, where
|
||||||
|
`archive-db` runs. The two were intentionally isolated: a compromise of any
|
||||||
|
observability container could not pivot to the application database.
|
||||||
|
|
||||||
|
The PO Overview's archive-progress and user-activity panels need rolling
|
||||||
|
7-day SQL aggregates that cannot be served by Prometheus or Loki. That
|
||||||
|
forces a connection from `obs-grafana` to `archive-db` for the first time.
|
||||||
|
|
||||||
|
Two implementation requirements shaped the design:
|
||||||
|
|
||||||
|
1. **Least privilege on the database side.** The Spring Boot application
|
||||||
|
role (`archiv`) has full read/write on every table. Letting Grafana
|
||||||
|
connect with that role would mean a Grafana compromise becomes an
|
||||||
|
application compromise. The dashboard only needs SELECT on three
|
||||||
|
tables; the role must reflect that and nothing more.
|
||||||
|
|
||||||
|
2. **Operational simplicity of secret rotation.** The role's password is
|
||||||
|
shared between the migration that sets it and the Grafana datasource
|
||||||
|
that uses it. A first version of this work put the password in a
|
||||||
|
versioned Flyway migration (V68), which Flyway only applies once —
|
||||||
|
leaving rotation as an out-of-band `psql ALTER ROLE` step that no
|
||||||
|
runbook documented. The shape must support rotation without manual
|
||||||
|
SQL.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
- Provision a dedicated PostgreSQL role `grafana_reader` with `LOGIN` plus
|
||||||
|
`GRANT SELECT` on `audit_log`, `documents`, `transcription_blocks` only.
|
||||||
|
No INSERT/UPDATE/DELETE on any table, no access to any other table —
|
||||||
|
enforced by the database, locked in by both positive and parameterized
|
||||||
|
negative tests in `GrafanaReaderRoleIntegrationTest`.
|
||||||
|
- Split the role's lifecycle across two migrations:
|
||||||
|
- `V68__add_grafana_reader_role.sql` — versioned, immutable, idempotent.
|
||||||
|
Creates the role and applies the grants. Runs exactly once per
|
||||||
|
database, like every other versioned migration.
|
||||||
|
- `R__grafana_reader_password.sql` — Flyway *repeatable* migration that
|
||||||
|
issues `ALTER ROLE grafana_reader WITH PASSWORD '${grafanaDbPassword}'`.
|
||||||
|
Flyway computes the checksum on the resolved content, so any change
|
||||||
|
to `GRAFANA_DB_PASSWORD` flips the checksum and re-applies the
|
||||||
|
migration on the next boot. Rotation becomes "bump env var, restart
|
||||||
|
backend, restart obs-grafana" — see the runbook in
|
||||||
|
`docs/DEPLOYMENT.md §4 → Rotate the grafana_reader DB password`.
|
||||||
|
- Resolve the password through Spring's `Environment` rather than a raw
|
||||||
|
`System.getenv()` call, so tests inject via `application.properties`
|
||||||
|
and the resolver is unit-testable with `MockEnvironment`. Fail closed
|
||||||
|
with `IllegalStateException` when the variable is unset — no fallback
|
||||||
|
string. Same shape as `UserDataInitializer`'s refusal to seed default
|
||||||
|
admin credentials outside dev/test/e2e.
|
||||||
|
- Join `obs-grafana` to `archiv-net` in addition to `obs-net`. Only the
|
||||||
|
Grafana container crosses the boundary; Loki, Tempo, Prometheus,
|
||||||
|
GlitchTip, and the worker containers remain `obs-net`-only.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
**Positive**
|
||||||
|
|
||||||
|
- Database-level least privilege: a Grafana compromise gains SELECT on
|
||||||
|
three tables. Cannot write, cannot read PII tables like `app_users`,
|
||||||
|
`persons`, `notifications`, `document_comments`, `geschichten`. The
|
||||||
|
parameterized PII negative sweep in `GrafanaReaderRoleIntegrationTest`
|
||||||
|
is the regression gate; new sensitive tables get added to that list.
|
||||||
|
- Rotation is documented, idempotent, and survives operator turnover.
|
||||||
|
No "the password set on day 1 is the password forever" failure mode.
|
||||||
|
- Tests pin down both sides of the boundary: positive grants must hold,
|
||||||
|
write-deny must hold, and the PII negative list must stay empty.
|
||||||
|
|
||||||
|
**Negative / trade-offs**
|
||||||
|
|
||||||
|
- `obs-net` is no longer fully isolated from `archiv-net`. A Grafana RCE
|
||||||
|
(e.g. via a future Grafana CVE) gains a TCP path to `archive-db` —
|
||||||
|
contained, but not impossible. The least-privilege role is the
|
||||||
|
mitigation; we accept that mitigation as sufficient for a single
|
||||||
|
bridged container.
|
||||||
|
- The backend must hold `GRAFANA_DB_PASSWORD` in its environment forever,
|
||||||
|
so Flyway can resolve the placeholder on every boot. A backend RCE
|
||||||
|
therefore also leaks the Grafana datasource password. Acceptable
|
||||||
|
because that password's blast radius is itself bounded by the
|
||||||
|
least-privilege grants on `grafana_reader`.
|
||||||
|
|
||||||
|
## Alternatives considered
|
||||||
|
|
||||||
|
- **Prometheus PostgreSQL exporter, no direct connection.** Loses ad-hoc
|
||||||
|
SQL aggregates — the dashboard would need every metric pre-defined as
|
||||||
|
an exporter query, with a redeploy to add a new one. The PO Overview
|
||||||
|
is the type of dashboard that grows panels over time; pre-defining
|
||||||
|
every aggregate is the wrong shape.
|
||||||
|
- **Read replica or logical-replication slot dedicated to Grafana.**
|
||||||
|
Real operational cost (extra Postgres instance, replication monitoring,
|
||||||
|
storage doubled) disproportionate to a weekly PO glance.
|
||||||
|
- **Versioned migration with `flyway repair` for rotation.** Rejected:
|
||||||
|
conflates schema lifecycle with credential lifecycle, requires manual
|
||||||
|
intervention to rotate, and the repair command's semantics are
|
||||||
|
surprising to operators unfamiliar with Flyway internals.
|
||||||
|
- **Hardcoded fallback password when env var is unset.** Rejected as a
|
||||||
|
security blocker: publishes a known credential for a role with read
|
||||||
|
access to user activity and full letter text. The fail-closed
|
||||||
|
behavior is the explicit defense.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- Issue #651 — PO Overview Grafana dashboard
|
||||||
|
- `backend/src/main/resources/db/migration/V68__add_grafana_reader_role.sql`
|
||||||
|
- `backend/src/main/resources/db/migration/R__grafana_reader_password.sql`
|
||||||
|
- `backend/src/main/java/org/raddatz/familienarchiv/config/FlywayConfig.java`
|
||||||
|
- `backend/src/test/java/org/raddatz/familienarchiv/config/GrafanaReaderRoleIntegrationTest.java`
|
||||||
|
- `infra/observability/grafana/provisioning/datasources/datasources.yml`
|
||||||
|
- `docker-compose.observability.yml` — `archiv-net` bridge on `obs-grafana`
|
||||||
|
- `docs/DEPLOYMENT.md §4` — rotation runbook
|
||||||
@@ -43,9 +43,12 @@ Rel(ocr, storage, "Fetches PDF via presigned URL", "HTTP / S3 presigned")
|
|||||||
Rel(mc, storage, "Bootstraps bucket + service account on startup", "MinIO Client CLI")
|
Rel(mc, storage, "Bootstraps bucket + service account on startup", "MinIO Client CLI")
|
||||||
Rel(promtail, loki, "Pushes log streams", "HTTP/Loki push API")
|
Rel(promtail, loki, "Pushes log streams", "HTTP/Loki push API")
|
||||||
Rel(backend, tempo, "Sends distributed traces via OTLP", "HTTP / OTLP / port 4318 (archiv-net)")
|
Rel(backend, tempo, "Sends distributed traces via OTLP", "HTTP / OTLP / port 4318 (archiv-net)")
|
||||||
|
Rel(prometheus, backend, "Scrapes JVM + HTTP metrics", "HTTP 8081 /actuator/prometheus")
|
||||||
|
Rel(prometheus, ocr, "Scrapes OCR + http_* metrics", "HTTP 8000 /metrics")
|
||||||
Rel(grafana, prometheus, "Queries metrics", "HTTP 9090")
|
Rel(grafana, prometheus, "Queries metrics", "HTTP 9090")
|
||||||
Rel(grafana, loki, "Queries logs", "HTTP 3100")
|
Rel(grafana, loki, "Queries logs", "HTTP 3100")
|
||||||
Rel(grafana, tempo, "Queries traces", "HTTP 3200")
|
Rel(grafana, tempo, "Queries traces", "HTTP 3200")
|
||||||
|
Rel(grafana, db, "Read-only dashboard queries via grafana_reader role", "PostgreSQL / archiv-net")
|
||||||
Rel(glitchtip, db, "Stores error events in glitchtip DB", "PostgreSQL / archiv-net")
|
Rel(glitchtip, db, "Stores error events in glitchtip DB", "PostgreSQL / archiv-net")
|
||||||
Rel(obs_glitchtip_worker, obs_redis, "Processes Celery tasks", "Redis / obs-net")
|
Rel(obs_glitchtip_worker, obs_redis, "Processes Celery tasks", "Redis / obs-net")
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ System_Boundary(frontend, "Web Frontend (SvelteKit / SSR)") {
|
|||||||
Component(geschichten, "/geschichten and /geschichten/[id]", "SvelteKit Routes", "Story list and detail pages. Loader: GET /api/geschichten?status=PUBLISHED.")
|
Component(geschichten, "/geschichten and /geschichten/[id]", "SvelteKit Routes", "Story list and detail pages. Loader: GET /api/geschichten?status=PUBLISHED.")
|
||||||
Component(geschichtenEdit, "/geschichten/[id]/edit and /geschichten/new", "SvelteKit Routes", "Story editor with rich text, person and document linking. Actions: PUT/POST /api/geschichten. Requires BLOG_WRITE permission.")
|
Component(geschichtenEdit, "/geschichten/[id]/edit and /geschichten/new", "SvelteKit Routes", "Story editor with rich text, person and document linking. Actions: PUT/POST /api/geschichten. Requires BLOG_WRITE permission.")
|
||||||
Component(stammbaum, "/stammbaum", "SvelteKit Route", "Family tree visualisation. Loader: GET /api/network (nodes + edges). Renders interactive family tree from network graph data.")
|
Component(stammbaum, "/stammbaum", "SvelteKit Route", "Family tree visualisation. Loader: GET /api/network (nodes + edges). Renders interactive family tree from network graph data.")
|
||||||
|
Component(themen, "/themen", "SvelteKit Route", "Browsable topic index. Shows all root tags as cards with color bars and child rows. ThemenWidget also embedded in the home dashboard (reader + editor sidebar). Loader: GET /api/tags/tree.")
|
||||||
Component(profilePage, "/profile", "SvelteKit Route", "Current user profile settings. Loader: GET /api/users/me/notification-preferences. Actions: update name/password and notification preferences.")
|
Component(profilePage, "/profile", "SvelteKit Route", "Current user profile settings. Loader: GET /api/users/me/notification-preferences. Actions: update name/password and notification preferences.")
|
||||||
Component(userProfile, "/users/[id]", "SvelteKit Route", "Public user profile view. Loader: GET /api/users/{id}.")
|
Component(userProfile, "/users/[id]", "SvelteKit Route", "Public user profile view. Loader: GET /api/users/{id}.")
|
||||||
}
|
}
|
||||||
@@ -26,6 +27,7 @@ Rel(aktivitaeten, backend, "GET /api/dashboard/activity, GET /api/notifications"
|
|||||||
Rel(geschichten, backend, "GET /api/geschichten", "HTTP / JSON")
|
Rel(geschichten, backend, "GET /api/geschichten", "HTTP / JSON")
|
||||||
Rel(geschichtenEdit, backend, "GET/PUT/POST /api/geschichten", "HTTP / JSON")
|
Rel(geschichtenEdit, backend, "GET/PUT/POST /api/geschichten", "HTTP / JSON")
|
||||||
Rel(stammbaum, backend, "GET /api/network", "HTTP / JSON")
|
Rel(stammbaum, backend, "GET /api/network", "HTTP / JSON")
|
||||||
|
Rel(themen, backend, "GET /api/tags/tree", "HTTP / JSON")
|
||||||
Rel(profilePage, backend, "GET/PUT /api/users/me, notification-preferences", "HTTP / JSON")
|
Rel(profilePage, backend, "GET/PUT /api/users/me, notification-preferences", "HTTP / JSON")
|
||||||
Rel(userProfile, backend, "GET /api/users/{id}", "HTTP / JSON")
|
Rel(userProfile, backend, "GET /api/users/{id}", "HTTP / JSON")
|
||||||
|
|
||||||
|
|||||||
@@ -1084,5 +1084,10 @@
|
|||||||
"timeline_dragging_aria_live": "Zeitraum {from} bis {to} ausgewählt",
|
"timeline_dragging_aria_live": "Zeitraum {from} bis {to} ausgewählt",
|
||||||
"error_page_id_label": "Fehler-ID",
|
"error_page_id_label": "Fehler-ID",
|
||||||
"error_copy_id_label": "ID kopieren",
|
"error_copy_id_label": "ID kopieren",
|
||||||
"error_copied": "Kopiert!"
|
"error_copied": "Kopiert!",
|
||||||
|
"themen_widget_title": "Themen",
|
||||||
|
"themen_alle": "Alle Themen",
|
||||||
|
"themen_leer": "Noch keine Themen vergeben.",
|
||||||
|
"themen_weitere": "+ {count} weitere",
|
||||||
|
"themen_dokumente": "{count} Dokumente"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1084,5 +1084,10 @@
|
|||||||
"timeline_dragging_aria_live": "Range {from} to {to} selected",
|
"timeline_dragging_aria_live": "Range {from} to {to} selected",
|
||||||
"error_page_id_label": "Error ID",
|
"error_page_id_label": "Error ID",
|
||||||
"error_copy_id_label": "Copy ID",
|
"error_copy_id_label": "Copy ID",
|
||||||
"error_copied": "Copied!"
|
"error_copied": "Copied!",
|
||||||
|
"themen_widget_title": "Topics",
|
||||||
|
"themen_alle": "All Topics",
|
||||||
|
"themen_leer": "No topics assigned yet.",
|
||||||
|
"themen_weitere": "+ {count} more",
|
||||||
|
"themen_dokumente": "{count} documents"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1084,5 +1084,10 @@
|
|||||||
"timeline_dragging_aria_live": "Rango {from} a {to} seleccionado",
|
"timeline_dragging_aria_live": "Rango {from} a {to} seleccionado",
|
||||||
"error_page_id_label": "ID de error",
|
"error_page_id_label": "ID de error",
|
||||||
"error_copy_id_label": "Copiar ID",
|
"error_copy_id_label": "Copiar ID",
|
||||||
"error_copied": "¡Copiado!"
|
"error_copied": "¡Copiado!",
|
||||||
|
"themen_widget_title": "Temas",
|
||||||
|
"themen_alle": "Todos los temas",
|
||||||
|
"themen_leer": "Aún no hay temas.",
|
||||||
|
"themen_weitere": "+ {count} más",
|
||||||
|
"themen_dokumente": "{count} documentos"
|
||||||
}
|
}
|
||||||
|
|||||||
10
frontend/package-lock.json
generated
10
frontend/package-lock.json
generated
@@ -23,9 +23,9 @@
|
|||||||
"@eslint/compat": "^1.4.0",
|
"@eslint/compat": "^1.4.0",
|
||||||
"@eslint/js": "^9.39.1",
|
"@eslint/js": "^9.39.1",
|
||||||
"@inlang/paraglide-js": "^2.5.0",
|
"@inlang/paraglide-js": "^2.5.0",
|
||||||
"@playwright/test": "^1.58.2",
|
"@playwright/test": "^1.60.0",
|
||||||
"@sveltejs/adapter-node": "^5.4.0",
|
"@sveltejs/adapter-node": "^5.5.4",
|
||||||
"@sveltejs/kit": "^2.48.5",
|
"@sveltejs/kit": "^2.60.1",
|
||||||
"@sveltejs/vite-plugin-svelte": "^6.2.1",
|
"@sveltejs/vite-plugin-svelte": "^6.2.1",
|
||||||
"@tailwindcss/forms": "^0.5.10",
|
"@tailwindcss/forms": "^0.5.10",
|
||||||
"@tailwindcss/typography": "^0.5.19",
|
"@tailwindcss/typography": "^0.5.19",
|
||||||
@@ -43,7 +43,7 @@
|
|||||||
"globals": "^16.5.0",
|
"globals": "^16.5.0",
|
||||||
"openapi-typescript": "^7.8.0",
|
"openapi-typescript": "^7.8.0",
|
||||||
"patch-package": "^8.0.0",
|
"patch-package": "^8.0.0",
|
||||||
"playwright": "^1.56.1",
|
"playwright": "^1.60.0",
|
||||||
"prettier": "^3.6.2",
|
"prettier": "^3.6.2",
|
||||||
"prettier-plugin-svelte": "^3.4.0",
|
"prettier-plugin-svelte": "^3.4.0",
|
||||||
"prettier-plugin-tailwindcss": "^0.7.1",
|
"prettier-plugin-tailwindcss": "^0.7.1",
|
||||||
@@ -52,7 +52,7 @@
|
|||||||
"tailwindcss": "^4.1.17",
|
"tailwindcss": "^4.1.17",
|
||||||
"typescript": "^5.9.3",
|
"typescript": "^5.9.3",
|
||||||
"typescript-eslint": "^8.47.0",
|
"typescript-eslint": "^8.47.0",
|
||||||
"vite": "^7.2.2",
|
"vite": "^7.3.3",
|
||||||
"vite-plugin-devtools-json": "^1.0.0",
|
"vite-plugin-devtools-json": "^1.0.0",
|
||||||
"vitest": "^4.0.10",
|
"vitest": "^4.0.10",
|
||||||
"vitest-browser-svelte": "^2.0.1"
|
"vitest-browser-svelte": "^2.0.1"
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import { clickOutside } from '$lib/shared/actions/clickOutside';
|
|||||||
import { formatDate } from '$lib/shared/utils/date';
|
import { formatDate } from '$lib/shared/utils/date';
|
||||||
|
|
||||||
type Document = components['schemas']['Document'];
|
type Document = components['schemas']['Document'];
|
||||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
selectedDocuments?: Document[];
|
selectedDocuments?: Document[];
|
||||||
@@ -45,8 +45,12 @@ function handleInput() {
|
|||||||
try {
|
try {
|
||||||
const res = await fetch(`/api/documents/search?q=${encodeURIComponent(searchTerm)}&size=10`);
|
const res = await fetch(`/api/documents/search?q=${encodeURIComponent(searchTerm)}&size=10`);
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const body: { items: DocumentSearchItem[] } = await res.json();
|
const body: { items: DocumentListItem[] } = await res.json();
|
||||||
const docs = body.items.map((it) => it.document);
|
const docs = body.items.map((it) => ({
|
||||||
|
id: it.id,
|
||||||
|
title: it.title,
|
||||||
|
documentDate: it.documentDate
|
||||||
|
})) as unknown as Document[];
|
||||||
results = docs.filter((d) => !selectedDocuments.some((s) => s.id === d.id));
|
results = docs.filter((d) => !selectedDocuments.some((s) => s.id === d.id));
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
|
|||||||
@@ -10,7 +10,19 @@ const docFactory = (id: string, title: string, date = '1880-01-01') => ({
|
|||||||
title,
|
title,
|
||||||
documentDate: date,
|
documentDate: date,
|
||||||
originalFilename: `${title}.pdf`,
|
originalFilename: `${title}.pdf`,
|
||||||
status: 'UPLOADED',
|
receivers: [],
|
||||||
|
tags: [],
|
||||||
|
completionPercentage: 0,
|
||||||
|
contributors: [],
|
||||||
|
matchData: {
|
||||||
|
titleOffsets: [],
|
||||||
|
senderMatched: false,
|
||||||
|
matchedReceiverIds: [],
|
||||||
|
matchedTagIds: [],
|
||||||
|
snippetOffsets: [],
|
||||||
|
summaryOffsets: []
|
||||||
|
},
|
||||||
|
status: 'UPLOADED' as const,
|
||||||
metadataComplete: false,
|
metadataComplete: false,
|
||||||
scriptType: 'UNKNOWN' as const,
|
scriptType: 'UNKNOWN' as const,
|
||||||
createdAt: '2024-01-01T00:00:00',
|
createdAt: '2024-01-01T00:00:00',
|
||||||
@@ -22,7 +34,7 @@ function mockSearchResponse(items: ReturnType<typeof docFactory>[]) {
|
|||||||
'fetch',
|
'fetch',
|
||||||
vi.fn().mockResolvedValue({
|
vi.fn().mockResolvedValue({
|
||||||
ok: true,
|
ok: true,
|
||||||
json: vi.fn().mockResolvedValue({ items: items.map((document) => ({ document })) })
|
json: vi.fn().mockResolvedValue({ items })
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -91,10 +103,7 @@ describe('DocumentMultiSelect — search and select', () => {
|
|||||||
const fetchMock = vi.fn().mockResolvedValue({
|
const fetchMock = vi.fn().mockResolvedValue({
|
||||||
ok: true,
|
ok: true,
|
||||||
json: vi.fn().mockResolvedValue({
|
json: vi.fn().mockResolvedValue({
|
||||||
items: [
|
items: [docFactory('d1', 'Already attached'), docFactory('d2', 'Not attached')]
|
||||||
{ document: docFactory('d1', 'Already attached') },
|
|
||||||
{ document: docFactory('d2', 'Not attached') }
|
|
||||||
]
|
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
vi.stubGlobal('fetch', fetchMock);
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
|||||||
@@ -9,11 +9,11 @@ import ProgressRing from '$lib/shared/primitives/ProgressRing.svelte';
|
|||||||
import ContributorStack from '$lib/shared/primitives/ContributorStack.svelte';
|
import ContributorStack from '$lib/shared/primitives/ContributorStack.svelte';
|
||||||
import DocumentThumbnail from './DocumentThumbnail.svelte';
|
import DocumentThumbnail from './DocumentThumbnail.svelte';
|
||||||
|
|
||||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
|
|
||||||
let { item, canWrite = false }: { item: DocumentSearchItem; canWrite?: boolean } = $props();
|
let { item, canWrite = false }: { item: DocumentListItem; canWrite?: boolean } = $props();
|
||||||
|
|
||||||
const doc = $derived(item.document);
|
const doc = $derived(item);
|
||||||
const titleText = $derived(doc.title || doc.originalFilename);
|
const titleText = $derived(doc.title || doc.originalFilename);
|
||||||
const titleOffsets = $derived(item.matchData?.titleOffsets ?? []);
|
const titleOffsets = $derived(item.matchData?.titleOffsets ?? []);
|
||||||
const titleSegments = $derived(applyOffsets(titleText, titleOffsets));
|
const titleSegments = $derived(applyOffsets(titleText, titleOffsets));
|
||||||
|
|||||||
@@ -14,24 +14,17 @@ afterEach(() => {
|
|||||||
bulkSelectionStore.clear();
|
bulkSelectionStore.clear();
|
||||||
});
|
});
|
||||||
|
|
||||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
|
|
||||||
function makeItem(overrides: Partial<DocumentSearchItem> = {}): DocumentSearchItem {
|
function makeItem(overrides: Partial<DocumentListItem> = {}): DocumentListItem {
|
||||||
return {
|
return {
|
||||||
document: {
|
|
||||||
id: '1',
|
id: '1',
|
||||||
title: 'Testbrief',
|
title: 'Testbrief',
|
||||||
originalFilename: 'testbrief.pdf',
|
originalFilename: 'testbrief.pdf',
|
||||||
status: 'UPLOADED',
|
|
||||||
documentDate: '2024-03-15',
|
documentDate: '2024-03-15',
|
||||||
sender: null,
|
sender: undefined,
|
||||||
receivers: [],
|
receivers: [],
|
||||||
tags: [],
|
tags: [],
|
||||||
createdAt: '2024-01-01T00:00:00Z',
|
|
||||||
updatedAt: '2024-01-01T00:00:00Z',
|
|
||||||
metadataComplete: false,
|
|
||||||
scriptType: 'UNKNOWN'
|
|
||||||
},
|
|
||||||
matchData: {
|
matchData: {
|
||||||
titleOffsets: [],
|
titleOffsets: [],
|
||||||
senderMatched: false,
|
senderMatched: false,
|
||||||
@@ -55,14 +48,14 @@ describe('DocumentRow – title', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('falls back to originalFilename when title is null', async () => {
|
it('falls back to originalFilename when title is null', async () => {
|
||||||
const item = makeItem({ document: { ...makeItem().document, title: null } });
|
const item = makeItem({ title: null as unknown as string });
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
await expect.element(page.getByRole('heading', { name: 'testbrief.pdf' })).toBeInTheDocument();
|
await expect.element(page.getByRole('heading', { name: 'testbrief.pdf' })).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('renders a mark element for highlighted title offsets', async () => {
|
it('renders a mark element for highlighted title offsets', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: { ...makeItem().document, title: 'Brief an Anna' },
|
title: 'Brief an Anna',
|
||||||
matchData: {
|
matchData: {
|
||||||
titleOffsets: [{ start: 0, length: 5 }],
|
titleOffsets: [{ start: 0, length: 5 }],
|
||||||
senderMatched: false,
|
senderMatched: false,
|
||||||
@@ -109,9 +102,12 @@ describe('DocumentRow – snippet', () => {
|
|||||||
describe('DocumentRow – sender', () => {
|
describe('DocumentRow – sender', () => {
|
||||||
it('shows sender display name', async () => {
|
it('shows sender display name', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: {
|
sender: {
|
||||||
...makeItem().document,
|
id: 's1',
|
||||||
sender: { id: 's1', displayName: 'Großmutter Maria' }
|
lastName: 'Maria',
|
||||||
|
displayName: 'Großmutter Maria',
|
||||||
|
personType: 'PERSON',
|
||||||
|
familyMember: false
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
@@ -126,9 +122,12 @@ describe('DocumentRow – sender', () => {
|
|||||||
|
|
||||||
it('highlights the sender when senderMatched is true', async () => {
|
it('highlights the sender when senderMatched is true', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: {
|
sender: {
|
||||||
...makeItem().document,
|
id: 's1',
|
||||||
sender: { id: 's1', displayName: 'Großmutter Maria' }
|
lastName: 'Maria',
|
||||||
|
displayName: 'Großmutter Maria',
|
||||||
|
personType: 'PERSON',
|
||||||
|
familyMember: false
|
||||||
},
|
},
|
||||||
matchData: {
|
matchData: {
|
||||||
...makeItem().matchData,
|
...makeItem().matchData,
|
||||||
@@ -142,10 +141,15 @@ describe('DocumentRow – sender', () => {
|
|||||||
|
|
||||||
it('highlights a receiver when matchedReceiverIds includes its id', async () => {
|
it('highlights a receiver when matchedReceiverIds includes its id', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: {
|
receivers: [
|
||||||
...makeItem().document,
|
{
|
||||||
receivers: [{ id: 'r1', displayName: 'Onkel Karl' }]
|
id: 'r1',
|
||||||
},
|
lastName: 'Karl',
|
||||||
|
displayName: 'Onkel Karl',
|
||||||
|
personType: 'PERSON',
|
||||||
|
familyMember: false
|
||||||
|
}
|
||||||
|
],
|
||||||
matchData: {
|
matchData: {
|
||||||
...makeItem().matchData,
|
...makeItem().matchData,
|
||||||
matchedReceiverIds: ['r1']
|
matchedReceiverIds: ['r1']
|
||||||
@@ -162,10 +166,7 @@ describe('DocumentRow – sender', () => {
|
|||||||
describe('DocumentRow – summary', () => {
|
describe('DocumentRow – summary', () => {
|
||||||
it('renders the document summary when present', async () => {
|
it('renders the document summary when present', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: {
|
|
||||||
...makeItem().document,
|
|
||||||
summary: 'Brief von Eugenie über die Heimreise aus dem Süden.'
|
summary: 'Brief von Eugenie über die Heimreise aus dem Süden.'
|
||||||
}
|
|
||||||
});
|
});
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
await expect
|
await expect
|
||||||
@@ -180,7 +181,7 @@ describe('DocumentRow – summary', () => {
|
|||||||
|
|
||||||
it('applies summary search-match highlight via summaryOffsets', async () => {
|
it('applies summary search-match highlight via summaryOffsets', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: { ...makeItem().document, summary: 'Brief über Menton' },
|
summary: 'Brief über Menton',
|
||||||
matchData: {
|
matchData: {
|
||||||
...makeItem().matchData,
|
...makeItem().matchData,
|
||||||
summaryOffsets: [{ start: 11, length: 6 }]
|
summaryOffsets: [{ start: 11, length: 6 }]
|
||||||
@@ -196,25 +197,19 @@ describe('DocumentRow – summary', () => {
|
|||||||
|
|
||||||
describe('DocumentRow – archive chips', () => {
|
describe('DocumentRow – archive chips', () => {
|
||||||
it('renders the archive box chip when set', async () => {
|
it('renders the archive box chip when set', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({ archiveBox: 'K3' });
|
||||||
document: { ...makeItem().document, archiveBox: 'K3' }
|
|
||||||
});
|
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
await expect.element(page.getByText('K3')).toBeInTheDocument();
|
await expect.element(page.getByText('K3')).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('renders the archive folder chip when set', async () => {
|
it('renders the archive folder chip when set', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({ archiveFolder: 'Mappe A' });
|
||||||
document: { ...makeItem().document, archiveFolder: 'Mappe A' }
|
|
||||||
});
|
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
await expect.element(page.getByText('Mappe A')).toBeInTheDocument();
|
await expect.element(page.getByText('Mappe A')).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('renders the location chip when meta_location is set', async () => {
|
it('renders the location chip when meta_location is set', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({ location: 'Berlin' });
|
||||||
document: { ...makeItem().document, location: 'Berlin' }
|
|
||||||
});
|
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
await expect.element(page.getByText('Berlin')).toBeInTheDocument();
|
await expect.element(page.getByText('Berlin')).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
@@ -225,10 +220,7 @@ describe('DocumentRow – archive chips', () => {
|
|||||||
describe('DocumentRow – tags', () => {
|
describe('DocumentRow – tags', () => {
|
||||||
it('renders tag buttons', async () => {
|
it('renders tag buttons', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: {
|
tags: [{ id: 't1', name: 'Familie' }]
|
||||||
...makeItem().document,
|
|
||||||
tags: [{ id: 't1', name: 'Familie', color: null, parentId: null }]
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
await expect.element(page.getByRole('button', { name: 'Familie' })).toBeInTheDocument();
|
await expect.element(page.getByRole('button', { name: 'Familie' })).toBeInTheDocument();
|
||||||
@@ -236,10 +228,7 @@ describe('DocumentRow – tags', () => {
|
|||||||
|
|
||||||
it('navigates to /documents?tag=… on tag click', async () => {
|
it('navigates to /documents?tag=… on tag click', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: {
|
tags: [{ id: 't1', name: 'Urlaub & Reise' }]
|
||||||
...makeItem().document,
|
|
||||||
tags: [{ id: 't1', name: 'Urlaub & Reise', color: null, parentId: null }]
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
// Tailwind CSS isn't loaded in the vitest-browser client project, so the
|
// Tailwind CSS isn't loaded in the vitest-browser client project, so the
|
||||||
@@ -255,10 +244,7 @@ describe('DocumentRow – tags', () => {
|
|||||||
|
|
||||||
it('tag click does not navigate to the document detail page', async () => {
|
it('tag click does not navigate to the document detail page', async () => {
|
||||||
const item = makeItem({
|
const item = makeItem({
|
||||||
document: {
|
tags: [{ id: 't2', name: 'Familie' }]
|
||||||
...makeItem().document,
|
|
||||||
tags: [{ id: 't2', name: 'Familie', color: null, parentId: null }]
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
render(DocumentRow, { item });
|
render(DocumentRow, { item });
|
||||||
const before = window.location.href;
|
const before = window.location.href;
|
||||||
@@ -281,7 +267,7 @@ describe('DocumentRow – bulk selection checkbox', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('checkbox aria-label includes the document title', async () => {
|
it('checkbox aria-label includes the document title', async () => {
|
||||||
const item = makeItem({ document: { ...makeItem().document, title: 'Brief an Anna' } });
|
const item = makeItem({ title: 'Brief an Anna' });
|
||||||
render(DocumentRow, { item, canWrite: true });
|
render(DocumentRow, { item, canWrite: true });
|
||||||
await expect
|
await expect
|
||||||
.element(page.getByRole('checkbox', { name: /Brief an Anna/i }))
|
.element(page.getByRole('checkbox', { name: /Brief an Anna/i }))
|
||||||
@@ -289,7 +275,7 @@ describe('DocumentRow – bulk selection checkbox', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('toggling the checkbox calls bulkSelectionStore.toggle', async () => {
|
it('toggling the checkbox calls bulkSelectionStore.toggle', async () => {
|
||||||
const item = makeItem({ document: { ...makeItem().document, id: 'doc-42' } });
|
const item = makeItem({ id: 'doc-42' });
|
||||||
render(DocumentRow, { item, canWrite: true });
|
render(DocumentRow, { item, canWrite: true });
|
||||||
expect(bulkSelectionStore.has('doc-42')).toBe(false);
|
expect(bulkSelectionStore.has('doc-42')).toBe(false);
|
||||||
|
|
||||||
@@ -300,7 +286,7 @@ describe('DocumentRow – bulk selection checkbox', () => {
|
|||||||
|
|
||||||
it('checked state mirrors the store', async () => {
|
it('checked state mirrors the store', async () => {
|
||||||
bulkSelectionStore.add('doc-99');
|
bulkSelectionStore.add('doc-99');
|
||||||
const item = makeItem({ document: { ...makeItem().document, id: 'doc-99' } });
|
const item = makeItem({ id: 'doc-99' });
|
||||||
render(DocumentRow, { item, canWrite: true });
|
render(DocumentRow, { item, canWrite: true });
|
||||||
await expect.element(page.getByRole('checkbox')).toBeChecked();
|
await expect.element(page.getByRole('checkbox')).toBeChecked();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -20,10 +20,31 @@ const { default: DocumentRow } = await import('./DocumentRow.svelte');
|
|||||||
|
|
||||||
afterEach(cleanup);
|
afterEach(cleanup);
|
||||||
|
|
||||||
const sender = { id: 's1', displayName: 'Anna Schmidt' };
|
const sender = {
|
||||||
const receiver = { id: 'r1', displayName: 'Bert Meier' };
|
id: 's1',
|
||||||
|
lastName: 'Schmidt',
|
||||||
|
displayName: 'Anna Schmidt',
|
||||||
|
personType: 'PERSON' as const,
|
||||||
|
familyMember: false
|
||||||
|
};
|
||||||
|
const receiver = {
|
||||||
|
id: 'r1',
|
||||||
|
lastName: 'Meier',
|
||||||
|
displayName: 'Bert Meier',
|
||||||
|
personType: 'PERSON' as const,
|
||||||
|
familyMember: false
|
||||||
|
};
|
||||||
|
|
||||||
const makeDoc = (overrides: Record<string, unknown> = {}) => ({
|
const emptyMatchData = {
|
||||||
|
titleOffsets: [],
|
||||||
|
senderMatched: false,
|
||||||
|
matchedReceiverIds: [],
|
||||||
|
matchedTagIds: [],
|
||||||
|
snippetOffsets: [],
|
||||||
|
summaryOffsets: []
|
||||||
|
};
|
||||||
|
|
||||||
|
const baseItem = (overrides: Record<string, unknown> = {}) => ({
|
||||||
id: 'd1',
|
id: 'd1',
|
||||||
title: 'Brief 1923',
|
title: 'Brief 1923',
|
||||||
originalFilename: 'b.pdf',
|
originalFilename: 'b.pdf',
|
||||||
@@ -31,20 +52,14 @@ const makeDoc = (overrides: Record<string, unknown> = {}) => ({
|
|||||||
sender,
|
sender,
|
||||||
receivers: [receiver],
|
receivers: [receiver],
|
||||||
tags: [],
|
tags: [],
|
||||||
thumbnailUrl: null,
|
summary: undefined,
|
||||||
contentType: 'application/pdf',
|
archiveBox: undefined,
|
||||||
summary: null,
|
archiveFolder: undefined,
|
||||||
archiveBox: null,
|
location: undefined,
|
||||||
archiveFolder: null,
|
matchData: emptyMatchData,
|
||||||
location: null,
|
|
||||||
...overrides
|
|
||||||
});
|
|
||||||
|
|
||||||
const baseItem = (docOverrides: Record<string, unknown> = {}) => ({
|
|
||||||
document: makeDoc(docOverrides),
|
|
||||||
matchData: null,
|
|
||||||
completionPercentage: 0,
|
completionPercentage: 0,
|
||||||
contributors: []
|
contributors: [],
|
||||||
|
...overrides
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('DocumentRow', () => {
|
describe('DocumentRow', () => {
|
||||||
@@ -121,12 +136,9 @@ describe('DocumentRow', () => {
|
|||||||
it('renders the snippet when matchData provides a transcriptionSnippet', async () => {
|
it('renders the snippet when matchData provides a transcriptionSnippet', async () => {
|
||||||
render(DocumentRow, {
|
render(DocumentRow, {
|
||||||
props: {
|
props: {
|
||||||
item: {
|
item: baseItem({
|
||||||
document: makeDoc(),
|
matchData: { ...emptyMatchData, transcriptionSnippet: 'Hello world snippet' }
|
||||||
matchData: { transcriptionSnippet: 'Hello world snippet' },
|
})
|
||||||
completionPercentage: 50,
|
|
||||||
contributors: []
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -2068,12 +2068,20 @@ export interface components {
|
|||||||
};
|
};
|
||||||
ImportStatus: {
|
ImportStatus: {
|
||||||
/** @enum {string} */
|
/** @enum {string} */
|
||||||
state?: "IDLE" | "RUNNING" | "DONE" | "FAILED";
|
state: "IDLE" | "RUNNING" | "DONE" | "FAILED";
|
||||||
statusCode?: string;
|
statusCode: string;
|
||||||
/** Format: int32 */
|
/** Format: int32 */
|
||||||
processed?: number;
|
processed: number;
|
||||||
|
skippedFiles: components["schemas"]["SkippedFile"][];
|
||||||
/** Format: date-time */
|
/** Format: date-time */
|
||||||
startedAt?: string;
|
startedAt?: string;
|
||||||
|
/** Format: int32 */
|
||||||
|
skipped?: number;
|
||||||
|
};
|
||||||
|
SkippedFile: {
|
||||||
|
filename: string;
|
||||||
|
/** @enum {string} */
|
||||||
|
reason: "INVALID_FILENAME_PATH_TRAVERSAL" | "INVALID_PDF_SIGNATURE" | "FILE_READ_ERROR" | "ALREADY_EXISTS" | "S3_UPLOAD_FAILED";
|
||||||
};
|
};
|
||||||
BackfillStatus: {
|
BackfillStatus: {
|
||||||
/** @enum {string} */
|
/** @enum {string} */
|
||||||
@@ -2313,10 +2321,10 @@ export interface components {
|
|||||||
/** Format: int32 */
|
/** Format: int32 */
|
||||||
number?: number;
|
number?: number;
|
||||||
sort?: components["schemas"]["SortObject"];
|
sort?: components["schemas"]["SortObject"];
|
||||||
first?: boolean;
|
|
||||||
last?: boolean;
|
|
||||||
/** Format: int32 */
|
/** Format: int32 */
|
||||||
numberOfElements?: number;
|
numberOfElements?: number;
|
||||||
|
first?: boolean;
|
||||||
|
last?: boolean;
|
||||||
empty?: boolean;
|
empty?: boolean;
|
||||||
};
|
};
|
||||||
PageableObject: {
|
PageableObject: {
|
||||||
@@ -2380,15 +2388,32 @@ export interface components {
|
|||||||
/** Format: int32 */
|
/** Format: int32 */
|
||||||
totalPages?: number;
|
totalPages?: number;
|
||||||
};
|
};
|
||||||
DocumentSearchItem: {
|
DocumentListItem: {
|
||||||
document: components["schemas"]["Document"];
|
/** Format: uuid */
|
||||||
matchData: components["schemas"]["SearchMatchData"];
|
id: string;
|
||||||
|
title: string;
|
||||||
|
originalFilename: string;
|
||||||
|
thumbnailUrl?: string;
|
||||||
|
/** Format: date */
|
||||||
|
documentDate?: string;
|
||||||
|
sender?: components["schemas"]["Person"];
|
||||||
|
receivers: components["schemas"]["Person"][];
|
||||||
|
tags: components["schemas"]["Tag"][];
|
||||||
|
archiveBox?: string;
|
||||||
|
archiveFolder?: string;
|
||||||
|
location?: string;
|
||||||
|
summary?: string;
|
||||||
/** Format: int32 */
|
/** Format: int32 */
|
||||||
completionPercentage: number;
|
completionPercentage: number;
|
||||||
contributors: components["schemas"]["ActivityActorDTO"][];
|
contributors: components["schemas"]["ActivityActorDTO"][];
|
||||||
|
matchData: components["schemas"]["SearchMatchData"];
|
||||||
|
/** Format: date-time */
|
||||||
|
createdAt: string;
|
||||||
|
/** Format: date-time */
|
||||||
|
updatedAt: string;
|
||||||
};
|
};
|
||||||
DocumentSearchResult: {
|
DocumentSearchResult: {
|
||||||
items: components["schemas"]["DocumentSearchItem"][];
|
items: components["schemas"]["DocumentListItem"][];
|
||||||
/** Format: int64 */
|
/** Format: int64 */
|
||||||
totalElements: number;
|
totalElements: number;
|
||||||
/** Format: int32 */
|
/** Format: int32 */
|
||||||
|
|||||||
@@ -3,16 +3,16 @@ import * as m from '$lib/paraglide/messages.js';
|
|||||||
import { relativeTimeDe } from '$lib/shared/relativeTime';
|
import { relativeTimeDe } from '$lib/shared/relativeTime';
|
||||||
import type { components } from '$lib/generated/api';
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
type Document = components['schemas']['Document'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
|
|
||||||
interface Props {
|
interface Props {
|
||||||
documents: Document[];
|
documents: DocumentListItem[];
|
||||||
}
|
}
|
||||||
|
|
||||||
const { documents }: Props = $props();
|
const { documents }: Props = $props();
|
||||||
|
|
||||||
function isNew(doc: Document): boolean {
|
function isNew(doc: DocumentListItem): boolean {
|
||||||
return new Date(doc.createdAt).getTime() === new Date(doc.updatedAt).getTime();
|
return new Date(doc.createdAt).getTime() > Date.now() - 7 * 24 * 60 * 60 * 1000;
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -5,24 +5,33 @@ import { page } from 'vitest/browser';
|
|||||||
import ReaderRecentDocs from './ReaderRecentDocs.svelte';
|
import ReaderRecentDocs from './ReaderRecentDocs.svelte';
|
||||||
import type { components } from '$lib/generated/api';
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
type Document = components['schemas']['Document'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
|
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
cleanup();
|
cleanup();
|
||||||
});
|
});
|
||||||
|
|
||||||
const baseDoc: Document = {
|
const baseDoc: DocumentListItem = {
|
||||||
id: 'doc1',
|
id: 'doc1',
|
||||||
title: 'Brief an Hans',
|
title: 'Brief an Hans',
|
||||||
originalFilename: 'brief.pdf',
|
originalFilename: 'brief.pdf',
|
||||||
status: 'UPLOADED',
|
completionPercentage: 0,
|
||||||
metadataComplete: true,
|
receivers: [],
|
||||||
scriptType: 'HANDWRITING_KURRENT',
|
tags: [],
|
||||||
|
contributors: [],
|
||||||
|
matchData: {
|
||||||
|
titleOffsets: [],
|
||||||
|
senderMatched: false,
|
||||||
|
matchedReceiverIds: [],
|
||||||
|
matchedTagIds: [],
|
||||||
|
snippetOffsets: [],
|
||||||
|
summaryOffsets: []
|
||||||
|
},
|
||||||
createdAt: '2025-01-01T12:00:00Z',
|
createdAt: '2025-01-01T12:00:00Z',
|
||||||
updatedAt: '2025-01-01T12:00:00Z'
|
updatedAt: '2025-01-01T12:00:00Z'
|
||||||
};
|
};
|
||||||
|
|
||||||
const updatedDoc: Document = {
|
const updatedDoc: DocumentListItem = {
|
||||||
...baseDoc,
|
...baseDoc,
|
||||||
id: 'doc2',
|
id: 'doc2',
|
||||||
title: 'Urkunde 1920',
|
title: 'Urkunde 1920',
|
||||||
@@ -88,8 +97,14 @@ describe('ReaderRecentDocs', () => {
|
|||||||
expect(thumb!.className).toMatch(/rounded-/);
|
expect(thumb!.className).toMatch(/rounded-/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('shows "Neu" accent-pill badge when createdAt equals updatedAt', async () => {
|
it('shows "Neu" accent-pill badge when document was created within the last 7 days', async () => {
|
||||||
render(ReaderRecentDocs, { documents: [baseDoc] });
|
const recentDoc: DocumentListItem = {
|
||||||
|
...baseDoc,
|
||||||
|
id: 'doc-recent',
|
||||||
|
createdAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString(),
|
||||||
|
updatedAt: new Date(Date.now() - 1 * 24 * 60 * 60 * 1000).toISOString()
|
||||||
|
};
|
||||||
|
render(ReaderRecentDocs, { documents: [recentDoc] });
|
||||||
const badge = page.getByText(/^Neu$/i);
|
const badge = page.getByText(/^Neu$/i);
|
||||||
await expect.element(badge).toBeInTheDocument();
|
await expect.element(badge).toBeInTheDocument();
|
||||||
const cls = ((await badge.element()) as HTMLElement).className;
|
const cls = ((await badge.element()) as HTMLElement).className;
|
||||||
@@ -98,7 +113,7 @@ describe('ReaderRecentDocs', () => {
|
|||||||
expect(cls).toMatch(/\btext-ink\b/);
|
expect(cls).toMatch(/\btext-ink\b/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('shows no badge when updatedAt differs from createdAt', async () => {
|
it('shows no badge when document was created more than 7 days ago', async () => {
|
||||||
render(ReaderRecentDocs, { documents: [updatedDoc] });
|
render(ReaderRecentDocs, { documents: [updatedDoc] });
|
||||||
const badge = page.getByText(/^Neu$/i);
|
const badge = page.getByText(/^Neu$/i);
|
||||||
await expect.element(badge).not.toBeInTheDocument();
|
await expect.element(badge).not.toBeInTheDocument();
|
||||||
@@ -106,20 +121,20 @@ describe('ReaderRecentDocs', () => {
|
|||||||
await expect.element(updatedBadge).not.toBeInTheDocument();
|
await expect.element(updatedBadge).not.toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('shows "Neu" badge when createdAt and updatedAt represent the same instant in different ISO formats', async () => {
|
it('shows "Neu" badge when document was created 6 days ago', async () => {
|
||||||
const sameInstantDoc: Document = {
|
const almostOldDoc: DocumentListItem = {
|
||||||
...baseDoc,
|
...baseDoc,
|
||||||
id: 'doc-same-instant',
|
id: 'doc-almost-old',
|
||||||
createdAt: '2025-01-01T12:00:00Z',
|
createdAt: new Date(Date.now() - 6 * 24 * 60 * 60 * 1000).toISOString(),
|
||||||
updatedAt: '2025-01-01T12:00:00.000Z'
|
updatedAt: new Date(Date.now() - 5 * 24 * 60 * 60 * 1000).toISOString()
|
||||||
};
|
};
|
||||||
render(ReaderRecentDocs, { documents: [sameInstantDoc] });
|
render(ReaderRecentDocs, { documents: [almostOldDoc] });
|
||||||
const badge = page.getByText(/^Neu$/i);
|
const badge = page.getByText(/^Neu$/i);
|
||||||
await expect.element(badge).toBeInTheDocument();
|
await expect.element(badge).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('renders sender name text when sender is present', async () => {
|
it('renders sender name text when sender is present', async () => {
|
||||||
const docWithSender: Document = {
|
const docWithSender: DocumentListItem = {
|
||||||
...baseDoc,
|
...baseDoc,
|
||||||
sender: {
|
sender: {
|
||||||
id: 'p1',
|
id: 'p1',
|
||||||
|
|||||||
@@ -31,25 +31,25 @@ describe('ReaderRecentDocs', () => {
|
|||||||
.toHaveAttribute('href', '/documents');
|
.toHaveAttribute('href', '/documents');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('renders the New badge when createdAt equals updatedAt', async () => {
|
it('renders the New badge when document was created within the last 7 days', async () => {
|
||||||
|
const recentDate = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000).toISOString();
|
||||||
|
const laterUpdate = new Date(Date.now() - 1 * 24 * 60 * 60 * 1000).toISOString();
|
||||||
render(ReaderRecentDocs, {
|
render(ReaderRecentDocs, {
|
||||||
props: {
|
props: {
|
||||||
documents: [
|
documents: [makeDoc({ createdAt: recentDate, updatedAt: laterUpdate })]
|
||||||
makeDoc({ createdAt: '2026-04-15T10:00:00Z', updatedAt: '2026-04-15T10:00:00Z' })
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect.element(page.getByText('Neu')).toBeVisible();
|
await expect.element(page.getByText('Neu')).toBeVisible();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('hides the New badge when document was updated after creation', async () => {
|
it('hides the New badge when document was created more than 7 days ago', async () => {
|
||||||
render(ReaderRecentDocs, {
|
render(ReaderRecentDocs, {
|
||||||
props: {
|
props: {
|
||||||
documents: [
|
documents: [
|
||||||
makeDoc({
|
makeDoc({
|
||||||
createdAt: '2026-04-15T10:00:00Z',
|
createdAt: '2026-04-15T10:00:00Z',
|
||||||
updatedAt: '2026-04-15T11:00:00Z'
|
updatedAt: '2026-04-15T10:00:00Z'
|
||||||
})
|
})
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
64
frontend/src/lib/shared/dashboard/ThemenWidget.svelte
Normal file
64
frontend/src/lib/shared/dashboard/ThemenWidget.svelte
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import * as m from '$lib/paraglide/messages.js';
|
||||||
|
import type { components } from '$lib/generated/api';
|
||||||
|
import { hasAnyDocuments } from '$lib/shared/utils/tagUtils';
|
||||||
|
|
||||||
|
type TagTreeNodeDTO = components['schemas']['TagTreeNodeDTO'];
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
tags: TagTreeNodeDTO[];
|
||||||
|
compact?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { tags, compact = false }: Props = $props();
|
||||||
|
|
||||||
|
const visibleTags = $derived.by(() => tags.filter(hasAnyDocuments));
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<section class="rounded-sm border border-line bg-surface p-5 shadow-sm">
|
||||||
|
<div class="mb-4 flex items-center justify-between">
|
||||||
|
<h2 class="font-sans text-xs font-bold tracking-widest text-ink-3 uppercase">
|
||||||
|
{m.themen_widget_title()}
|
||||||
|
</h2>
|
||||||
|
<a
|
||||||
|
href="/themen"
|
||||||
|
class="font-sans text-xs text-brand-mint underline-offset-2 hover:underline focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:outline-none"
|
||||||
|
>
|
||||||
|
{m.themen_alle()} →
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if visibleTags.length === 0}
|
||||||
|
<p class="font-sans text-sm text-ink-3">{m.themen_leer()}</p>
|
||||||
|
{:else}
|
||||||
|
<div
|
||||||
|
class="grid gap-2 {compact ? 'grid-cols-1' : 'grid-cols-1 sm:grid-cols-2'}"
|
||||||
|
data-compact={compact}
|
||||||
|
>
|
||||||
|
{#each visibleTags as tag (tag.id)}
|
||||||
|
<a
|
||||||
|
href="/?tag={encodeURIComponent(tag.name)}"
|
||||||
|
aria-label="{tag.name}{tag.documentCount > 0
|
||||||
|
? ', ' + m.themen_dokumente({ count: tag.documentCount })
|
||||||
|
: ''}"
|
||||||
|
class="flex cursor-pointer items-stretch overflow-hidden rounded-sm border border-line bg-canvas hover:bg-surface focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:outline-none"
|
||||||
|
style="min-height: 56px"
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
class="w-1 flex-shrink-0 self-stretch"
|
||||||
|
aria-hidden="true"
|
||||||
|
style="background: var(--c-tag-{tag.color ?? 'slate'})"
|
||||||
|
></span>
|
||||||
|
<span class="flex min-w-0 flex-1 flex-col justify-center gap-0.5 px-3 py-3">
|
||||||
|
<span class="truncate font-serif text-sm font-semibold text-ink">{tag.name}</span>
|
||||||
|
{#if tag.documentCount > 0}
|
||||||
|
<span class="font-sans text-xs text-ink-3 tabular-nums">
|
||||||
|
{m.themen_dokumente({ count: tag.documentCount })}
|
||||||
|
</span>
|
||||||
|
{/if}
|
||||||
|
</span>
|
||||||
|
</a>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</section>
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import { describe, it, expect, afterEach } from 'vitest';
|
||||||
|
import { cleanup, render } from 'vitest-browser-svelte';
|
||||||
|
import ThemenWidget from './ThemenWidget.svelte';
|
||||||
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
type TagTreeNodeDTO = components['schemas']['TagTreeNodeDTO'];
|
||||||
|
|
||||||
|
function makeTag(
|
||||||
|
name: string,
|
||||||
|
documentCount: number,
|
||||||
|
children: TagTreeNodeDTO[] = []
|
||||||
|
): TagTreeNodeDTO {
|
||||||
|
return { id: 'id-' + name, name, documentCount, children };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('ThemenWidget', () => {
|
||||||
|
it('renders a card link per visible tag', async () => {
|
||||||
|
const tags = [makeTag('Briefe', 5), makeTag('Fotos', 3)];
|
||||||
|
const { getByRole } = render(ThemenWidget, { tags });
|
||||||
|
await expect.element(getByRole('link', { name: /Briefe/ })).toBeInTheDocument();
|
||||||
|
await expect.element(getByRole('link', { name: /Fotos/ })).toBeInTheDocument();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('hides tags where no document exists in the subtree', async () => {
|
||||||
|
const tags = [makeTag('Briefe', 5), makeTag('Leer', 0)];
|
||||||
|
render(ThemenWidget, { tags });
|
||||||
|
expect(document.body.textContent).toContain('Briefe');
|
||||||
|
expect(document.body.textContent).not.toContain('Leer');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shows the empty state text when all tags are filtered out', async () => {
|
||||||
|
render(ThemenWidget, { tags: [makeTag('Leer', 0)] });
|
||||||
|
expect(document.body.textContent).toMatch(/Noch keine Themen/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shows empty state when tags array is empty', async () => {
|
||||||
|
render(ThemenWidget, { tags: [] });
|
||||||
|
expect(document.body.textContent).toMatch(/Noch keine Themen/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('renders in compact single-column mode when compact prop is true', async () => {
|
||||||
|
const tags = [makeTag('Briefe', 5)];
|
||||||
|
const { container } = render(ThemenWidget, { tags, compact: true });
|
||||||
|
const grid = container.querySelector('[data-compact="true"]');
|
||||||
|
expect(grid).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('links to "Alle Themen" page', async () => {
|
||||||
|
const tags = [makeTag('Briefe', 5)];
|
||||||
|
const { getByRole } = render(ThemenWidget, { tags });
|
||||||
|
const link = getByRole('link', { name: /Alle Themen/ });
|
||||||
|
await expect.element(link).toHaveAttribute('href', '/themen');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -409,19 +409,24 @@ describe('PersonMentionEditor — onExit cancels pending debounce', () => {
|
|||||||
await new Promise((r) => setTimeout(r, SEARCH_DEBOUNCE_MS + POST_DEBOUNCE_SLACK_MS));
|
await new Promise((r) => setTimeout(r, SEARCH_DEBOUNCE_MS + POST_DEBOUNCE_SLACK_MS));
|
||||||
const fetchesBeforeEscape = fetchMock.mock.calls.length;
|
const fetchesBeforeEscape = fetchMock.mock.calls.length;
|
||||||
|
|
||||||
// Trigger a new debounced search (queues runSearch after 150 ms), then
|
// Freeze setTimeout so the 150 ms debounce cannot fire before Escape
|
||||||
// immediately Escape *while focus is back in the editor* so Tiptap's
|
// triggers onExit. We install fake timers only now — after the setup
|
||||||
// suggestion-plugin Escape handler fires onExit before the debounce.
|
// above — so that vi.waitFor()'s real-timer polling still worked.
|
||||||
// Without onExit cancelling the pending debounce, runSearch executes
|
vi.useFakeTimers();
|
||||||
// against the now-unmounted dropdown's state.
|
try {
|
||||||
|
// fill() dispatches the input event synchronously via CDP; by the
|
||||||
|
// time the await resolves, onSearch('Walter') has run and the fake
|
||||||
|
// debounce timer is set.
|
||||||
await page.getByRole('searchbox').fill('Walter');
|
await page.getByRole('searchbox').fill('Walter');
|
||||||
// Focus the editor so the Escape lands on Tiptap's suggestion handler.
|
// Focus the editor so the Escape lands on Tiptap's suggestion handler.
|
||||||
(page.getByRole('textbox').element() as HTMLElement).focus();
|
(page.getByRole('textbox').element() as HTMLElement).focus();
|
||||||
await userEvent.keyboard('{Escape}');
|
await userEvent.keyboard('{Escape}');
|
||||||
|
// onExit has now called debouncedSearch.cancel(). Advance past the
|
||||||
// Wait past the debounce window. If onExit did not cancel the pending
|
// debounce window — the cancelled timer must not fire.
|
||||||
// debounce, a fetch with q=Walter would still fire here.
|
await vi.advanceTimersByTimeAsync(SEARCH_DEBOUNCE_MS + POST_DEBOUNCE_SLACK_MS);
|
||||||
await new Promise((r) => setTimeout(r, SEARCH_DEBOUNCE_MS + POST_DEBOUNCE_SLACK_MS));
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
|
||||||
const newFetches = fetchMock.mock.calls.slice(fetchesBeforeEscape);
|
const newFetches = fetchMock.mock.calls.slice(fetchesBeforeEscape);
|
||||||
const walterFetches = newFetches.filter(
|
const walterFetches = newFetches.filter(
|
||||||
|
|||||||
29
frontend/src/lib/shared/utils/tagUtils.test.ts
Normal file
29
frontend/src/lib/shared/utils/tagUtils.test.ts
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import { describe, it, expect } from 'vitest';
|
||||||
|
import { hasAnyDocuments } from './tagUtils';
|
||||||
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
|
type TagTreeNodeDTO = components['schemas']['TagTreeNodeDTO'];
|
||||||
|
|
||||||
|
function makeNode(documentCount: number, children: TagTreeNodeDTO[] = []): TagTreeNodeDTO {
|
||||||
|
return { id: 'id', name: 'name', documentCount, children };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('hasAnyDocuments', () => {
|
||||||
|
it('returns false for a leaf node with documentCount=0', () => {
|
||||||
|
expect(hasAnyDocuments(makeNode(0))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns true for a leaf node with documentCount=3', () => {
|
||||||
|
expect(hasAnyDocuments(makeNode(3))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns true for a root with documentCount=0 but a child with documentCount=5', () => {
|
||||||
|
const node = makeNode(0, [makeNode(5)]);
|
||||||
|
expect(hasAnyDocuments(node)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for a root with documentCount=0 and all children also 0', () => {
|
||||||
|
const node = makeNode(0, [makeNode(0), makeNode(0)]);
|
||||||
|
expect(hasAnyDocuments(node)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
7
frontend/src/lib/shared/utils/tagUtils.ts
Normal file
7
frontend/src/lib/shared/utils/tagUtils.ts
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
|
type TagTreeNodeDTO = components['schemas']['TagTreeNodeDTO'];
|
||||||
|
|
||||||
|
export function hasAnyDocuments(node: TagTreeNodeDTO): boolean {
|
||||||
|
return (node.documentCount ?? 0) > 0 || (node.children ?? []).some(hasAnyDocuments);
|
||||||
|
}
|
||||||
@@ -10,8 +10,9 @@ type DashboardPulseDTO = components['schemas']['DashboardPulseDTO'];
|
|||||||
type ActivityFeedItemDTO = components['schemas']['ActivityFeedItemDTO'];
|
type ActivityFeedItemDTO = components['schemas']['ActivityFeedItemDTO'];
|
||||||
type IncompleteDocumentDTO = components['schemas']['IncompleteDocumentDTO'];
|
type IncompleteDocumentDTO = components['schemas']['IncompleteDocumentDTO'];
|
||||||
type PersonSummaryDTO = components['schemas']['PersonSummaryDTO'];
|
type PersonSummaryDTO = components['schemas']['PersonSummaryDTO'];
|
||||||
type Document = components['schemas']['Document'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
type Geschichte = components['schemas']['Geschichte'];
|
type Geschichte = components['schemas']['Geschichte'];
|
||||||
|
type TagTreeNodeDTO = components['schemas']['TagTreeNodeDTO'];
|
||||||
|
|
||||||
function settled<T>(res: PromiseSettledResult<unknown> | undefined): T | null {
|
function settled<T>(res: PromiseSettledResult<unknown> | undefined): T | null {
|
||||||
if (res?.status !== 'fulfilled') return null;
|
if (res?.status !== 'fulfilled') return null;
|
||||||
@@ -40,7 +41,8 @@ export async function load({ fetch, parent }) {
|
|||||||
api.GET('/api/documents/search', {
|
api.GET('/api/documents/search', {
|
||||||
params: { query: { sort: 'UPDATED_AT', dir: 'DESC', size: 5 } }
|
params: { query: { sort: 'UPDATED_AT', dir: 'DESC', size: 5 } }
|
||||||
}),
|
}),
|
||||||
api.GET('/api/geschichten', { params: { query: { status: 'PUBLISHED', limit: 3 } } })
|
api.GET('/api/geschichten', { params: { query: { status: 'PUBLISHED', limit: 3 } } }),
|
||||||
|
api.GET('/api/tags/tree')
|
||||||
];
|
];
|
||||||
if (canBlogWrite) {
|
if (canBlogWrite) {
|
||||||
readerFetches.push(
|
readerFetches.push(
|
||||||
@@ -48,14 +50,15 @@ export async function load({ fetch, parent }) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const [statsRes, topPersonsRes, recentDocsRes, recentStoriesRes, draftsRes] =
|
const [statsRes, topPersonsRes, recentDocsRes, recentStoriesRes, tagTreeRes, draftsRes] =
|
||||||
await Promise.allSettled(readerFetches);
|
await Promise.allSettled(readerFetches);
|
||||||
|
|
||||||
const readerStats = settled<StatsDTO>(statsRes);
|
const readerStats = settled<StatsDTO>(statsRes);
|
||||||
const topPersons = settled<PersonSummaryDTO[]>(topPersonsRes) ?? [];
|
const topPersons = settled<PersonSummaryDTO[]>(topPersonsRes) ?? [];
|
||||||
const searchData = settled<{ items: { document: Document }[] }>(recentDocsRes);
|
const searchData = settled<{ items: DocumentListItem[] }>(recentDocsRes);
|
||||||
const recentDocs = searchData?.items.map((i) => i.document) ?? [];
|
const recentDocs = searchData?.items ?? [];
|
||||||
const recentStories = settled<Geschichte[]>(recentStoriesRes) ?? [];
|
const recentStories = settled<Geschichte[]>(recentStoriesRes) ?? [];
|
||||||
|
const tagTree = settled<TagTreeNodeDTO[]>(tagTreeRes) ?? [];
|
||||||
const drafts = settled<Geschichte[]>(draftsRes) ?? [];
|
const drafts = settled<Geschichte[]>(draftsRes) ?? [];
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -65,6 +68,7 @@ export async function load({ fetch, parent }) {
|
|||||||
topPersons,
|
topPersons,
|
||||||
recentDocs,
|
recentDocs,
|
||||||
recentStories,
|
recentStories,
|
||||||
|
tagTree,
|
||||||
drafts,
|
drafts,
|
||||||
error: null as string | null
|
error: null as string | null
|
||||||
};
|
};
|
||||||
@@ -80,7 +84,8 @@ export async function load({ fetch, parent }) {
|
|||||||
readyResult,
|
readyResult,
|
||||||
weeklyStatsResult,
|
weeklyStatsResult,
|
||||||
incompleteResult,
|
incompleteResult,
|
||||||
incompleteCountResult
|
incompleteCountResult,
|
||||||
|
tagTreeResult
|
||||||
] = await Promise.allSettled([
|
] = await Promise.allSettled([
|
||||||
api.GET('/api/stats'),
|
api.GET('/api/stats'),
|
||||||
api.GET('/api/dashboard/resume'),
|
api.GET('/api/dashboard/resume'),
|
||||||
@@ -91,7 +96,8 @@ export async function load({ fetch, parent }) {
|
|||||||
api.GET('/api/transcription/ready-to-read'),
|
api.GET('/api/transcription/ready-to-read'),
|
||||||
api.GET('/api/transcription/weekly-stats'),
|
api.GET('/api/transcription/weekly-stats'),
|
||||||
api.GET('/api/documents/incomplete', { params: { query: { size: 5 } } }),
|
api.GET('/api/documents/incomplete', { params: { query: { size: 5 } } }),
|
||||||
api.GET('/api/documents/incomplete-count')
|
api.GET('/api/documents/incomplete-count'),
|
||||||
|
api.GET('/api/tags/tree')
|
||||||
]);
|
]);
|
||||||
|
|
||||||
let stats: StatsDTO | null = null;
|
let stats: StatsDTO | null = null;
|
||||||
@@ -104,6 +110,7 @@ export async function load({ fetch, parent }) {
|
|||||||
let weeklyStats: TranscriptionWeeklyStatsDTO | null = null;
|
let weeklyStats: TranscriptionWeeklyStatsDTO | null = null;
|
||||||
let incompleteDocs: IncompleteDocumentDTO[] = [];
|
let incompleteDocs: IncompleteDocumentDTO[] = [];
|
||||||
let incompleteTotal = 0;
|
let incompleteTotal = 0;
|
||||||
|
let tagTree: TagTreeNodeDTO[] = [];
|
||||||
|
|
||||||
if (statsResult.status === 'fulfilled' && statsResult.value.response.ok) {
|
if (statsResult.status === 'fulfilled' && statsResult.value.response.ok) {
|
||||||
stats = statsResult.value.data ?? null;
|
stats = statsResult.value.data ?? null;
|
||||||
@@ -135,6 +142,9 @@ export async function load({ fetch, parent }) {
|
|||||||
if (incompleteCountResult.status === 'fulfilled' && incompleteCountResult.value.response.ok) {
|
if (incompleteCountResult.status === 'fulfilled' && incompleteCountResult.value.response.ok) {
|
||||||
incompleteTotal = (incompleteCountResult.value.data?.count as number | undefined) ?? 0;
|
incompleteTotal = (incompleteCountResult.value.data?.count as number | undefined) ?? 0;
|
||||||
}
|
}
|
||||||
|
if (tagTreeResult.status === 'fulfilled' && tagTreeResult.value.response.ok) {
|
||||||
|
tagTree = (tagTreeResult.value.data as TagTreeNodeDTO[]) ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
isReader: false as const,
|
isReader: false as const,
|
||||||
@@ -148,6 +158,7 @@ export async function load({ fetch, parent }) {
|
|||||||
weeklyStats,
|
weeklyStats,
|
||||||
incompleteDocs,
|
incompleteDocs,
|
||||||
incompleteTotal,
|
incompleteTotal,
|
||||||
|
tagTree,
|
||||||
error: null as string | null
|
error: null as string | null
|
||||||
};
|
};
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -167,8 +178,9 @@ export async function load({ fetch, parent }) {
|
|||||||
incompleteTotal: 0,
|
incompleteTotal: 0,
|
||||||
readerStats: null,
|
readerStats: null,
|
||||||
topPersons: [] as PersonSummaryDTO[],
|
topPersons: [] as PersonSummaryDTO[],
|
||||||
recentDocs: [] as Document[],
|
recentDocs: [] as DocumentListItem[],
|
||||||
recentStories: [] as Geschichte[],
|
recentStories: [] as Geschichte[],
|
||||||
|
tagTree: [] as TagTreeNodeDTO[],
|
||||||
drafts: [] as Geschichte[],
|
drafts: [] as Geschichte[],
|
||||||
error: 'Daten konnten nicht geladen werden.' as string | null
|
error: 'Daten konnten nicht geladen werden.' as string | null
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import ReaderPersonChips from '$lib/shared/dashboard/ReaderPersonChips.svelte';
|
|||||||
import ReaderDraftsModule from '$lib/shared/dashboard/ReaderDraftsModule.svelte';
|
import ReaderDraftsModule from '$lib/shared/dashboard/ReaderDraftsModule.svelte';
|
||||||
import ReaderRecentDocs from '$lib/shared/dashboard/ReaderRecentDocs.svelte';
|
import ReaderRecentDocs from '$lib/shared/dashboard/ReaderRecentDocs.svelte';
|
||||||
import ReaderRecentStories from '$lib/shared/dashboard/ReaderRecentStories.svelte';
|
import ReaderRecentStories from '$lib/shared/dashboard/ReaderRecentStories.svelte';
|
||||||
|
import ThemenWidget from '$lib/shared/dashboard/ThemenWidget.svelte';
|
||||||
import { m } from '$lib/paraglide/messages.js';
|
import { m } from '$lib/paraglide/messages.js';
|
||||||
|
|
||||||
let { data } = $props();
|
let { data } = $props();
|
||||||
@@ -45,6 +46,8 @@ const greetingText = $derived.by(() => {
|
|||||||
|
|
||||||
<ReaderPersonChips persons={data.topPersons ?? []} />
|
<ReaderPersonChips persons={data.topPersons ?? []} />
|
||||||
|
|
||||||
|
<ThemenWidget tags={data.tagTree ?? []} />
|
||||||
|
|
||||||
<div class="grid grid-cols-1 gap-1.5 sm:grid-cols-2">
|
<div class="grid grid-cols-1 gap-1.5 sm:grid-cols-2">
|
||||||
<ReaderRecentDocs documents={data.recentDocs ?? []} />
|
<ReaderRecentDocs documents={data.recentDocs ?? []} />
|
||||||
<ReaderRecentStories stories={data.recentStories ?? []} />
|
<ReaderRecentStories stories={data.recentStories ?? []} />
|
||||||
@@ -82,6 +85,7 @@ const greetingText = $derived.by(() => {
|
|||||||
|
|
||||||
<div class="flex flex-col gap-5 lg:sticky lg:top-[80px]">
|
<div class="flex flex-col gap-5 lg:sticky lg:top-[80px]">
|
||||||
<DashboardFamilyPulse pulse={data.pulse ?? null} />
|
<DashboardFamilyPulse pulse={data.pulse ?? null} />
|
||||||
|
<ThemenWidget tags={data.tagTree ?? []} compact={true} />
|
||||||
<DashboardActivityFeed feed={data.activityFeed ?? []} />
|
<DashboardActivityFeed feed={data.activityFeed ?? []} />
|
||||||
{#if data.canWrite}
|
{#if data.canWrite}
|
||||||
<DropZone onUploadComplete={(count) => (bannerCount = count)} />
|
<DropZone onUploadComplete={(count) => (bannerCount = count)} />
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import DocumentRow from '$lib/document/DocumentRow.svelte';
|
|||||||
import { SvelteMap } from 'svelte/reactivity';
|
import { SvelteMap } from 'svelte/reactivity';
|
||||||
import type { components } from '$lib/generated/api';
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
|
|
||||||
type SortMode = 'DATE' | 'TITLE' | 'SENDER' | 'RECEIVER' | 'UPLOAD_DATE' | 'RELEVANCE';
|
type SortMode = 'DATE' | 'TITLE' | 'SENDER' | 'RECEIVER' | 'UPLOAD_DATE' | 'RELEVANCE';
|
||||||
|
|
||||||
@@ -17,7 +17,7 @@ let {
|
|||||||
q = '',
|
q = '',
|
||||||
sort = 'DATE'
|
sort = 'DATE'
|
||||||
}: {
|
}: {
|
||||||
items: DocumentSearchItem[];
|
items: DocumentListItem[];
|
||||||
canWrite: boolean;
|
canWrite: boolean;
|
||||||
error?: string | null;
|
error?: string | null;
|
||||||
total?: number;
|
total?: number;
|
||||||
@@ -31,10 +31,10 @@ const groups = $derived.by(() => {
|
|||||||
return groupByYear(items);
|
return groupByYear(items);
|
||||||
});
|
});
|
||||||
|
|
||||||
function groupByYear(docItems: DocumentSearchItem[]) {
|
function groupByYear(docItems: DocumentListItem[]) {
|
||||||
const map = new SvelteMap<string, DocumentSearchItem[]>();
|
const map = new SvelteMap<string, DocumentListItem[]>();
|
||||||
for (const item of docItems) {
|
for (const item of docItems) {
|
||||||
const label = item.document.documentDate?.substring(0, 4) ?? m.docs_group_undated();
|
const label = item.documentDate?.substring(0, 4) ?? m.docs_group_undated();
|
||||||
const bucket = map.get(label);
|
const bucket = map.get(label);
|
||||||
if (bucket) bucket.push(item);
|
if (bucket) bucket.push(item);
|
||||||
else map.set(label, [item]);
|
else map.set(label, [item]);
|
||||||
@@ -42,10 +42,10 @@ function groupByYear(docItems: DocumentSearchItem[]) {
|
|||||||
return Array.from(map.entries()).map(([label, groupItems]) => ({ label, items: groupItems }));
|
return Array.from(map.entries()).map(([label, groupItems]) => ({ label, items: groupItems }));
|
||||||
}
|
}
|
||||||
|
|
||||||
function groupBySender(docItems: DocumentSearchItem[]) {
|
function groupBySender(docItems: DocumentListItem[]) {
|
||||||
const map = new SvelteMap<string, DocumentSearchItem[]>();
|
const map = new SvelteMap<string, DocumentListItem[]>();
|
||||||
for (const item of docItems) {
|
for (const item of docItems) {
|
||||||
const label = item.document.sender?.displayName ?? m.docs_group_unknown_sender();
|
const label = item.sender?.displayName ?? m.docs_group_unknown_sender();
|
||||||
const bucket = map.get(label);
|
const bucket = map.get(label);
|
||||||
if (bucket) bucket.push(item);
|
if (bucket) bucket.push(item);
|
||||||
else map.set(label, [item]);
|
else map.set(label, [item]);
|
||||||
@@ -53,10 +53,10 @@ function groupBySender(docItems: DocumentSearchItem[]) {
|
|||||||
return Array.from(map.entries()).map(([label, groupItems]) => ({ label, items: groupItems }));
|
return Array.from(map.entries()).map(([label, groupItems]) => ({ label, items: groupItems }));
|
||||||
}
|
}
|
||||||
|
|
||||||
function groupByReceiver(docItems: DocumentSearchItem[]) {
|
function groupByReceiver(docItems: DocumentListItem[]) {
|
||||||
const map = new SvelteMap<string, DocumentSearchItem[]>();
|
const map = new SvelteMap<string, DocumentListItem[]>();
|
||||||
for (const item of docItems) {
|
for (const item of docItems) {
|
||||||
const receivers = item.document.receivers ?? [];
|
const receivers = item.receivers ?? [];
|
||||||
const labels =
|
const labels =
|
||||||
receivers.length > 0
|
receivers.length > 0
|
||||||
? receivers.map((r) => r.displayName)
|
? receivers.map((r) => r.displayName)
|
||||||
@@ -99,7 +99,7 @@ function groupByReceiver(docItems: DocumentSearchItem[]) {
|
|||||||
>
|
>
|
||||||
</div>
|
</div>
|
||||||
<ul class="divide-y divide-line">
|
<ul class="divide-y divide-line">
|
||||||
{#each group.items as item (group.label + '-' + item.document.id)}
|
{#each group.items as item (group.label + '-' + item.id)}
|
||||||
<DocumentRow item={item} canWrite={canWrite} />
|
<DocumentRow item={item} canWrite={canWrite} />
|
||||||
{/each}
|
{/each}
|
||||||
</ul>
|
</ul>
|
||||||
|
|||||||
@@ -8,24 +8,17 @@ vi.mock('$app/navigation', () => ({ goto: vi.fn() }));
|
|||||||
|
|
||||||
afterEach(() => cleanup());
|
afterEach(() => cleanup());
|
||||||
|
|
||||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
|
|
||||||
function makeItem(overrides: Partial<DocumentSearchItem> = {}): DocumentSearchItem {
|
function makeItem(overrides: Partial<DocumentListItem> = {}): DocumentListItem {
|
||||||
return {
|
return {
|
||||||
document: {
|
|
||||||
id: '1',
|
id: '1',
|
||||||
title: 'Testbrief',
|
title: 'Testbrief',
|
||||||
originalFilename: 'testbrief.pdf',
|
originalFilename: 'testbrief.pdf',
|
||||||
status: 'UPLOADED',
|
|
||||||
documentDate: '2024-03-15',
|
documentDate: '2024-03-15',
|
||||||
sender: undefined,
|
sender: undefined,
|
||||||
receivers: [],
|
receivers: [],
|
||||||
tags: [],
|
tags: [],
|
||||||
createdAt: '2024-01-01T00:00:00Z',
|
|
||||||
updatedAt: '2024-01-01T00:00:00Z',
|
|
||||||
metadataComplete: false,
|
|
||||||
scriptType: 'UNKNOWN'
|
|
||||||
},
|
|
||||||
matchData: {
|
matchData: {
|
||||||
titleOffsets: [],
|
titleOffsets: [],
|
||||||
senderMatched: false,
|
senderMatched: false,
|
||||||
@@ -75,8 +68,8 @@ describe('DocumentList – empty state', () => {
|
|||||||
describe('DocumentList – year grouping', () => {
|
describe('DocumentList – year grouping', () => {
|
||||||
it('groups documents by year into separate cards', async () => {
|
it('groups documents by year into separate cards', async () => {
|
||||||
const items = [
|
const items = [
|
||||||
makeItem({ document: { ...makeItem().document, id: '1', documentDate: '1923-04-12' } }),
|
makeItem({ id: '1', documentDate: '1923-04-12' }),
|
||||||
makeItem({ document: { ...makeItem().document, id: '2', documentDate: '1965-08-03' } })
|
makeItem({ id: '2', documentDate: '1965-08-03' })
|
||||||
];
|
];
|
||||||
render(DocumentList, { ...baseProps, items, total: 2 });
|
render(DocumentList, { ...baseProps, items, total: 2 });
|
||||||
const groupCards = page.getByTestId('group-card');
|
const groupCards = page.getByTestId('group-card');
|
||||||
@@ -85,17 +78,15 @@ describe('DocumentList – year grouping', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('uses undated label for items with no documentDate', async () => {
|
it('uses undated label for items with no documentDate', async () => {
|
||||||
const items = [
|
const items = [makeItem({ id: '1', documentDate: undefined })];
|
||||||
makeItem({ document: { ...makeItem().document, id: '1', documentDate: undefined } })
|
|
||||||
];
|
|
||||||
render(DocumentList, { ...baseProps, items, total: 1 });
|
render(DocumentList, { ...baseProps, items, total: 1 });
|
||||||
await expect.element(page.getByText('Undatiert')).toBeInTheDocument();
|
await expect.element(page.getByText('Undatiert')).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('single year renders one group-card', async () => {
|
it('single year renders one group-card', async () => {
|
||||||
const items = [
|
const items = [
|
||||||
makeItem({ document: { ...makeItem().document, id: '1', documentDate: '1938-01-01' } }),
|
makeItem({ id: '1', documentDate: '1938-01-01' }),
|
||||||
makeItem({ document: { ...makeItem().document, id: '2', documentDate: '1938-06-15' } })
|
makeItem({ id: '2', documentDate: '1938-06-15' })
|
||||||
];
|
];
|
||||||
render(DocumentList, { ...baseProps, items, total: 2 });
|
render(DocumentList, { ...baseProps, items, total: 2 });
|
||||||
const groupCards = page.getByTestId('group-card');
|
const groupCards = page.getByTestId('group-card');
|
||||||
@@ -108,9 +99,7 @@ describe('DocumentList – year grouping', () => {
|
|||||||
|
|
||||||
describe('DocumentList – sort fallback', () => {
|
describe('DocumentList – sort fallback', () => {
|
||||||
it('falls back to year grouping when sort is not SENDER or RECEIVER', async () => {
|
it('falls back to year grouping when sort is not SENDER or RECEIVER', async () => {
|
||||||
const items = [
|
const items = [makeItem({ id: '1', documentDate: '2024-03-15' })];
|
||||||
makeItem({ document: { ...makeItem().document, id: '1', documentDate: '2024-03-15' } })
|
|
||||||
];
|
|
||||||
render(DocumentList, { ...baseProps, items, total: 1, sort: 'TITLE' });
|
render(DocumentList, { ...baseProps, items, total: 1, sort: 'TITLE' });
|
||||||
await expect
|
await expect
|
||||||
.element(page.getByTestId('group-header').filter({ hasText: '2024' }))
|
.element(page.getByTestId('group-header').filter({ hasText: '2024' }))
|
||||||
@@ -124,8 +113,6 @@ describe('DocumentList – sender grouping', () => {
|
|||||||
it('groups by sender displayName when sort is SENDER', async () => {
|
it('groups by sender displayName when sort is SENDER', async () => {
|
||||||
const items = [
|
const items = [
|
||||||
makeItem({
|
makeItem({
|
||||||
document: {
|
|
||||||
...makeItem().document,
|
|
||||||
id: '1',
|
id: '1',
|
||||||
sender: {
|
sender: {
|
||||||
id: 's1',
|
id: 's1',
|
||||||
@@ -134,11 +121,8 @@ describe('DocumentList – sender grouping', () => {
|
|||||||
personType: 'PERSON',
|
personType: 'PERSON',
|
||||||
familyMember: false
|
familyMember: false
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}),
|
}),
|
||||||
makeItem({
|
makeItem({
|
||||||
document: {
|
|
||||||
...makeItem().document,
|
|
||||||
id: '2',
|
id: '2',
|
||||||
sender: {
|
sender: {
|
||||||
id: 's2',
|
id: 's2',
|
||||||
@@ -147,7 +131,6 @@ describe('DocumentList – sender grouping', () => {
|
|||||||
personType: 'PERSON',
|
personType: 'PERSON',
|
||||||
familyMember: false
|
familyMember: false
|
||||||
}
|
}
|
||||||
}
|
|
||||||
})
|
})
|
||||||
];
|
];
|
||||||
render(DocumentList, { ...baseProps, items, total: 2, sort: 'SENDER' });
|
render(DocumentList, { ...baseProps, items, total: 2, sort: 'SENDER' });
|
||||||
@@ -167,10 +150,7 @@ describe('DocumentList – sender grouping', () => {
|
|||||||
personType: 'PERSON' as const,
|
personType: 'PERSON' as const,
|
||||||
familyMember: false
|
familyMember: false
|
||||||
};
|
};
|
||||||
const items = [
|
const items = [makeItem({ id: '1', sender }), makeItem({ id: '2', sender })];
|
||||||
makeItem({ document: { ...makeItem().document, id: '1', sender } }),
|
|
||||||
makeItem({ document: { ...makeItem().document, id: '2', sender } })
|
|
||||||
];
|
|
||||||
render(DocumentList, { ...baseProps, items, total: 2, sort: 'SENDER' });
|
render(DocumentList, { ...baseProps, items, total: 2, sort: 'SENDER' });
|
||||||
const cards = page.getByTestId('group-card');
|
const cards = page.getByTestId('group-card');
|
||||||
await expect.element(cards.first()).toBeInTheDocument();
|
await expect.element(cards.first()).toBeInTheDocument();
|
||||||
@@ -178,7 +158,7 @@ describe('DocumentList – sender grouping', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('places items with no sender under fallback label', async () => {
|
it('places items with no sender under fallback label', async () => {
|
||||||
const items = [makeItem({ document: { ...makeItem().document, id: '1', sender: undefined } })];
|
const items = [makeItem({ id: '1', sender: undefined })];
|
||||||
render(DocumentList, { ...baseProps, items, total: 1, sort: 'SENDER' });
|
render(DocumentList, { ...baseProps, items, total: 1, sort: 'SENDER' });
|
||||||
await expect.element(page.getByText('Unbekannter Absender')).toBeInTheDocument();
|
await expect.element(page.getByText('Unbekannter Absender')).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
@@ -190,8 +170,6 @@ describe('DocumentList – receiver grouping', () => {
|
|||||||
it('groups by receiver displayName when sort is RECEIVER', async () => {
|
it('groups by receiver displayName when sort is RECEIVER', async () => {
|
||||||
const items = [
|
const items = [
|
||||||
makeItem({
|
makeItem({
|
||||||
document: {
|
|
||||||
...makeItem().document,
|
|
||||||
id: '1',
|
id: '1',
|
||||||
receivers: [
|
receivers: [
|
||||||
{
|
{
|
||||||
@@ -202,7 +180,6 @@ describe('DocumentList – receiver grouping', () => {
|
|||||||
familyMember: false
|
familyMember: false
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
|
||||||
})
|
})
|
||||||
];
|
];
|
||||||
render(DocumentList, { ...baseProps, items, total: 1, sort: 'RECEIVER' });
|
render(DocumentList, { ...baseProps, items, total: 1, sort: 'RECEIVER' });
|
||||||
@@ -214,8 +191,6 @@ describe('DocumentList – receiver grouping', () => {
|
|||||||
it('duplicates a document into each receiver group', async () => {
|
it('duplicates a document into each receiver group', async () => {
|
||||||
const items = [
|
const items = [
|
||||||
makeItem({
|
makeItem({
|
||||||
document: {
|
|
||||||
...makeItem().document,
|
|
||||||
id: '1',
|
id: '1',
|
||||||
title: 'Rundbriefchen',
|
title: 'Rundbriefchen',
|
||||||
receivers: [
|
receivers: [
|
||||||
@@ -234,7 +209,6 @@ describe('DocumentList – receiver grouping', () => {
|
|||||||
familyMember: false
|
familyMember: false
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
|
||||||
})
|
})
|
||||||
];
|
];
|
||||||
render(DocumentList, { ...baseProps, items, total: 1, sort: 'RECEIVER' });
|
render(DocumentList, { ...baseProps, items, total: 1, sort: 'RECEIVER' });
|
||||||
@@ -249,7 +223,7 @@ describe('DocumentList – receiver grouping', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('places items with no receivers under fallback label', async () => {
|
it('places items with no receivers under fallback label', async () => {
|
||||||
const items = [makeItem({ document: { ...makeItem().document, id: '1', receivers: [] } })];
|
const items = [makeItem({ id: '1', receivers: [] })];
|
||||||
render(DocumentList, { ...baseProps, items, total: 1, sort: 'RECEIVER' });
|
render(DocumentList, { ...baseProps, items, total: 1, sort: 'RECEIVER' });
|
||||||
await expect.element(page.getByText('Unbekannter Empfänger')).toBeInTheDocument();
|
await expect.element(page.getByText('Unbekannter Empfänger')).toBeInTheDocument();
|
||||||
});
|
});
|
||||||
@@ -261,7 +235,7 @@ describe('DocumentList – DocumentRow delegation', () => {
|
|||||||
it('shows transcription snippet when matchData has one', async () => {
|
it('shows transcription snippet when matchData has one', async () => {
|
||||||
const items = [
|
const items = [
|
||||||
makeItem({
|
makeItem({
|
||||||
document: { ...makeItem().document, id: 'doc1' },
|
id: 'doc1',
|
||||||
matchData: {
|
matchData: {
|
||||||
transcriptionSnippet: 'Er schrieb einen langen Brief',
|
transcriptionSnippet: 'Er schrieb einen langen Brief',
|
||||||
titleOffsets: [],
|
titleOffsets: [],
|
||||||
@@ -278,7 +252,7 @@ describe('DocumentList – DocumentRow delegation', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('does not render snippet when matchData has no transcription snippet', async () => {
|
it('does not render snippet when matchData has no transcription snippet', async () => {
|
||||||
const items = [makeItem({ document: { ...makeItem().document, id: 'doc1' } })];
|
const items = [makeItem({ id: 'doc1' })];
|
||||||
render(DocumentList, { ...baseProps, items, total: 1 });
|
render(DocumentList, { ...baseProps, items, total: 1 });
|
||||||
await expect.element(page.getByTestId('search-snippet')).not.toBeInTheDocument();
|
await expect.element(page.getByTestId('search-snippet')).not.toBeInTheDocument();
|
||||||
});
|
});
|
||||||
@@ -286,7 +260,8 @@ describe('DocumentList – DocumentRow delegation', () => {
|
|||||||
it('renders mark for title highlight when titleOffsets present', async () => {
|
it('renders mark for title highlight when titleOffsets present', async () => {
|
||||||
const items = [
|
const items = [
|
||||||
makeItem({
|
makeItem({
|
||||||
document: { ...makeItem().document, id: 'doc1', title: 'Brief an Anna' },
|
id: 'doc1',
|
||||||
|
title: 'Brief an Anna',
|
||||||
matchData: {
|
matchData: {
|
||||||
titleOffsets: [{ start: 0, length: 5 }], // "Brief"
|
titleOffsets: [{ start: 0, length: 5 }], // "Brief"
|
||||||
senderMatched: false,
|
senderMatched: false,
|
||||||
|
|||||||
@@ -20,11 +20,31 @@ const { default: DocumentList } = await import('./DocumentList.svelte');
|
|||||||
|
|
||||||
afterEach(cleanup);
|
afterEach(cleanup);
|
||||||
|
|
||||||
const sender = { id: 's1', displayName: 'Anna Schmidt' };
|
const sender = {
|
||||||
const receiver = { id: 'r1', displayName: 'Bert Meier' };
|
id: 's1',
|
||||||
|
lastName: 'Schmidt',
|
||||||
|
displayName: 'Anna Schmidt',
|
||||||
|
personType: 'PERSON' as const,
|
||||||
|
familyMember: false
|
||||||
|
};
|
||||||
|
const receiver = {
|
||||||
|
id: 'r1',
|
||||||
|
lastName: 'Meier',
|
||||||
|
displayName: 'Bert Meier',
|
||||||
|
personType: 'PERSON' as const,
|
||||||
|
familyMember: false
|
||||||
|
};
|
||||||
|
|
||||||
|
const emptyMatchData = {
|
||||||
|
titleOffsets: [],
|
||||||
|
senderMatched: false,
|
||||||
|
matchedReceiverIds: [],
|
||||||
|
matchedTagIds: [],
|
||||||
|
snippetOffsets: [],
|
||||||
|
summaryOffsets: []
|
||||||
|
};
|
||||||
|
|
||||||
const makeItem = (overrides: Record<string, unknown> = {}) => ({
|
const makeItem = (overrides: Record<string, unknown> = {}) => ({
|
||||||
document: {
|
|
||||||
id: 'd1',
|
id: 'd1',
|
||||||
title: 'Brief 1923',
|
title: 'Brief 1923',
|
||||||
originalFilename: 'b.pdf',
|
originalFilename: 'b.pdf',
|
||||||
@@ -32,17 +52,14 @@ const makeItem = (overrides: Record<string, unknown> = {}) => ({
|
|||||||
sender,
|
sender,
|
||||||
receivers: [receiver],
|
receivers: [receiver],
|
||||||
tags: [],
|
tags: [],
|
||||||
thumbnailUrl: null,
|
summary: undefined,
|
||||||
contentType: 'application/pdf',
|
archiveBox: undefined,
|
||||||
summary: null,
|
archiveFolder: undefined,
|
||||||
archiveBox: null,
|
location: undefined,
|
||||||
archiveFolder: null,
|
matchData: emptyMatchData,
|
||||||
location: null,
|
|
||||||
...overrides
|
|
||||||
},
|
|
||||||
matchData: null,
|
|
||||||
completionPercentage: 0,
|
completionPercentage: 0,
|
||||||
contributors: []
|
contributors: [],
|
||||||
|
...overrides
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('DocumentList', () => {
|
describe('DocumentList', () => {
|
||||||
@@ -87,8 +104,26 @@ describe('DocumentList', () => {
|
|||||||
render(DocumentList, {
|
render(DocumentList, {
|
||||||
props: {
|
props: {
|
||||||
items: [
|
items: [
|
||||||
makeItem({ id: 'd1', sender: { id: 's1', displayName: 'Anna Schmidt' } }),
|
makeItem({
|
||||||
makeItem({ id: 'd2', sender: { id: 's2', displayName: 'Bert Meier' } })
|
id: 'd1',
|
||||||
|
sender: {
|
||||||
|
id: 's1',
|
||||||
|
lastName: 'Schmidt',
|
||||||
|
displayName: 'Anna Schmidt',
|
||||||
|
personType: 'PERSON',
|
||||||
|
familyMember: false
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
makeItem({
|
||||||
|
id: 'd2',
|
||||||
|
sender: {
|
||||||
|
id: 's2',
|
||||||
|
lastName: 'Meier',
|
||||||
|
displayName: 'Bert Meier',
|
||||||
|
personType: 'PERSON',
|
||||||
|
familyMember: false
|
||||||
|
}
|
||||||
|
})
|
||||||
],
|
],
|
||||||
canWrite: false,
|
canWrite: false,
|
||||||
sort: 'SENDER' as const
|
sort: 'SENDER' as const
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ async function resolvePersonName(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type DocumentSearchItem = components['schemas']['DocumentSearchItem'];
|
type DocumentListItem = components['schemas']['DocumentListItem'];
|
||||||
|
|
||||||
const VALID_SORTS = ['DATE', 'TITLE', 'SENDER', 'RECEIVER', 'UPLOAD_DATE', 'RELEVANCE'] as const;
|
const VALID_SORTS = ['DATE', 'TITLE', 'SENDER', 'RECEIVER', 'UPLOAD_DATE', 'RELEVANCE'] as const;
|
||||||
type ValidSort = (typeof VALID_SORTS)[number];
|
type ValidSort = (typeof VALID_SORTS)[number];
|
||||||
@@ -77,7 +77,7 @@ export async function load({ url, fetch }) {
|
|||||||
]);
|
]);
|
||||||
} catch {
|
} catch {
|
||||||
return {
|
return {
|
||||||
items: [] as DocumentSearchItem[],
|
items: [] as DocumentListItem[],
|
||||||
totalElements: 0,
|
totalElements: 0,
|
||||||
pageNumber: 0,
|
pageNumber: 0,
|
||||||
pageSize: PAGE_SIZE,
|
pageSize: PAGE_SIZE,
|
||||||
@@ -107,7 +107,7 @@ export async function load({ url, fetch }) {
|
|||||||
: null;
|
: null;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
items: (result.data?.items ?? []) as DocumentSearchItem[],
|
items: (result.data?.items ?? []) as DocumentListItem[],
|
||||||
totalElements: result.data?.totalElements ?? 0,
|
totalElements: result.data?.totalElements ?? 0,
|
||||||
pageNumber: result.data?.pageNumber ?? page,
|
pageNumber: result.data?.pageNumber ?? page,
|
||||||
pageSize: result.data?.pageSize ?? PAGE_SIZE,
|
pageSize: result.data?.pageSize ?? PAGE_SIZE,
|
||||||
|
|||||||
@@ -140,15 +140,12 @@ describe('documents/+ page', () => {
|
|||||||
data: baseData({
|
data: baseData({
|
||||||
items: [
|
items: [
|
||||||
{
|
{
|
||||||
document: {
|
|
||||||
id: 'd1',
|
id: 'd1',
|
||||||
title: 'Brief 1899',
|
title: 'Brief 1899',
|
||||||
status: 'TRANSCRIBED',
|
|
||||||
documentDate: '1899-04-14',
|
documentDate: '1899-04-14',
|
||||||
summary: '',
|
|
||||||
originalFilename: 'b1.pdf',
|
originalFilename: 'b1.pdf',
|
||||||
receivers: []
|
receivers: [],
|
||||||
},
|
tags: [],
|
||||||
matchData: {
|
matchData: {
|
||||||
titleOffsets: [],
|
titleOffsets: [],
|
||||||
senderMatched: false,
|
senderMatched: false,
|
||||||
|
|||||||
@@ -108,7 +108,8 @@ describe('home page load — dashboard', () => {
|
|||||||
data: { segmentationCount: 0, transcriptionCount: 0, readyCount: 0 }
|
data: { segmentationCount: 0, transcriptionCount: 0, readyCount: 0 }
|
||||||
}) // weekly-stats
|
}) // weekly-stats
|
||||||
.mockResolvedValueOnce({ response: { ok: true }, data: [] }) // incomplete
|
.mockResolvedValueOnce({ response: { ok: true }, data: [] }) // incomplete
|
||||||
.mockResolvedValueOnce({ response: { ok: true }, data: { count: 0 } }); // incomplete-count
|
.mockResolvedValueOnce({ response: { ok: true }, data: { count: 0 } }) // incomplete-count
|
||||||
|
.mockResolvedValueOnce({ response: { ok: true }, data: [] }); // tags/tree
|
||||||
vi.mocked(createApiClient).mockReturnValue({ GET: mockGet } as ReturnType<
|
vi.mocked(createApiClient).mockReturnValue({ GET: mockGet } as ReturnType<
|
||||||
typeof createApiClient
|
typeof createApiClient
|
||||||
>);
|
>);
|
||||||
@@ -146,7 +147,8 @@ describe('home page load — dashboard', () => {
|
|||||||
data: { segmentationCount: 0, transcriptionCount: 0, readyCount: 0 }
|
data: { segmentationCount: 0, transcriptionCount: 0, readyCount: 0 }
|
||||||
}) // weekly-stats
|
}) // weekly-stats
|
||||||
.mockResolvedValueOnce({ response: { ok: true }, data: [] }) // incomplete
|
.mockResolvedValueOnce({ response: { ok: true }, data: [] }) // incomplete
|
||||||
.mockResolvedValueOnce({ response: { ok: true }, data: { count: 0 } }); // incomplete-count
|
.mockResolvedValueOnce({ response: { ok: true }, data: { count: 0 } }) // incomplete-count
|
||||||
|
.mockResolvedValueOnce({ response: { ok: true }, data: [] }); // tags/tree
|
||||||
vi.mocked(createApiClient).mockReturnValue({ GET: mockGet } as ReturnType<
|
vi.mocked(createApiClient).mockReturnValue({ GET: mockGet } as ReturnType<
|
||||||
typeof createApiClient
|
typeof createApiClient
|
||||||
>);
|
>);
|
||||||
@@ -394,6 +396,55 @@ describe('home page load — reader branch (isReader = !canWrite && !canAnnotate
|
|||||||
expect(result.isReader).toBe(false);
|
expect(result.isReader).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('maps search result items directly to recentDocs without wrapping in a .document property', async () => {
|
||||||
|
const searchItem = {
|
||||||
|
id: 'd1',
|
||||||
|
title: 'Liebesbrief',
|
||||||
|
originalFilename: 'letter.pdf',
|
||||||
|
completionPercentage: 80,
|
||||||
|
receivers: [],
|
||||||
|
tags: [],
|
||||||
|
contributors: [],
|
||||||
|
matchData: { titleOffsets: [], senderMatched: false },
|
||||||
|
createdAt: '2026-05-01T10:00:00Z',
|
||||||
|
updatedAt: '2026-05-10T08:00:00Z'
|
||||||
|
};
|
||||||
|
const mockGet = vi
|
||||||
|
.fn()
|
||||||
|
.mockResolvedValueOnce({ response: { ok: true, status: 200 }, data: [] }) // initial persons
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
response: { ok: true },
|
||||||
|
data: { totalDocuments: 1, totalPersons: 1 }
|
||||||
|
}) // stats
|
||||||
|
.mockResolvedValueOnce({ response: { ok: true }, data: [] }) // topPersons
|
||||||
|
.mockResolvedValueOnce({
|
||||||
|
response: { ok: true },
|
||||||
|
data: { items: [searchItem], totalElements: 1, pageNumber: 0, pageSize: 5, totalPages: 1 }
|
||||||
|
}) // search
|
||||||
|
.mockResolvedValueOnce({ response: { ok: true }, data: [] }); // stories
|
||||||
|
vi.mocked(createApiClient).mockReturnValue({ GET: mockGet } as ReturnType<
|
||||||
|
typeof createApiClient
|
||||||
|
>);
|
||||||
|
|
||||||
|
const result = await load({
|
||||||
|
url: makeUrl(),
|
||||||
|
request: new Request('http://localhost/'),
|
||||||
|
fetch: vi.fn() as unknown as typeof fetch,
|
||||||
|
parent: vi
|
||||||
|
.fn()
|
||||||
|
.mockResolvedValue({ canWrite: false, canAnnotate: false, canBlogWrite: false })
|
||||||
|
} as Parameters<typeof load>[0]);
|
||||||
|
|
||||||
|
expect(result.isReader).toBe(true);
|
||||||
|
if (result.isReader) {
|
||||||
|
expect(result.recentDocs).toHaveLength(1);
|
||||||
|
expect(result.recentDocs[0]).toBeDefined();
|
||||||
|
expect(result.recentDocs[0].id).toBe('d1');
|
||||||
|
expect(result.recentDocs[0].createdAt).toBe('2026-05-01T10:00:00Z');
|
||||||
|
expect(result.recentDocs[0].updatedAt).toBe('2026-05-10T08:00:00Z');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('returns topPersons=[] when topPersons fetch fails, rest of data still loads', async () => {
|
it('returns topPersons=[] when topPersons fetch fails, rest of data still loads', async () => {
|
||||||
const okStats = {
|
const okStats = {
|
||||||
response: { ok: true, status: 200 },
|
response: { ok: true, status: 200 },
|
||||||
@@ -409,7 +460,8 @@ describe('home page load — reader branch (isReader = !canWrite && !canAnnotate
|
|||||||
.mockResolvedValueOnce(okStats)
|
.mockResolvedValueOnce(okStats)
|
||||||
.mockReturnValueOnce(failPersons)
|
.mockReturnValueOnce(failPersons)
|
||||||
.mockResolvedValueOnce(okSearch)
|
.mockResolvedValueOnce(okSearch)
|
||||||
.mockResolvedValueOnce(okStories);
|
.mockResolvedValueOnce(okStories)
|
||||||
|
.mockResolvedValueOnce({ response: { ok: true, status: 200 }, data: [] }); // tags/tree
|
||||||
vi.mocked(createApiClient).mockReturnValue({ GET: mockGet } as ReturnType<
|
vi.mocked(createApiClient).mockReturnValue({ GET: mockGet } as ReturnType<
|
||||||
typeof createApiClient
|
typeof createApiClient
|
||||||
>);
|
>);
|
||||||
|
|||||||
12
frontend/src/routes/themen/+page.server.ts
Normal file
12
frontend/src/routes/themen/+page.server.ts
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
import { error } from '@sveltejs/kit';
|
||||||
|
import { createApiClient } from '$lib/shared/api.server';
|
||||||
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
|
type TagTreeNodeDTO = components['schemas']['TagTreeNodeDTO'];
|
||||||
|
|
||||||
|
export async function load({ fetch }: Parameters<import('./$types').PageServerLoad>[0]) {
|
||||||
|
const api = createApiClient(fetch);
|
||||||
|
const result = await api.GET('/api/tags/tree');
|
||||||
|
if (!result.response.ok) throw error(500, 'Themen konnten nicht geladen werden.');
|
||||||
|
return { tree: (result.data ?? []) as TagTreeNodeDTO[] };
|
||||||
|
}
|
||||||
85
frontend/src/routes/themen/+page.svelte
Normal file
85
frontend/src/routes/themen/+page.svelte
Normal file
@@ -0,0 +1,85 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import * as m from '$lib/paraglide/messages.js';
|
||||||
|
import BackButton from '$lib/shared/primitives/BackButton.svelte';
|
||||||
|
import { hasAnyDocuments } from '$lib/shared/utils/tagUtils';
|
||||||
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
|
type TagTreeNodeDTO = components['schemas']['TagTreeNodeDTO'];
|
||||||
|
|
||||||
|
const MAX_VISIBLE_CHILDREN = 5;
|
||||||
|
|
||||||
|
let { data }: { data: { tree: TagTreeNodeDTO[] } } = $props();
|
||||||
|
|
||||||
|
const visibleTree = $derived.by(() => data.tree.filter(hasAnyDocuments));
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:head>
|
||||||
|
<title>{m.themen_widget_title()}</title>
|
||||||
|
</svelte:head>
|
||||||
|
|
||||||
|
<main class="mx-auto max-w-7xl px-4 py-8 sm:px-6 lg:px-8">
|
||||||
|
<div class="mb-6 flex items-center gap-3">
|
||||||
|
<BackButton />
|
||||||
|
<h1 class="font-serif text-2xl font-semibold text-ink">{m.themen_widget_title()}</h1>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{#if visibleTree.length === 0}
|
||||||
|
<p class="font-sans text-sm text-ink-3">{m.themen_leer()}</p>
|
||||||
|
{:else}
|
||||||
|
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-3">
|
||||||
|
{#each visibleTree as tag (tag.id)}
|
||||||
|
{@const visibleChildren = (tag.children ?? []).filter(hasAnyDocuments)}
|
||||||
|
{@const shownChildren = visibleChildren.slice(0, MAX_VISIBLE_CHILDREN)}
|
||||||
|
{@const hiddenCount = visibleChildren.length - shownChildren.length}
|
||||||
|
|
||||||
|
<div class="overflow-hidden rounded-sm border border-line bg-surface shadow-sm">
|
||||||
|
<div
|
||||||
|
class="h-1.5 w-full flex-shrink-0"
|
||||||
|
aria-hidden="true"
|
||||||
|
style="background: var(--c-tag-{tag.color ?? 'slate'})"
|
||||||
|
></div>
|
||||||
|
|
||||||
|
<a
|
||||||
|
href="/?tag={encodeURIComponent(tag.name)}"
|
||||||
|
aria-label="{tag.name}{tag.documentCount > 0
|
||||||
|
? ', ' + m.themen_dokumente({ count: tag.documentCount })
|
||||||
|
: ''}"
|
||||||
|
class="flex min-h-[56px] items-center justify-between px-4 pt-4 pb-3 hover:bg-canvas focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:outline-none focus-visible:ring-inset"
|
||||||
|
>
|
||||||
|
<span class="font-serif text-base font-semibold text-ink">{tag.name}</span>
|
||||||
|
<span class="mr-1 ml-auto font-sans text-sm text-ink-3 tabular-nums">
|
||||||
|
{#if tag.documentCount > 0}{tag.documentCount}{/if}
|
||||||
|
</span>
|
||||||
|
<span aria-hidden="true" class="h-3.5 w-3.5 flex-shrink-0 text-brand-mint">›</span>
|
||||||
|
</a>
|
||||||
|
|
||||||
|
{#if shownChildren.length > 0}
|
||||||
|
<div class="mx-4 border-t border-line"></div>
|
||||||
|
|
||||||
|
{#each shownChildren as child (child.id)}
|
||||||
|
<a
|
||||||
|
href="/?tag={encodeURIComponent(child.name)}"
|
||||||
|
class="flex min-h-[44px] items-center justify-between px-4 py-2.5 hover:bg-canvas focus-visible:bg-canvas focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:outline-none focus-visible:ring-inset"
|
||||||
|
>
|
||||||
|
<span class="font-sans text-sm text-ink">{child.name}</span>
|
||||||
|
<span class="mr-1 ml-auto font-sans text-xs text-ink-3 tabular-nums">
|
||||||
|
{#if child.documentCount > 0}{child.documentCount}{/if}
|
||||||
|
</span>
|
||||||
|
<span aria-hidden="true" class="h-3 w-3 flex-shrink-0 text-brand-mint">›</span>
|
||||||
|
</a>
|
||||||
|
{/each}
|
||||||
|
|
||||||
|
{#if hiddenCount > 0}
|
||||||
|
<a
|
||||||
|
href="/?tag={encodeURIComponent(tag.name)}"
|
||||||
|
class="block min-h-[44px] px-4 py-2.5 font-sans text-sm text-ink-3 hover:bg-canvas hover:text-ink focus-visible:ring-2 focus-visible:ring-brand-navy focus-visible:outline-none focus-visible:ring-inset"
|
||||||
|
>
|
||||||
|
{m.themen_weitere({ count: hiddenCount })} →
|
||||||
|
</a>
|
||||||
|
{/if}
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</main>
|
||||||
60
frontend/src/routes/themen/page.server.spec.ts
Normal file
60
frontend/src/routes/themen/page.server.spec.ts
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
import { describe, expect, it, vi, beforeEach } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('$lib/shared/api.server', () => ({
|
||||||
|
createApiClient: vi.fn(),
|
||||||
|
extractErrorCode: (e: unknown) => (e as { code?: string } | undefined)?.code
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { createApiClient } from '$lib/shared/api.server';
|
||||||
|
|
||||||
|
beforeEach(() => vi.clearAllMocks());
|
||||||
|
|
||||||
|
function mockApiGet(ok: boolean, data: unknown) {
|
||||||
|
vi.mocked(createApiClient).mockReturnValue({
|
||||||
|
GET: vi.fn().mockResolvedValue({ response: { ok }, data })
|
||||||
|
} as ReturnType<typeof createApiClient>);
|
||||||
|
}
|
||||||
|
|
||||||
|
const makeTag = (name: string, documentCount = 0) => ({
|
||||||
|
id: 'id-' + name,
|
||||||
|
name,
|
||||||
|
documentCount,
|
||||||
|
children: []
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('/themen +page.server load', () => {
|
||||||
|
function makeLoadEvent() {
|
||||||
|
return {
|
||||||
|
fetch: vi.fn() as unknown as typeof fetch,
|
||||||
|
request: new Request('http://localhost/themen'),
|
||||||
|
url: new URL('http://localhost/themen')
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it('returns tag tree when API succeeds', async () => {
|
||||||
|
const tree = [makeTag('Briefe', 5), makeTag('Fotos', 3)];
|
||||||
|
mockApiGet(true, tree);
|
||||||
|
|
||||||
|
const { load } = await import('./+page.server');
|
||||||
|
const result = await load(makeLoadEvent());
|
||||||
|
|
||||||
|
expect(result.tree).toEqual(tree);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns empty array when API returns empty list', async () => {
|
||||||
|
mockApiGet(true, []);
|
||||||
|
|
||||||
|
const { load } = await import('./+page.server');
|
||||||
|
const result = await load(makeLoadEvent());
|
||||||
|
|
||||||
|
expect(result.tree).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('throws 500 when API call fails', async () => {
|
||||||
|
mockApiGet(false, null);
|
||||||
|
|
||||||
|
const { load } = await import('./+page.server');
|
||||||
|
|
||||||
|
await expect(load(makeLoadEvent())).rejects.toMatchObject({ status: 500 });
|
||||||
|
});
|
||||||
|
});
|
||||||
57
frontend/src/routes/themen/page.svelte.spec.ts
Normal file
57
frontend/src/routes/themen/page.svelte.spec.ts
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
import { describe, it, expect, afterEach } from 'vitest';
|
||||||
|
import { cleanup, render } from 'vitest-browser-svelte';
|
||||||
|
import ThemenPage from './+page.svelte';
|
||||||
|
import type { components } from '$lib/generated/api';
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
type TagTreeNodeDTO = components['schemas']['TagTreeNodeDTO'];
|
||||||
|
|
||||||
|
function makeTag(
|
||||||
|
name: string,
|
||||||
|
documentCount: number,
|
||||||
|
children: TagTreeNodeDTO[] = []
|
||||||
|
): TagTreeNodeDTO {
|
||||||
|
return { id: 'id-' + name, name, documentCount, children };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('/themen +page', () => {
|
||||||
|
it('renders one card per visible root tag', async () => {
|
||||||
|
const tree = [makeTag('Briefe', 5), makeTag('Fotos', 3)];
|
||||||
|
render(ThemenPage, { data: { tree } });
|
||||||
|
expect(document.body.textContent).toContain('Briefe');
|
||||||
|
expect(document.body.textContent).toContain('Fotos');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not render a tag with no documents in its subtree', async () => {
|
||||||
|
const tree = [makeTag('Briefe', 5), makeTag('Leer', 0)];
|
||||||
|
render(ThemenPage, { data: { tree } });
|
||||||
|
expect(document.body.textContent).not.toContain('Leer');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shows empty state when all tags filtered out', async () => {
|
||||||
|
render(ThemenPage, { data: { tree: [makeTag('Leer', 0)] } });
|
||||||
|
expect(document.body.textContent).toMatch(/Noch keine Themen/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shows empty state when tree is empty', async () => {
|
||||||
|
render(ThemenPage, { data: { tree: [] } });
|
||||||
|
expect(document.body.textContent).toMatch(/Noch keine Themen/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('renders child tags for a root tag', async () => {
|
||||||
|
const tree = [makeTag('Briefe', 5, [makeTag('Brautbriefe', 3), makeTag('Kriegsbriefe', 2)])];
|
||||||
|
render(ThemenPage, { data: { tree } });
|
||||||
|
expect(document.body.textContent).toContain('Brautbriefe');
|
||||||
|
expect(document.body.textContent).toContain('Kriegsbriefe');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shows "+ N weitere" when a root tag has more than 5 children', async () => {
|
||||||
|
const children = Array.from({ length: 7 }, (_, i) => makeTag(`Kind${i}`, i + 1));
|
||||||
|
const tree = [makeTag('Briefe', 10, children)];
|
||||||
|
render(ThemenPage, { data: { tree } });
|
||||||
|
expect(document.body.textContent).toMatch(/\+\s*2\s*weitere/);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,702 @@
|
|||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": { "type": "grafana", "uid": "grafana" },
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"description": "Product owner overview — system health, user activity, archive progress, and OCR quality at a weekly glance.",
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 0,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"liveNow": false,
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||||
|
"id": 100,
|
||||||
|
"title": "System Health",
|
||||||
|
"type": "row",
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"title": "Backend Status",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "up{job=\"spring-boot\"}",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [
|
||||||
|
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
|
||||||
|
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
|
||||||
|
],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "red", "value": null },
|
||||||
|
{ "color": "green", "value": 1 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "value"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"title": "Server Errors (5xx)",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(increase(http_server_requests_seconds_count{status=~\"5..\"}[$__range]))",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 1 },
|
||||||
|
{ "color": "red", "value": 6 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"title": "Response Time (p95)",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket[$__range])) by (le))",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s",
|
||||||
|
"decimals": 2,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 0.5 },
|
||||||
|
{ "color": "red", "value": 2 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"title": "Error Log Count",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(count_over_time({compose_service=\"backend\"} | json | level=\"ERROR\" [$__range]))",
|
||||||
|
"queryType": "instant",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 1 },
|
||||||
|
{ "color": "red", "value": 10 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"title": "CPU Usage",
|
||||||
|
"type": "bargauge",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 5, "w": 8, "x": 0, "y": 5 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percent",
|
||||||
|
"min": 0,
|
||||||
|
"max": 100,
|
||||||
|
"decimals": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 70 },
|
||||||
|
{ "color": "red", "value": 85 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"showUnfilled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"title": "Memory Usage",
|
||||||
|
"type": "bargauge",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 5, "w": 8, "x": 8, "y": 5 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percent",
|
||||||
|
"min": 0,
|
||||||
|
"max": 100,
|
||||||
|
"decimals": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 70 },
|
||||||
|
{ "color": "red", "value": 85 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"showUnfilled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"title": "Disk Usage",
|
||||||
|
"type": "bargauge",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 5, "w": 8, "x": 16, "y": 5 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "(1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})) * 100",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percent",
|
||||||
|
"min": 0,
|
||||||
|
"max": 100,
|
||||||
|
"decimals": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 70 },
|
||||||
|
{ "color": "red", "value": 80 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"showUnfilled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 },
|
||||||
|
"id": 101,
|
||||||
|
"title": "User Activity",
|
||||||
|
"type": "row",
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"title": "Active Users",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 11 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT COUNT(DISTINCT actor_id) AS value FROM audit_log WHERE happened_at >= NOW() - INTERVAL '7 days' AND kind = 'LOGIN_SUCCESS'",
|
||||||
|
"format": "table",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 9,
|
||||||
|
"title": "Total Logins",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 11 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT COUNT(*) AS value FROM audit_log WHERE happened_at >= NOW() - INTERVAL '7 days' AND kind = 'LOGIN_SUCCESS'",
|
||||||
|
"format": "table",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 10,
|
||||||
|
"title": "Failed Login Attempts",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 11 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT COUNT(*) AS value FROM audit_log WHERE happened_at >= NOW() - INTERVAL '7 days' AND kind IN ('LOGIN_FAILED', 'LOGIN_RATE_LIMITED')",
|
||||||
|
"format": "table",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 1 },
|
||||||
|
{ "color": "red", "value": 4 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"title": "Daily Logins (last 7 days)",
|
||||||
|
"type": "barchart",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 15 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT DATE_TRUNC('day', happened_at) AS time, COUNT(*) AS logins FROM audit_log WHERE happened_at >= NOW() - INTERVAL '7 days' AND kind = 'LOGIN_SUCCESS' GROUP BY 1 ORDER BY 1",
|
||||||
|
"format": "time_series",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": { "displayMode": "hidden" },
|
||||||
|
"orientation": "auto",
|
||||||
|
"showValue": "auto",
|
||||||
|
"stacking": "none",
|
||||||
|
"xTickLabelRotation": 0,
|
||||||
|
"xTickLabelSpacing": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 },
|
||||||
|
"id": 102,
|
||||||
|
"title": "Archive Progress",
|
||||||
|
"type": "row",
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 12,
|
||||||
|
"title": "Transcription Coverage",
|
||||||
|
"type": "bargauge",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 5, "w": 24, "x": 0, "y": 23 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT (COUNT(*) FILTER (WHERE text IS NOT NULL AND text <> ''))::float * 100.0 / NULLIF(COUNT(*), 0) AS percent_complete FROM transcription_blocks",
|
||||||
|
"format": "table",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percent",
|
||||||
|
"min": 0,
|
||||||
|
"max": 100,
|
||||||
|
"decimals": 1,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "red", "value": null },
|
||||||
|
{ "color": "yellow", "value": 25 },
|
||||||
|
{ "color": "green", "value": 75 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"showUnfilled": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 13,
|
||||||
|
"title": "Total Documents",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 28 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT COUNT(*) AS value FROM documents WHERE status <> 'PLACEHOLDER'",
|
||||||
|
"format": "table",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 14,
|
||||||
|
"title": "Uploads This Week",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 28 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT COUNT(*) AS value FROM audit_log WHERE happened_at >= NOW() - INTERVAL '7 days' AND kind = 'FILE_UPLOADED'",
|
||||||
|
"format": "table",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 15,
|
||||||
|
"title": "Blocks Transcribed This Week",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 28 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT COUNT(*) AS value FROM audit_log WHERE happened_at >= NOW() - INTERVAL '7 days' AND kind = 'TEXT_SAVED'",
|
||||||
|
"format": "table",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 16,
|
||||||
|
"title": "Blocks Reviewed This Week",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "postgres", "uid": "postgres" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 28 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"rawSql": "SELECT COUNT(*) AS value FROM audit_log WHERE happened_at >= NOW() - INTERVAL '7 days' AND kind = 'BLOCK_REVIEWED'",
|
||||||
|
"format": "table",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
|
||||||
|
"id": 103,
|
||||||
|
"title": "OCR Health",
|
||||||
|
"type": "row",
|
||||||
|
"panels": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 17,
|
||||||
|
"title": "OCR Jobs",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 33 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(increase(ocr_jobs_total[$__range]))",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"decimals": 0,
|
||||||
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 18,
|
||||||
|
"title": "OCR Page Error Rate",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 33 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(increase(ocr_skipped_pages_total[$__range])) / clamp_min(sum(increase(ocr_pages_total[$__range])), 1)",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percentunit",
|
||||||
|
"decimals": 1,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 0.01 },
|
||||||
|
{ "color": "red", "value": 0.05 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 19,
|
||||||
|
"title": "Illegible Word Rate",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 33 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(increase(ocr_illegible_words_total[$__range])) / clamp_min(sum(increase(ocr_words_total[$__range])), 1)",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percentunit",
|
||||||
|
"decimals": 1,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 0.1 },
|
||||||
|
{ "color": "red", "value": 0.25 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 20,
|
||||||
|
"title": "OCR Service Status",
|
||||||
|
"type": "stat",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 33 },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "ocr_models_ready",
|
||||||
|
"instant": true,
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [
|
||||||
|
{ "type": "value", "options": { "0": { "text": "NOT READY", "color": "red" } } },
|
||||||
|
{ "type": "value", "options": { "1": { "text": "READY", "color": "green" } } }
|
||||||
|
],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "red", "value": null },
|
||||||
|
{ "color": "green", "value": 1 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"color": { "mode": "thresholds" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "value"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": ["po-overview", "familienarchiv"],
|
||||||
|
"templating": { "list": [] },
|
||||||
|
"time": { "from": "now-7d", "to": "now" },
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "browser",
|
||||||
|
"title": "PO Overview",
|
||||||
|
"uid": "po-overview",
|
||||||
|
"version": 1,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
@@ -36,3 +36,19 @@ datasources:
|
|||||||
datasourceUid: prometheus
|
datasourceUid: prometheus
|
||||||
nodeGraph:
|
nodeGraph:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|
||||||
|
# Read-only PostgreSQL datasource for the PO Overview dashboard (issue #651).
|
||||||
|
# Uses the grafana_reader role provisioned by Flyway V68. Traffic stays inside
|
||||||
|
# archiv-net, so sslmode=disable is the deliberate, accepted setting.
|
||||||
|
- name: PostgreSQL
|
||||||
|
type: postgres
|
||||||
|
uid: postgres
|
||||||
|
url: archive-db:5432
|
||||||
|
user: grafana_reader
|
||||||
|
editable: false
|
||||||
|
secureJsonData:
|
||||||
|
password: ${GRAFANA_DB_PASSWORD}
|
||||||
|
jsonData:
|
||||||
|
database: ${POSTGRES_DB}
|
||||||
|
sslmode: disable
|
||||||
|
postgresVersion: 1600
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ GLITCHTIP_DOMAIN=https://glitchtip.archiv.raddatz.cloud
|
|||||||
|
|
||||||
POSTGRES_USER=archiv
|
POSTGRES_USER=archiv
|
||||||
|
|
||||||
|
# Note: GRAFANA_DB_PASSWORD is a secret and is injected by CI from
|
||||||
|
# obs-secrets.env (see .env.example for the local-dev declaration).
|
||||||
|
# It is consumed by both archive-backend (Flyway V68 placeholder) and
|
||||||
|
# obs-grafana (PostgreSQL datasource).
|
||||||
|
|
||||||
# PostgreSQL hostname for GlitchTip db-init and workers.
|
# PostgreSQL hostname for GlitchTip db-init and workers.
|
||||||
# The actual value depends on the Compose project name — it is not a fixed string.
|
# The actual value depends on the Compose project name — it is not a fixed string.
|
||||||
# CI sets POSTGRES_HOST in obs-secrets.env per environment:
|
# CI sets POSTGRES_HOST in obs-secrets.env per environment:
|
||||||
|
|||||||
@@ -20,7 +20,4 @@ scrape_configs:
|
|||||||
- job_name: ocr-service
|
- job_name: ocr-service
|
||||||
metrics_path: /metrics
|
metrics_path: /metrics
|
||||||
static_configs:
|
static_configs:
|
||||||
# TODO: remove or add prometheus-client to ocr-service.
|
|
||||||
# The Python OCR service does not currently expose Prometheus metrics.
|
|
||||||
# This target will show as DOWN until prometheus-client is added to ocr-service.
|
|
||||||
- targets: ['ocr:8000']
|
- targets: ['ocr:8000']
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import glob
|
import glob
|
||||||
|
import inspect
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
@@ -10,9 +11,11 @@ import re
|
|||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
import zipfile
|
import zipfile
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from typing import Awaitable, Callable
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
@@ -20,8 +23,11 @@ import pypdfium2 as pdfium
|
|||||||
from fastapi import FastAPI, Form, Header, HTTPException, UploadFile
|
from fastapi import FastAPI, Form, Header, HTTPException, UploadFile
|
||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
from prometheus_client import REGISTRY
|
||||||
|
from prometheus_fastapi_instrumentator import Instrumentator
|
||||||
|
|
||||||
from confidence import apply_confidence_markers, get_threshold
|
from confidence import apply_confidence_markers, get_threshold
|
||||||
|
from metrics import OcrMetrics, build_metrics
|
||||||
from spell_check import correct_text, load_spell_checker
|
from spell_check import correct_text, load_spell_checker
|
||||||
from engines import kraken as kraken_engine
|
from engines import kraken as kraken_engine
|
||||||
from engines import surya as surya_engine
|
from engines import surya as surya_engine
|
||||||
@@ -37,6 +43,12 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
_models_ready = False
|
_models_ready = False
|
||||||
|
|
||||||
|
# One-shot import-time binding to the default REGISTRY. Tests that need a
|
||||||
|
# clean counter state must monkeypatch `main.metrics` with a container built
|
||||||
|
# from a fresh CollectorRegistry — rebinding through the registry directly
|
||||||
|
# will not retarget the references stored in the OcrMetrics dataclass.
|
||||||
|
metrics: OcrMetrics = build_metrics(REGISTRY)
|
||||||
|
|
||||||
ALLOWED_PDF_HOSTS = set(
|
ALLOWED_PDF_HOSTS = set(
|
||||||
h.strip() for h in os.getenv("ALLOWED_PDF_HOSTS", "minio,localhost,127.0.0.1").split(",")
|
h.strip() for h in os.getenv("ALLOWED_PDF_HOSTS", "minio,localhost,127.0.0.1").split(",")
|
||||||
)
|
)
|
||||||
@@ -44,6 +56,42 @@ ALLOWED_PDF_HOSTS = set(
|
|||||||
_SPELL_CHECK_SCRIPT_TYPES = {"HANDWRITING_KURRENT", "HANDWRITING_LATIN"}
|
_SPELL_CHECK_SCRIPT_TYPES = {"HANDWRITING_KURRENT", "HANDWRITING_LATIN"}
|
||||||
|
|
||||||
|
|
||||||
|
async def _record_training(
|
||||||
|
runner: Callable[[], Awaitable[dict] | dict],
|
||||||
|
kind: str,
|
||||||
|
) -> dict:
|
||||||
|
"""Run a training callable and record outcome + accuracy metrics.
|
||||||
|
|
||||||
|
Wraps the per-endpoint try/except + outcome counter + accuracy gauge
|
||||||
|
block that used to be repeated at /train, /train-sender, and /segtrain.
|
||||||
|
The runner returns a dict with at least an `accuracy` key; if its value
|
||||||
|
is None, the gauge is left at its default.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
result = runner()
|
||||||
|
if inspect.isawaitable(result):
|
||||||
|
result = await result
|
||||||
|
except Exception:
|
||||||
|
metrics.ocr_training_runs_total.labels(kind=kind, outcome="error").inc()
|
||||||
|
raise
|
||||||
|
metrics.ocr_training_runs_total.labels(kind=kind, outcome="success").inc()
|
||||||
|
if result.get("accuracy") is not None:
|
||||||
|
metrics.ocr_model_accuracy.labels(kind=kind).set(result["accuracy"])
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _observe_block_words(words: list[dict], threshold: float) -> None:
|
||||||
|
"""Record per-block word counts and below-threshold word counts.
|
||||||
|
|
||||||
|
Pre: `words` is non-empty. Caller checks for that — keeping the helper
|
||||||
|
branch-free makes the call sites read as a single line.
|
||||||
|
"""
|
||||||
|
metrics.ocr_words_total.inc(len(words))
|
||||||
|
metrics.ocr_illegible_words_total.inc(
|
||||||
|
sum(1 for w in words if w["confidence"] < threshold)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _validate_url(url: str) -> None:
|
def _validate_url(url: str) -> None:
|
||||||
"""Validate that the PDF URL points to an allowed host (SSRF protection)."""
|
"""Validate that the PDF URL points to an allowed host (SSRF protection)."""
|
||||||
parsed = urlparse(url)
|
parsed = urlparse(url)
|
||||||
@@ -63,6 +111,7 @@ async def lifespan(app: FastAPI):
|
|||||||
kraken_engine.load_models()
|
kraken_engine.load_models()
|
||||||
load_spell_checker()
|
load_spell_checker()
|
||||||
_models_ready = True
|
_models_ready = True
|
||||||
|
metrics.ocr_models_ready.set(1)
|
||||||
logger.info("Startup complete — ready to accept requests")
|
logger.info("Startup complete — ready to accept requests")
|
||||||
|
|
||||||
yield
|
yield
|
||||||
@@ -72,6 +121,28 @@ async def lifespan(app: FastAPI):
|
|||||||
|
|
||||||
app = FastAPI(title="Familienarchiv OCR Service", lifespan=lifespan)
|
app = FastAPI(title="Familienarchiv OCR Service", lifespan=lifespan)
|
||||||
|
|
||||||
|
# /metrics is unauthenticated — relies on Docker-internal-network exposure
|
||||||
|
# only (CWE-200 risk if `ports:` ever maps 8000 to host). See
|
||||||
|
# docs/OBSERVABILITY.md §Internal-only endpoints for the Caddy block snippet.
|
||||||
|
Instrumentator(excluded_handlers=["/health", "/metrics"]).instrument(app).expose(app)
|
||||||
|
|
||||||
|
|
||||||
|
class MetricsPathFilter(logging.Filter):
|
||||||
|
"""Drop uvicorn.access entries for /metrics and /health to keep logs focused."""
|
||||||
|
|
||||||
|
_SUPPRESSED_PATHS = {"/metrics", "/health"}
|
||||||
|
|
||||||
|
def filter(self, record: logging.LogRecord) -> bool:
|
||||||
|
# uvicorn.access formats as: '%s - "%s %s HTTP/%s" %d'
|
||||||
|
if record.args and len(record.args) >= 3:
|
||||||
|
path = record.args[2]
|
||||||
|
if isinstance(path, str) and path in self._SUPPRESSED_PATHS:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
logging.getLogger("uvicorn.access").addFilter(MetricsPathFilter())
|
||||||
|
|
||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
def health():
|
def health():
|
||||||
@@ -99,7 +170,9 @@ async def run_ocr(request: OcrRequest):
|
|||||||
del img
|
del img
|
||||||
|
|
||||||
script_type = request.scriptType.upper()
|
script_type = request.scriptType.upper()
|
||||||
|
engine_name = "kraken" if script_type == "HANDWRITING_KURRENT" else "surya"
|
||||||
|
|
||||||
|
extract_started = time.monotonic()
|
||||||
if script_type == "HANDWRITING_KURRENT":
|
if script_type == "HANDWRITING_KURRENT":
|
||||||
if not kraken_engine.is_available():
|
if not kraken_engine.is_available():
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
@@ -111,11 +184,18 @@ async def run_ocr(request: OcrRequest):
|
|||||||
else:
|
else:
|
||||||
# TYPEWRITER, HANDWRITING_LATIN, UNKNOWN — all use Surya
|
# TYPEWRITER, HANDWRITING_LATIN, UNKNOWN — all use Surya
|
||||||
blocks = await asyncio.to_thread(surya_engine.extract_blocks, images, request.language)
|
blocks = await asyncio.to_thread(surya_engine.extract_blocks, images, request.language)
|
||||||
|
metrics.ocr_processing_seconds.labels(engine=engine_name).observe(
|
||||||
|
time.monotonic() - extract_started
|
||||||
|
)
|
||||||
|
|
||||||
|
metrics.ocr_jobs_total.labels(engine=engine_name, script_type=script_type).inc()
|
||||||
|
|
||||||
threshold = get_threshold(script_type)
|
threshold = get_threshold(script_type)
|
||||||
for block in blocks:
|
for block in blocks:
|
||||||
if block.get("words"):
|
words = block.get("words") or []
|
||||||
block["text"] = apply_confidence_markers(block["words"], threshold)
|
if words:
|
||||||
|
_observe_block_words(words, threshold)
|
||||||
|
block["text"] = apply_confidence_markers(words, threshold)
|
||||||
block.pop("words", None)
|
block.pop("words", None)
|
||||||
if script_type in _SPELL_CHECK_SCRIPT_TYPES:
|
if script_type in _SPELL_CHECK_SCRIPT_TYPES:
|
||||||
block["text"] = correct_text(block["text"])
|
block["text"] = correct_text(block["text"])
|
||||||
@@ -146,6 +226,9 @@ async def run_ocr_stream(request: OcrRequest):
|
|||||||
)
|
)
|
||||||
|
|
||||||
engine = kraken_engine if use_kraken else surya_engine
|
engine = kraken_engine if use_kraken else surya_engine
|
||||||
|
engine_name = "kraken" if use_kraken else "surya"
|
||||||
|
|
||||||
|
metrics.ocr_jobs_total.labels(engine=engine_name, script_type=script_type).inc()
|
||||||
|
|
||||||
if request.regions:
|
if request.regions:
|
||||||
# Guided mode: recognize only the user-drawn annotation regions
|
# Guided mode: recognize only the user-drawn annotation regions
|
||||||
@@ -176,12 +259,15 @@ async def run_ocr_stream(request: OcrRequest):
|
|||||||
image = await asyncio.to_thread(preprocess_page, image)
|
image = await asyncio.to_thread(preprocess_page, image)
|
||||||
blocks = []
|
blocks = []
|
||||||
sender_path = request.senderModelPath if use_kraken else None
|
sender_path = request.senderModelPath if use_kraken else None
|
||||||
|
engine_seconds = 0.0
|
||||||
for region in page_regions:
|
for region in page_regions:
|
||||||
|
region_started = time.monotonic()
|
||||||
text = await asyncio.to_thread(
|
text = await asyncio.to_thread(
|
||||||
engine.extract_region_text, image,
|
engine.extract_region_text, image,
|
||||||
region.x, region.y, region.width, region.height,
|
region.x, region.y, region.width, region.height,
|
||||||
sender_path,
|
sender_path,
|
||||||
)
|
)
|
||||||
|
engine_seconds += time.monotonic() - region_started
|
||||||
if script_type in _SPELL_CHECK_SCRIPT_TYPES:
|
if script_type in _SPELL_CHECK_SCRIPT_TYPES:
|
||||||
text = correct_text(text)
|
text = correct_text(text)
|
||||||
blocks.append({
|
blocks.append({
|
||||||
@@ -195,7 +281,11 @@ async def run_ocr_stream(request: OcrRequest):
|
|||||||
"annotationId": region.annotationId,
|
"annotationId": region.annotationId,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
metrics.ocr_processing_seconds.labels(engine=engine_name).observe(
|
||||||
|
engine_seconds
|
||||||
|
)
|
||||||
total_blocks += len(blocks)
|
total_blocks += len(blocks)
|
||||||
|
metrics.ocr_pages_total.labels(engine=engine_name).inc()
|
||||||
yield json.dumps({
|
yield json.dumps({
|
||||||
"type": "page",
|
"type": "page",
|
||||||
"pageNumber": page_idx,
|
"pageNumber": page_idx,
|
||||||
@@ -205,6 +295,7 @@ async def run_ocr_stream(request: OcrRequest):
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Guided OCR failed on page %d", page_idx)
|
logger.exception("Guided OCR failed on page %d", page_idx)
|
||||||
skipped_pages += 1
|
skipped_pages += 1
|
||||||
|
metrics.ocr_skipped_pages_total.inc()
|
||||||
yield json.dumps({
|
yield json.dumps({
|
||||||
"type": "error",
|
"type": "error",
|
||||||
"pageNumber": page_idx,
|
"pageNumber": page_idx,
|
||||||
@@ -238,18 +329,25 @@ async def run_ocr_stream(request: OcrRequest):
|
|||||||
yield json.dumps({"type": "preprocessing", "pageNumber": page_idx}) + "\n"
|
yield json.dumps({"type": "preprocessing", "pageNumber": page_idx}) + "\n"
|
||||||
image = await asyncio.to_thread(preprocess_page, image)
|
image = await asyncio.to_thread(preprocess_page, image)
|
||||||
sender_path = request.senderModelPath if use_kraken else None
|
sender_path = request.senderModelPath if use_kraken else None
|
||||||
|
page_started = time.monotonic()
|
||||||
blocks = await asyncio.to_thread(
|
blocks = await asyncio.to_thread(
|
||||||
engine.extract_page_blocks, image, page_idx, request.language, sender_path
|
engine.extract_page_blocks, image, page_idx, request.language, sender_path
|
||||||
)
|
)
|
||||||
|
metrics.ocr_processing_seconds.labels(engine=engine_name).observe(
|
||||||
|
time.monotonic() - page_started
|
||||||
|
)
|
||||||
|
|
||||||
for block in blocks:
|
for block in blocks:
|
||||||
if block.get("words"):
|
words = block.get("words") or []
|
||||||
block["text"] = apply_confidence_markers(block["words"], threshold)
|
if words:
|
||||||
|
_observe_block_words(words, threshold)
|
||||||
|
block["text"] = apply_confidence_markers(words, threshold)
|
||||||
block.pop("words", None)
|
block.pop("words", None)
|
||||||
if script_type in _SPELL_CHECK_SCRIPT_TYPES:
|
if script_type in _SPELL_CHECK_SCRIPT_TYPES:
|
||||||
block["text"] = correct_text(block["text"])
|
block["text"] = correct_text(block["text"])
|
||||||
|
|
||||||
total_blocks += len(blocks)
|
total_blocks += len(blocks)
|
||||||
|
metrics.ocr_pages_total.labels(engine=engine_name).inc()
|
||||||
yield json.dumps({
|
yield json.dumps({
|
||||||
"type": "page",
|
"type": "page",
|
||||||
"pageNumber": page_idx,
|
"pageNumber": page_idx,
|
||||||
@@ -259,6 +357,7 @@ async def run_ocr_stream(request: OcrRequest):
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("OCR failed on page %d", page_idx)
|
logger.exception("OCR failed on page %d", page_idx)
|
||||||
skipped_pages += 1
|
skipped_pages += 1
|
||||||
|
metrics.ocr_skipped_pages_total.inc()
|
||||||
yield json.dumps({
|
yield json.dumps({
|
||||||
"type": "error",
|
"type": "error",
|
||||||
"pageNumber": page_idx,
|
"pageNumber": page_idx,
|
||||||
@@ -438,8 +537,7 @@ async def train_model(
|
|||||||
|
|
||||||
return {"loss": None, "accuracy": accuracy, "cer": cer, "epochs": epochs}
|
return {"loss": None, "accuracy": accuracy, "cer": cer, "epochs": epochs}
|
||||||
|
|
||||||
result = await asyncio.to_thread(_run_training)
|
return await _record_training(lambda: asyncio.to_thread(_run_training), kind="recognition")
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
@app.post("/train-sender")
|
@app.post("/train-sender")
|
||||||
@@ -518,8 +616,9 @@ async def train_sender_model(
|
|||||||
|
|
||||||
return {"loss": None, "accuracy": accuracy, "cer": cer, "epochs": epochs}
|
return {"loss": None, "accuracy": accuracy, "cer": cer, "epochs": epochs}
|
||||||
|
|
||||||
result = await asyncio.to_thread(_run_sender_training)
|
return await _record_training(
|
||||||
return result
|
lambda: asyncio.to_thread(_run_sender_training), kind="recognition"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/segtrain")
|
@app.post("/segtrain")
|
||||||
@@ -628,8 +727,7 @@ async def segtrain_model(
|
|||||||
|
|
||||||
return {"loss": None, "accuracy": accuracy, "cer": cer, "epochs": epochs}
|
return {"loss": None, "accuracy": accuracy, "cer": cer, "epochs": epochs}
|
||||||
|
|
||||||
result = await asyncio.to_thread(_run_segtrain)
|
return await _record_training(lambda: asyncio.to_thread(_run_segtrain), kind="segmentation")
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
async def _download_and_convert_pdf(url: str) -> list[Image.Image]:
|
async def _download_and_convert_pdf(url: str) -> list[Image.Image]:
|
||||||
|
|||||||
92
ocr-service/metrics.py
Normal file
92
ocr-service/metrics.py
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
"""Prometheus metric definitions for the OCR service.
|
||||||
|
|
||||||
|
`build_metrics(registry)` returns a fresh `OcrMetrics` instance bound to the
|
||||||
|
given `CollectorRegistry`. Production code calls it once at module load with
|
||||||
|
the default `REGISTRY`; tests pass a per-test `CollectorRegistry()` to keep
|
||||||
|
counter values isolated between cases (decision #3 on issue #652).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class OcrMetrics:
|
||||||
|
"""Container for every custom OCR metric.
|
||||||
|
|
||||||
|
Counters and gauges are immutable references to `prometheus_client`
|
||||||
|
instances. Mutating them (`.inc()`, `.observe()`, `.set()`) is safe;
|
||||||
|
rebinding the field on the dataclass is not — use `build_metrics` to get
|
||||||
|
a new container.
|
||||||
|
"""
|
||||||
|
|
||||||
|
ocr_jobs_total: Counter
|
||||||
|
ocr_pages_total: Counter
|
||||||
|
ocr_skipped_pages_total: Counter
|
||||||
|
ocr_words_total: Counter
|
||||||
|
ocr_illegible_words_total: Counter
|
||||||
|
ocr_processing_seconds: Histogram
|
||||||
|
ocr_training_runs_total: Counter
|
||||||
|
ocr_model_accuracy: Gauge
|
||||||
|
ocr_models_ready: Gauge
|
||||||
|
|
||||||
|
|
||||||
|
def build_metrics(registry: CollectorRegistry) -> OcrMetrics:
|
||||||
|
"""Create one OcrMetrics instance bound to `registry`."""
|
||||||
|
return OcrMetrics(
|
||||||
|
ocr_jobs_total=Counter(
|
||||||
|
"ocr_jobs_total",
|
||||||
|
"Number of OCR jobs processed, labelled by engine and script type.",
|
||||||
|
["engine", "script_type"],
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
ocr_pages_total=Counter(
|
||||||
|
"ocr_pages_total",
|
||||||
|
"Number of pages successfully OCR'd, labelled by engine.",
|
||||||
|
["engine"],
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
ocr_skipped_pages_total=Counter(
|
||||||
|
"ocr_skipped_pages_total",
|
||||||
|
"Number of pages skipped because the OCR engine raised.",
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
ocr_words_total=Counter(
|
||||||
|
"ocr_words_total",
|
||||||
|
"Number of words recognized across all OCR blocks.",
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
ocr_illegible_words_total=Counter(
|
||||||
|
"ocr_illegible_words_total",
|
||||||
|
"Number of words below the confidence threshold "
|
||||||
|
"(replaced with [unleserlich]).",
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
ocr_processing_seconds=Histogram(
|
||||||
|
"ocr_processing_seconds",
|
||||||
|
"OCR processing time per page (streaming) or per document (non-streaming).",
|
||||||
|
["engine"],
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
ocr_training_runs_total=Counter(
|
||||||
|
"ocr_training_runs_total",
|
||||||
|
"Number of training runs, labelled by kind (recognition|segmentation) "
|
||||||
|
"and outcome (success|error).",
|
||||||
|
["kind", "outcome"],
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
ocr_model_accuracy=Gauge(
|
||||||
|
"ocr_model_accuracy",
|
||||||
|
"Latest model accuracy reported by a successful training run.",
|
||||||
|
["kind"],
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
ocr_models_ready=Gauge(
|
||||||
|
"ocr_models_ready",
|
||||||
|
"1 once the lifespan startup has finished loading models, 0 before.",
|
||||||
|
registry=registry,
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -10,3 +10,5 @@ pyvips>=2.2.0
|
|||||||
httpx==0.28.1
|
httpx==0.28.1
|
||||||
pyspellchecker==0.9.0
|
pyspellchecker==0.9.0
|
||||||
opencv-python-headless==4.11.0.86
|
opencv-python-headless==4.11.0.86
|
||||||
|
prometheus-fastapi-instrumentator==7.0.0
|
||||||
|
prometheus-client==0.25.0
|
||||||
|
|||||||
638
ocr-service/test_metrics.py
Normal file
638
ocr-service/test_metrics.py
Normal file
@@ -0,0 +1,638 @@
|
|||||||
|
"""Tests for Prometheus metrics exposed by the OCR service.
|
||||||
|
|
||||||
|
Each test that asserts on a counter/gauge value uses a fresh CollectorRegistry
|
||||||
|
(see decision #3 on issue #652) to keep the metrics isolated between tests.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import io
|
||||||
|
import zipfile
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from httpx import ASGITransport, AsyncClient
|
||||||
|
from PIL import Image
|
||||||
|
from prometheus_client import CollectorRegistry
|
||||||
|
|
||||||
|
from main import app
|
||||||
|
from metrics import build_metrics
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.asynccontextmanager
|
||||||
|
async def ocr_client(*, raise_app_exceptions: bool = True):
|
||||||
|
"""Yield an AsyncClient with model-loaders patched and _models_ready forced on.
|
||||||
|
|
||||||
|
The shared setup for almost every metrics test: stub the heavy lifecycle
|
||||||
|
hooks (kraken_engine.load_models, load_spell_checker), flip the readiness
|
||||||
|
flag so request handlers do not 503, and restore it afterwards.
|
||||||
|
"""
|
||||||
|
with patch("main.kraken_engine.load_models"), \
|
||||||
|
patch("main.load_spell_checker"):
|
||||||
|
transport = ASGITransport(app=app, raise_app_exceptions=raise_app_exceptions)
|
||||||
|
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||||
|
import main as main_module
|
||||||
|
main_module._models_ready = True
|
||||||
|
try:
|
||||||
|
yield client
|
||||||
|
finally:
|
||||||
|
main_module._models_ready = False
|
||||||
|
|
||||||
|
|
||||||
|
def _minimal_zip() -> bytes:
|
||||||
|
"""Return a ZIP containing one fake .xml so endpoint validation passes."""
|
||||||
|
buf = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(buf, "w") as zf:
|
||||||
|
zf.writestr("page_01.xml", "<PcGts/>")
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_training_result(accuracy: float = 0.91) -> dict:
|
||||||
|
return {"loss": None, "accuracy": accuracy, "cer": round(1 - accuracy, 4), "epochs": 5}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fresh_metrics(monkeypatch):
|
||||||
|
"""Replace the module-level `main.metrics` with one bound to a fresh registry."""
|
||||||
|
registry = CollectorRegistry()
|
||||||
|
test_metrics = build_metrics(registry)
|
||||||
|
monkeypatch.setattr("main.metrics", test_metrics)
|
||||||
|
return test_metrics
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_metrics_endpoint_returns_200():
|
||||||
|
"""`GET /metrics` returns 200 with Prometheus exposition content.
|
||||||
|
|
||||||
|
Uses the global REGISTRY by design — does NOT take the `fresh_metrics` fixture.
|
||||||
|
The `/metrics` endpoint is wired by `prometheus-fastapi-instrumentator`, which
|
||||||
|
binds to the default REGISTRY at app-construction time; swapping `main.metrics`
|
||||||
|
via the fixture would not redirect what `/metrics` exposes. This test only
|
||||||
|
asserts response shape (status code + content-type substring), not numeric
|
||||||
|
counter values, so cross-test state leakage cannot affect it.
|
||||||
|
"""
|
||||||
|
with patch("main.kraken_engine.load_models"), \
|
||||||
|
patch("main.load_spell_checker"):
|
||||||
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||||
|
response = await client.get("/metrics")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert "text/plain" in response.headers.get("content-type", "")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_metrics_includes_http_request_metrics_after_ocr_call():
|
||||||
|
"""After a request to /ocr, `/metrics` exposes auto-instrumented http_* metrics.
|
||||||
|
|
||||||
|
Uses the global REGISTRY by design — does NOT take the `fresh_metrics` fixture.
|
||||||
|
The `http_requests_total` / `http_request_duration_seconds` metrics live on
|
||||||
|
the instrumentator's default REGISTRY (not on `main.metrics`), so a fresh
|
||||||
|
CollectorRegistry would never see them. This test only asserts response shape
|
||||||
|
(substring presence in the exposition body), not numeric counter values, so
|
||||||
|
cross-test state leakage cannot affect it.
|
||||||
|
"""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100))]
|
||||||
|
mock_blocks = [{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0,
|
||||||
|
"polygon": None, "text": "hi", "words": []}]
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.surya_engine.extract_blocks", return_value=mock_blocks):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
ocr_response = await client.post("/ocr", json={
|
||||||
|
"pdfUrl": "http://minio/doc.pdf",
|
||||||
|
"scriptType": "TYPEWRITER",
|
||||||
|
"language": "de",
|
||||||
|
})
|
||||||
|
assert ocr_response.status_code == 200, ocr_response.text
|
||||||
|
|
||||||
|
metrics_response = await client.get("/metrics")
|
||||||
|
|
||||||
|
body = metrics_response.text
|
||||||
|
assert "http_requests_total" in body
|
||||||
|
assert "http_request_duration_seconds" in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_metrics_registers_all_custom_metrics_on_given_registry():
|
||||||
|
"""`build_metrics` returns an OcrMetrics bound to the supplied registry."""
|
||||||
|
registry = CollectorRegistry()
|
||||||
|
metrics = build_metrics(registry)
|
||||||
|
|
||||||
|
metric_names = {m.name for m in registry.collect()}
|
||||||
|
expected = {
|
||||||
|
"ocr_jobs",
|
||||||
|
"ocr_pages",
|
||||||
|
"ocr_skipped_pages",
|
||||||
|
"ocr_words",
|
||||||
|
"ocr_illegible_words",
|
||||||
|
"ocr_processing_seconds",
|
||||||
|
"ocr_training_runs",
|
||||||
|
"ocr_model_accuracy",
|
||||||
|
"ocr_models_ready",
|
||||||
|
}
|
||||||
|
assert expected <= metric_names, f"missing: {expected - metric_names}"
|
||||||
|
|
||||||
|
# A second registry yields a separate container — no shared state.
|
||||||
|
other_metrics = build_metrics(CollectorRegistry())
|
||||||
|
assert metrics is not other_metrics
|
||||||
|
|
||||||
|
|
||||||
|
async def _drive_ocr(client: AsyncClient, *, script_type: str) -> None:
|
||||||
|
"""Helper — fires /ocr with a single mocked page and asserts a 200."""
|
||||||
|
response = await client.post("/ocr", json={
|
||||||
|
"pdfUrl": "http://minio/doc.pdf",
|
||||||
|
"scriptType": script_type,
|
||||||
|
"language": "de",
|
||||||
|
})
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_jobs_total_incremented_with_kraken_engine_label_for_kurrent(fresh_metrics):
|
||||||
|
"""A /ocr call with HANDWRITING_KURRENT increments engine=kraken."""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100))]
|
||||||
|
mock_blocks = [{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0,
|
||||||
|
"polygon": None, "text": "hi", "words": []}]
|
||||||
|
|
||||||
|
with patch("main.correct_text", side_effect=lambda t: t), \
|
||||||
|
patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.kraken_engine.is_available", return_value=True), \
|
||||||
|
patch("main.kraken_engine.extract_blocks", return_value=mock_blocks):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
await _drive_ocr(client, script_type="HANDWRITING_KURRENT")
|
||||||
|
|
||||||
|
value = fresh_metrics.ocr_jobs_total.labels(
|
||||||
|
engine="kraken", script_type="HANDWRITING_KURRENT"
|
||||||
|
)._value.get()
|
||||||
|
assert value == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_jobs_total_incremented_with_surya_engine_label_for_typewriter(fresh_metrics):
|
||||||
|
"""A /ocr call with TYPEWRITER increments engine=surya."""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100))]
|
||||||
|
mock_blocks = [{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0,
|
||||||
|
"polygon": None, "text": "hi", "words": []}]
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.surya_engine.extract_blocks", return_value=mock_blocks):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
await _drive_ocr(client, script_type="TYPEWRITER")
|
||||||
|
|
||||||
|
value = fresh_metrics.ocr_jobs_total.labels(
|
||||||
|
engine="surya", script_type="TYPEWRITER"
|
||||||
|
)._value.get()
|
||||||
|
assert value == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_pages_total_incremented_once_per_page_in_stream(fresh_metrics):
|
||||||
|
"""The /ocr/stream generator increments ocr_pages_total per successful page."""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100)) for _ in range(3)]
|
||||||
|
mock_blocks = [{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0,
|
||||||
|
"polygon": None, "text": "hi", "words": []}]
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.surya_engine.extract_page_blocks", return_value=mock_blocks):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
async with client.stream("POST", "/ocr/stream", json={
|
||||||
|
"pdfUrl": "http://minio/doc.pdf",
|
||||||
|
"scriptType": "TYPEWRITER",
|
||||||
|
"language": "de",
|
||||||
|
}) as response:
|
||||||
|
assert response.status_code == 200
|
||||||
|
# Drain the stream so all per-page increments fire.
|
||||||
|
async for _ in response.aiter_lines():
|
||||||
|
pass
|
||||||
|
|
||||||
|
value = fresh_metrics.ocr_pages_total.labels(engine="surya")._value.get()
|
||||||
|
assert value == 3.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_skipped_pages_total_incremented_when_engine_raises_for_a_page(fresh_metrics):
|
||||||
|
"""When the engine raises on a page, ocr_skipped_pages_total bumps and the stream finishes."""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100)) for _ in range(2)]
|
||||||
|
good_blocks = [{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0,
|
||||||
|
"polygon": None, "text": "ok", "words": []}]
|
||||||
|
|
||||||
|
call_count = {"n": 0}
|
||||||
|
|
||||||
|
def extract_side_effect(*args, **kwargs):
|
||||||
|
call_count["n"] += 1
|
||||||
|
if call_count["n"] == 1:
|
||||||
|
raise RuntimeError("synthetic engine failure")
|
||||||
|
return good_blocks
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.surya_engine.extract_page_blocks", side_effect=extract_side_effect):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
async with client.stream("POST", "/ocr/stream", json={
|
||||||
|
"pdfUrl": "http://minio/doc.pdf",
|
||||||
|
"scriptType": "TYPEWRITER",
|
||||||
|
"language": "de",
|
||||||
|
}) as response:
|
||||||
|
assert response.status_code == 200
|
||||||
|
saw_error = False
|
||||||
|
async for line in response.aiter_lines():
|
||||||
|
if line and '"type": "error"' in line:
|
||||||
|
saw_error = True
|
||||||
|
assert saw_error
|
||||||
|
|
||||||
|
assert fresh_metrics.ocr_skipped_pages_total._value.get() == 1.0
|
||||||
|
# The second page still succeeds.
|
||||||
|
assert fresh_metrics.ocr_pages_total.labels(engine="surya")._value.get() == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_words_and_illegible_words_total_sum_across_blocks(fresh_metrics):
|
||||||
|
"""Counters reflect totals summed over every block in the request.
|
||||||
|
|
||||||
|
Threshold defaults to THRESHOLD_DEFAULT (0.3) for non-Kurrent scripts. Two
|
||||||
|
blocks: 3 words above + 2 words below threshold across blocks.
|
||||||
|
"""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100))]
|
||||||
|
mock_blocks = [
|
||||||
|
{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0,
|
||||||
|
"polygon": None, "text": "ignored",
|
||||||
|
"words": [{"text": "Lieber", "confidence": 0.9},
|
||||||
|
{"text": "Freund", "confidence": 0.1}]},
|
||||||
|
{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0,
|
||||||
|
"polygon": None, "text": "ignored",
|
||||||
|
"words": [{"text": "Gruss", "confidence": 0.8},
|
||||||
|
{"text": "verschmiert", "confidence": 0.05},
|
||||||
|
{"text": "Karl", "confidence": 0.95}]},
|
||||||
|
]
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.surya_engine.extract_blocks", return_value=mock_blocks):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
await _drive_ocr(client, script_type="TYPEWRITER")
|
||||||
|
|
||||||
|
assert fresh_metrics.ocr_words_total._value.get() == 5.0
|
||||||
|
assert fresh_metrics.ocr_illegible_words_total._value.get() == 2.0
|
||||||
|
|
||||||
|
|
||||||
|
def _histogram_count_sum(histogram, **labels) -> tuple[float, float]:
|
||||||
|
"""Read the per-label-set _count and _sum from a prometheus_client Histogram."""
|
||||||
|
child = histogram.labels(**labels)
|
||||||
|
return child._sum.get(), sum(b.get() for b in child._buckets)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_processing_seconds_histogram_observed_per_page_in_stream(fresh_metrics):
|
||||||
|
"""The streaming generator observes ocr_processing_seconds once per page."""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100)) for _ in range(2)]
|
||||||
|
mock_blocks = [{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0,
|
||||||
|
"polygon": None, "text": "ok", "words": []}]
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.surya_engine.extract_page_blocks", return_value=mock_blocks):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
async with client.stream("POST", "/ocr/stream", json={
|
||||||
|
"pdfUrl": "http://minio/doc.pdf",
|
||||||
|
"scriptType": "TYPEWRITER",
|
||||||
|
"language": "de",
|
||||||
|
}) as response:
|
||||||
|
assert response.status_code == 200
|
||||||
|
async for _ in response.aiter_lines():
|
||||||
|
pass
|
||||||
|
|
||||||
|
sum_seconds, count = _histogram_count_sum(
|
||||||
|
fresh_metrics.ocr_processing_seconds, engine="surya"
|
||||||
|
)
|
||||||
|
assert count == 2.0
|
||||||
|
assert sum_seconds >= 0.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_training_runs_total_incremented_with_recognition_success_label(fresh_metrics):
|
||||||
|
"""/train success increments ocr_training_runs_total{kind=recognition, outcome=success}."""
|
||||||
|
async def fake_to_thread(func, *args, **kwargs):
|
||||||
|
return _fake_training_result()
|
||||||
|
|
||||||
|
with patch("main.TRAINING_TOKEN", "secret-token"), \
|
||||||
|
patch("main._models_ready", True), \
|
||||||
|
patch("main.asyncio.to_thread", side_effect=fake_to_thread):
|
||||||
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||||
|
response = await client.post(
|
||||||
|
"/train",
|
||||||
|
files={"file": ("training.zip", _minimal_zip(), "application/zip")},
|
||||||
|
headers={"X-Training-Token": "secret-token"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert fresh_metrics.ocr_training_runs_total.labels(
|
||||||
|
kind="recognition", outcome="success"
|
||||||
|
)._value.get() == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_training_runs_total_incremented_with_recognition_error_label(fresh_metrics):
|
||||||
|
"""When ketos exits non-zero, the error counter bumps and the exception propagates.
|
||||||
|
|
||||||
|
Uses the narrowest available seam — `subprocess.run` returning a failing
|
||||||
|
CompletedProcess — instead of stubbing the asyncio.to_thread boundary,
|
||||||
|
so the test exercises the real _run_training error path.
|
||||||
|
"""
|
||||||
|
from subprocess import CompletedProcess
|
||||||
|
|
||||||
|
failing_proc = CompletedProcess(
|
||||||
|
args=["ketos"], returncode=1, stdout="", stderr="synthetic ketos failure"
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("main.TRAINING_TOKEN", "secret-token"), \
|
||||||
|
patch("main._models_ready", True), \
|
||||||
|
patch("main.subprocess.run", return_value=failing_proc):
|
||||||
|
transport = ASGITransport(app=app, raise_app_exceptions=False)
|
||||||
|
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||||
|
response = await client.post(
|
||||||
|
"/train",
|
||||||
|
files={"file": ("training.zip", _minimal_zip(), "application/zip")},
|
||||||
|
headers={"X-Training-Token": "secret-token"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 500
|
||||||
|
assert fresh_metrics.ocr_training_runs_total.labels(
|
||||||
|
kind="recognition", outcome="error"
|
||||||
|
)._value.get() == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_training_runs_total_incremented_with_segmentation_success_label(fresh_metrics):
|
||||||
|
"""/segtrain success increments ocr_training_runs_total{kind=segmentation, outcome=success}."""
|
||||||
|
async def fake_to_thread(func, *args, **kwargs):
|
||||||
|
return _fake_training_result(accuracy=0.83)
|
||||||
|
|
||||||
|
with patch("main.TRAINING_TOKEN", "secret-token"), \
|
||||||
|
patch("main._models_ready", True), \
|
||||||
|
patch("main.asyncio.to_thread", side_effect=fake_to_thread):
|
||||||
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||||
|
response = await client.post(
|
||||||
|
"/segtrain",
|
||||||
|
files={"file": ("training.zip", _minimal_zip(), "application/zip")},
|
||||||
|
headers={"X-Training-Token": "secret-token"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert fresh_metrics.ocr_training_runs_total.labels(
|
||||||
|
kind="segmentation", outcome="success"
|
||||||
|
)._value.get() == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_training_runs_total_incremented_with_recognition_success_label_for_train_sender(fresh_metrics):
|
||||||
|
"""/train-sender success increments ocr_training_runs_total{kind=recognition, outcome=success}."""
|
||||||
|
async def fake_to_thread(func, *args, **kwargs):
|
||||||
|
return _fake_training_result()
|
||||||
|
|
||||||
|
with patch("main.TRAINING_TOKEN", "secret-token"), \
|
||||||
|
patch("main._models_ready", True), \
|
||||||
|
patch("main.asyncio.to_thread", side_effect=fake_to_thread):
|
||||||
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||||
|
response = await client.post(
|
||||||
|
"/train-sender",
|
||||||
|
files={"file": ("training.zip", _minimal_zip(), "application/zip")},
|
||||||
|
data={"output_model_path": "/app/models/sender_test.mlmodel"},
|
||||||
|
headers={"X-Training-Token": "secret-token"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
assert fresh_metrics.ocr_training_runs_total.labels(
|
||||||
|
kind="recognition", outcome="success"
|
||||||
|
)._value.get() == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_model_accuracy_gauge_stays_default_when_training_returns_no_accuracy(fresh_metrics):
|
||||||
|
"""When the runner returns accuracy=None, ocr_model_accuracy must remain at its default 0."""
|
||||||
|
async def fake_to_thread(func, *args, **kwargs):
|
||||||
|
return {"loss": None, "accuracy": None, "cer": None, "epochs": 5}
|
||||||
|
|
||||||
|
with patch("main.TRAINING_TOKEN", "secret-token"), \
|
||||||
|
patch("main._models_ready", True), \
|
||||||
|
patch("main.asyncio.to_thread", side_effect=fake_to_thread):
|
||||||
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||||
|
response = await client.post(
|
||||||
|
"/train",
|
||||||
|
files={"file": ("training.zip", _minimal_zip(), "application/zip")},
|
||||||
|
headers={"X-Training-Token": "secret-token"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
# Gauge was never .set() — accessing the label child still creates it with default 0.0.
|
||||||
|
assert fresh_metrics.ocr_model_accuracy.labels(
|
||||||
|
kind="recognition"
|
||||||
|
)._value.get() == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_model_accuracy_gauge_set_per_kind_after_successful_training(fresh_metrics):
|
||||||
|
"""After /train and /segtrain succeed, ocr_model_accuracy{kind=...} reflects the result."""
|
||||||
|
recognition_accuracy = 0.917
|
||||||
|
segmentation_accuracy = 0.834
|
||||||
|
|
||||||
|
async def fake_recognition_to_thread(func, *args, **kwargs):
|
||||||
|
return _fake_training_result(accuracy=recognition_accuracy)
|
||||||
|
|
||||||
|
async def fake_segmentation_to_thread(func, *args, **kwargs):
|
||||||
|
return _fake_training_result(accuracy=segmentation_accuracy)
|
||||||
|
|
||||||
|
with patch("main.TRAINING_TOKEN", "secret-token"), \
|
||||||
|
patch("main._models_ready", True):
|
||||||
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||||
|
with patch("main.asyncio.to_thread", side_effect=fake_recognition_to_thread):
|
||||||
|
rec_resp = await client.post(
|
||||||
|
"/train",
|
||||||
|
files={"file": ("training.zip", _minimal_zip(), "application/zip")},
|
||||||
|
headers={"X-Training-Token": "secret-token"},
|
||||||
|
)
|
||||||
|
assert rec_resp.status_code == 200
|
||||||
|
with patch("main.asyncio.to_thread", side_effect=fake_segmentation_to_thread):
|
||||||
|
seg_resp = await client.post(
|
||||||
|
"/segtrain",
|
||||||
|
files={"file": ("training.zip", _minimal_zip(), "application/zip")},
|
||||||
|
headers={"X-Training-Token": "secret-token"},
|
||||||
|
)
|
||||||
|
assert seg_resp.status_code == 200
|
||||||
|
|
||||||
|
assert fresh_metrics.ocr_model_accuracy.labels(kind="recognition")._value.get() == pytest.approx(recognition_accuracy)
|
||||||
|
assert fresh_metrics.ocr_model_accuracy.labels(kind="segmentation")._value.get() == pytest.approx(segmentation_accuracy)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ocr_models_ready_gauge_defaults_to_zero():
|
||||||
|
"""A freshly-built OcrMetrics has ocr_models_ready=0 before lifespan runs."""
|
||||||
|
metrics = build_metrics(CollectorRegistry())
|
||||||
|
assert metrics.ocr_models_ready._value.get() == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_models_ready_gauge_is_one_after_lifespan_startup(fresh_metrics):
|
||||||
|
"""The lifespan flips ocr_models_ready to 1 once load_models / load_spell_checker return.
|
||||||
|
|
||||||
|
ASGITransport does not run lifespan by default, so the lifespan context
|
||||||
|
manager is driven directly to exercise the startup code path.
|
||||||
|
"""
|
||||||
|
assert fresh_metrics.ocr_models_ready._value.get() == 0.0
|
||||||
|
with patch("main.kraken_engine.load_models"), \
|
||||||
|
patch("main.load_spell_checker"):
|
||||||
|
async with app.router.lifespan_context(app):
|
||||||
|
assert fresh_metrics.ocr_models_ready._value.get() == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_processing_seconds_histogram_observed_per_page_in_guided_stream(fresh_metrics):
|
||||||
|
"""The guided streaming generator observes ocr_processing_seconds once per page."""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100)) for _ in range(2)]
|
||||||
|
regions = [
|
||||||
|
{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 0.5, "height": 0.5, "annotationId": "a1"},
|
||||||
|
{"pageNumber": 2, "x": 0.0, "y": 0.0, "width": 1.0, "height": 1.0, "annotationId": "a2"},
|
||||||
|
]
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.surya_engine.extract_region_text", return_value="text"):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
async with client.stream("POST", "/ocr/stream", json={
|
||||||
|
"pdfUrl": "http://minio/doc.pdf",
|
||||||
|
"scriptType": "TYPEWRITER",
|
||||||
|
"language": "de",
|
||||||
|
"regions": regions,
|
||||||
|
}) as response:
|
||||||
|
assert response.status_code == 200
|
||||||
|
async for _ in response.aiter_lines():
|
||||||
|
pass
|
||||||
|
|
||||||
|
sum_seconds, count = _histogram_count_sum(
|
||||||
|
fresh_metrics.ocr_processing_seconds, engine="surya"
|
||||||
|
)
|
||||||
|
assert count == 2.0
|
||||||
|
assert sum_seconds >= 0.0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_processing_seconds_histogram_excludes_spell_check_time_in_guided_stream(fresh_metrics):
|
||||||
|
"""The guided observation must time engine work only, not the spell-check pass.
|
||||||
|
|
||||||
|
Wall-clock bound rather than a structural `patch("main.time.monotonic")`:
|
||||||
|
the patched attribute is the *global* `time.monotonic`, which httpx and
|
||||||
|
asyncio also consume — they exhaust the deterministic sequence before the
|
||||||
|
request reaches the engine loop. Bound is sized against the failure mode,
|
||||||
|
not the noise floor: spell-check sleeps 0.05s × 2 regions = 0.1s, so a
|
||||||
|
timer that accidentally wrapped `correct_text` would observe >= 0.1s. The
|
||||||
|
0.09s ceiling catches that bug while leaving ~90ms of slack for slow CI
|
||||||
|
runners (engine work is instantaneous under the mock).
|
||||||
|
"""
|
||||||
|
mock_images = [Image.new("RGB", (100, 100))]
|
||||||
|
regions = [
|
||||||
|
{"pageNumber": 1, "x": 0.0, "y": 0.0, "width": 0.5, "height": 0.5, "annotationId": "a1"},
|
||||||
|
{"pageNumber": 1, "x": 0.5, "y": 0.0, "width": 0.5, "height": 0.5, "annotationId": "a2"},
|
||||||
|
]
|
||||||
|
|
||||||
|
def slow_correct(text):
|
||||||
|
import time as _time
|
||||||
|
_time.sleep(0.05)
|
||||||
|
return text
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new_callable=AsyncMock, return_value=mock_images), \
|
||||||
|
patch("main.preprocess_page", side_effect=lambda img: img), \
|
||||||
|
patch("main.kraken_engine.is_available", return_value=True), \
|
||||||
|
patch("main.kraken_engine.extract_region_text", return_value="text"), \
|
||||||
|
patch("main.correct_text", side_effect=slow_correct):
|
||||||
|
async with ocr_client() as client:
|
||||||
|
async with client.stream("POST", "/ocr/stream", json={
|
||||||
|
"pdfUrl": "http://minio/doc.pdf",
|
||||||
|
"scriptType": "HANDWRITING_KURRENT",
|
||||||
|
"language": "de",
|
||||||
|
"regions": regions,
|
||||||
|
}) as response:
|
||||||
|
assert response.status_code == 200
|
||||||
|
async for _ in response.aiter_lines():
|
||||||
|
pass
|
||||||
|
|
||||||
|
sum_seconds, _ = _histogram_count_sum(
|
||||||
|
fresh_metrics.ocr_processing_seconds, engine="kraken"
|
||||||
|
)
|
||||||
|
assert sum_seconds < 0.09, f"timing must exclude spell-check; got sum={sum_seconds}"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_ocr_jobs_total_not_incremented_when_pdf_download_fails_in_stream(fresh_metrics):
|
||||||
|
"""If `_download_and_convert_pdf` raises, ocr_jobs_total is NOT incremented.
|
||||||
|
|
||||||
|
Mirrors the /ocr endpoint's semantics: the counter only records jobs that
|
||||||
|
actually started OCR work, not failed downloads.
|
||||||
|
"""
|
||||||
|
async def fail_download(url):
|
||||||
|
raise RuntimeError("synthetic download failure")
|
||||||
|
|
||||||
|
with patch("main._download_and_convert_pdf", new=fail_download):
|
||||||
|
async with ocr_client(raise_app_exceptions=False) as client:
|
||||||
|
response = await client.post("/ocr/stream", json={
|
||||||
|
"pdfUrl": "http://minio/doc.pdf",
|
||||||
|
"scriptType": "TYPEWRITER",
|
||||||
|
"language": "de",
|
||||||
|
})
|
||||||
|
|
||||||
|
assert response.status_code == 500
|
||||||
|
assert fresh_metrics.ocr_jobs_total.labels(
|
||||||
|
engine="surya", script_type="TYPEWRITER"
|
||||||
|
)._value.get() == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_uvicorn_access_log_filter_fails_open_on_short_or_missing_args():
|
||||||
|
"""The filter must default-allow records when args is None or shorter than expected.
|
||||||
|
|
||||||
|
Locks in fail-open behavior: if uvicorn ever changes its format we keep
|
||||||
|
forwarding records to the handler rather than silently dropping logs.
|
||||||
|
"""
|
||||||
|
import logging as _logging
|
||||||
|
from main import MetricsPathFilter
|
||||||
|
|
||||||
|
filt = MetricsPathFilter()
|
||||||
|
none_record = _logging.LogRecord(
|
||||||
|
name="uvicorn.access", level=_logging.INFO, pathname="", lineno=0,
|
||||||
|
msg="some message", args=None, exc_info=None,
|
||||||
|
)
|
||||||
|
short_record = _logging.LogRecord(
|
||||||
|
name="uvicorn.access", level=_logging.INFO, pathname="", lineno=0,
|
||||||
|
msg="%s %s", args=("a", "b"), exc_info=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert filt.filter(none_record) is True
|
||||||
|
assert filt.filter(short_record) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_uvicorn_access_log_filter_skips_metrics_path():
|
||||||
|
"""The MetricsPathFilter drops uvicorn.access log records that target /metrics."""
|
||||||
|
import logging as _logging
|
||||||
|
from main import MetricsPathFilter
|
||||||
|
|
||||||
|
filt = MetricsPathFilter()
|
||||||
|
metrics_record = _logging.LogRecord(
|
||||||
|
name="uvicorn.access", level=_logging.INFO, pathname="", lineno=0,
|
||||||
|
msg='%s - "%s %s HTTP/%s" %d',
|
||||||
|
args=("127.0.0.1:1234", "GET", "/metrics", "1.1", 200),
|
||||||
|
exc_info=None,
|
||||||
|
)
|
||||||
|
health_record = _logging.LogRecord(
|
||||||
|
name="uvicorn.access", level=_logging.INFO, pathname="", lineno=0,
|
||||||
|
msg='%s - "%s %s HTTP/%s" %d',
|
||||||
|
args=("127.0.0.1:1234", "GET", "/health", "1.1", 200),
|
||||||
|
exc_info=None,
|
||||||
|
)
|
||||||
|
ocr_record = _logging.LogRecord(
|
||||||
|
name="uvicorn.access", level=_logging.INFO, pathname="", lineno=0,
|
||||||
|
msg='%s - "%s %s HTTP/%s" %d',
|
||||||
|
args=("127.0.0.1:1234", "POST", "/ocr", "1.1", 200),
|
||||||
|
exc_info=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert filt.filter(metrics_record) is False
|
||||||
|
assert filt.filter(health_record) is False
|
||||||
|
assert filt.filter(ocr_record) is True
|
||||||
Reference in New Issue
Block a user