feat(importing): add orchestrator, wire admin, retire raw-spreadsheet path
CanonicalImportOrchestrator runs the four loaders in an explicit dependency
DAG (TagTree -> PersonRegister -> PersonTree -> Document), owns the async
runner + ImportStatus state machine the admin UI consumes, smoke-checks all
four artifacts are present before starting (fail-fast IMPORT_FAILED_ARTIFACT
rather than a half-run), and fails closed on a malformed artifact.
AdminController now depends on the orchestrator; the {state, statusCode,
processed, skippedFiles, skipped} response shape is unchanged so
ImportStatusCard.svelte keeps working.
Deletes the legacy MassImportService (positional @Value app.import.col.*,
ISO-only parseDate, Java name classification) and the ODS/XXE
XxeSafeXmlParser path now that the loaders cover them — the security guards
were ported to DocumentImporter first (previous commit). Replaces the
positional column config in application.yaml with the canonical artifact
directory.
Refs #669
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,94 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Runs the four canonical loaders in their real dependency order — encoded explicitly
|
||||
* here, not implied by call order — and owns the async runner plus the {@link ImportStatus}
|
||||
* state machine the admin UI consumes. The orchestrator smoke-checks that all four
|
||||
* artifacts are present before starting, failing fast rather than half-loading tags but no
|
||||
* documents. A malformed artifact (a loader throwing) sets {@code FAILED}; an individual
|
||||
* bad file is surfaced through the {@link ImportStatus.SkippedFile} mechanism instead.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class CanonicalImportOrchestrator {
|
||||
|
||||
private static final String TAG_TREE_ARTIFACT = "canonical-tag-tree.xlsx";
|
||||
private static final String PERSONS_ARTIFACT = "canonical-persons.xlsx";
|
||||
private static final String PERSONS_TREE_ARTIFACT = "canonical-persons-tree.json";
|
||||
private static final String DOCUMENTS_ARTIFACT = "canonical-documents.xlsx";
|
||||
|
||||
private final TagTreeImporter tagTreeImporter;
|
||||
private final PersonRegisterImporter personRegisterImporter;
|
||||
private final PersonTreeImporter personTreeImporter;
|
||||
private final DocumentImporter documentImporter;
|
||||
|
||||
@Value("${app.import.dir:/import}")
|
||||
private String canonicalDir;
|
||||
|
||||
private volatile ImportStatus currentStatus = new ImportStatus(
|
||||
ImportStatus.State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
|
||||
public ImportStatus getStatus() {
|
||||
return currentStatus;
|
||||
}
|
||||
|
||||
@Async
|
||||
public void runImportAsync() {
|
||||
if (currentStatus.state() == ImportStatus.State.RUNNING) {
|
||||
throw DomainException.conflict(ErrorCode.IMPORT_ALREADY_RUNNING, "A mass import is already in progress");
|
||||
}
|
||||
runImport();
|
||||
}
|
||||
|
||||
/** Synchronous entry point — wrapped by {@link #runImportAsync()} and called directly in tests. */
|
||||
void runImport() {
|
||||
currentStatus = new ImportStatus(ImportStatus.State.RUNNING, "IMPORT_RUNNING",
|
||||
"Import läuft...", 0, List.of(), LocalDateTime.now());
|
||||
try {
|
||||
File tagTree = requireArtifact(TAG_TREE_ARTIFACT);
|
||||
File persons = requireArtifact(PERSONS_ARTIFACT);
|
||||
File personsTree = requireArtifact(PERSONS_TREE_ARTIFACT);
|
||||
File documents = requireArtifact(DOCUMENTS_ARTIFACT);
|
||||
|
||||
// Dependency DAG: documents need persons + tags; the tree needs persons.
|
||||
tagTreeImporter.load(tagTree);
|
||||
personRegisterImporter.load(persons);
|
||||
personTreeImporter.load(personsTree);
|
||||
DocumentImporter.LoadResult result = documentImporter.load(documents);
|
||||
|
||||
currentStatus = new ImportStatus(ImportStatus.State.DONE, "IMPORT_DONE",
|
||||
"Import abgeschlossen. " + result.processed() + " Dokumente verarbeitet.",
|
||||
result.processed(), result.skippedFiles(), currentStatus.startedAt());
|
||||
} catch (DomainException e) {
|
||||
log.error("Canonical import failed: {}", e.getMessage());
|
||||
currentStatus = new ImportStatus(ImportStatus.State.FAILED, "IMPORT_FAILED_ARTIFACT",
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
} catch (Exception e) {
|
||||
log.error("Canonical import failed", e);
|
||||
currentStatus = new ImportStatus(ImportStatus.State.FAILED, "IMPORT_FAILED_INTERNAL",
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
}
|
||||
}
|
||||
|
||||
private File requireArtifact(String name) {
|
||||
File artifact = new File(canonicalDir, name);
|
||||
if (!artifact.isFile()) {
|
||||
throw DomainException.badRequest(ErrorCode.IMPORT_ARTIFACT_INVALID,
|
||||
"Missing canonical artifact: " + name);
|
||||
}
|
||||
return artifact;
|
||||
}
|
||||
}
|
||||
@@ -1,509 +0,0 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.poi.ss.usermodel.*;
|
||||
import java.util.Objects;
|
||||
import org.raddatz.familienarchiv.exception.DomainException;
|
||||
import org.raddatz.familienarchiv.exception.ErrorCode;
|
||||
import org.raddatz.familienarchiv.document.Document;
|
||||
import org.raddatz.familienarchiv.document.DocumentService;
|
||||
import org.raddatz.familienarchiv.document.DocumentStatus;
|
||||
import org.raddatz.familienarchiv.document.ThumbnailAsyncRunner;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.tag.Tag;
|
||||
import org.raddatz.familienarchiv.person.Person;
|
||||
import org.raddatz.familienarchiv.person.PersonNameParser;
|
||||
import org.raddatz.familienarchiv.person.PersonService;
|
||||
import org.raddatz.familienarchiv.tag.TagService;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.zip.ZipFile;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class MassImportService {
|
||||
|
||||
public enum State { IDLE, RUNNING, DONE, FAILED }
|
||||
|
||||
public enum SkipReason {
|
||||
INVALID_FILENAME_PATH_TRAVERSAL,
|
||||
INVALID_PDF_SIGNATURE,
|
||||
FILE_READ_ERROR,
|
||||
ALREADY_EXISTS,
|
||||
S3_UPLOAD_FAILED
|
||||
}
|
||||
|
||||
public record SkippedFile(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) String filename,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) SkipReason reason
|
||||
) {}
|
||||
|
||||
public record ImportStatus(
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) State state,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) String statusCode,
|
||||
@JsonIgnore String message,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) int processed,
|
||||
@Schema(requiredMode = Schema.RequiredMode.REQUIRED) List<SkippedFile> skippedFiles,
|
||||
LocalDateTime startedAt
|
||||
) {
|
||||
// Note: @Schema on a record accessor method is not picked up by SpringDoc; the
|
||||
// "skipped" count is a computed convenience field derived from skippedFiles.size().
|
||||
@JsonProperty("skipped")
|
||||
public int skipped() { return skippedFiles.size(); }
|
||||
|
||||
/** Defensive-copy constructor — callers cannot mutate the stored list after construction. */
|
||||
public ImportStatus {
|
||||
skippedFiles = List.copyOf(skippedFiles);
|
||||
}
|
||||
}
|
||||
|
||||
record ProcessResult(int processed, List<SkippedFile> skippedFiles) {}
|
||||
|
||||
private volatile ImportStatus currentStatus = new ImportStatus(State.IDLE, "IMPORT_IDLE", "Kein Import gestartet.", 0, List.of(), null);
|
||||
|
||||
public ImportStatus getStatus() {
|
||||
return currentStatus;
|
||||
}
|
||||
|
||||
private final DocumentService documentService;
|
||||
private final PersonService personService;
|
||||
private final TagService tagService;
|
||||
private final S3Client s3Client;
|
||||
private final ThumbnailAsyncRunner thumbnailAsyncRunner;
|
||||
|
||||
@Value("${app.s3.bucket}")
|
||||
private String bucketName;
|
||||
|
||||
@Value("${app.import.col.index:0}")
|
||||
private int colIndex;
|
||||
|
||||
@Value("${app.import.col.box:1}")
|
||||
private int colBox;
|
||||
|
||||
@Value("${app.import.col.folder:2}")
|
||||
private int colFolder;
|
||||
|
||||
@Value("${app.import.col.sender:3}")
|
||||
private int colSender;
|
||||
|
||||
@Value("${app.import.col.receivers:5}")
|
||||
private int colReceivers;
|
||||
|
||||
@Value("${app.import.col.date:7}")
|
||||
private int colDate;
|
||||
|
||||
@Value("${app.import.col.location:9}")
|
||||
private int colLocation;
|
||||
|
||||
@Value("${app.import.col.tags:10}")
|
||||
private int colTags;
|
||||
|
||||
@Value("${app.import.col.summary:11}")
|
||||
private int colSummary;
|
||||
|
||||
@Value("${app.import.col.transcription:13}")
|
||||
private int colTranscription;
|
||||
|
||||
@Value("${app.import.dir:/import}")
|
||||
private String importDir;
|
||||
|
||||
private static final DateTimeFormatter GERMAN_DATE = DateTimeFormatter.ofPattern("d. MMMM yyyy", Locale.GERMAN);
|
||||
|
||||
// ODS XML namespaces
|
||||
private static final String NS_TABLE = "urn:oasis:names:tc:opendocument:xmlns:table:1.0";
|
||||
private static final String NS_TEXT = "urn:oasis:names:tc:opendocument:xmlns:text:1.0";
|
||||
|
||||
// We only need up to this many columns; caps repeated-empty-cell expansion
|
||||
private static final int MAX_COLS = 20;
|
||||
|
||||
@Async
|
||||
public void runImportAsync() {
|
||||
if (currentStatus.state() == State.RUNNING) {
|
||||
throw DomainException.conflict(ErrorCode.IMPORT_ALREADY_RUNNING, "A mass import is already in progress");
|
||||
}
|
||||
currentStatus = new ImportStatus(State.RUNNING, "IMPORT_RUNNING", "Import läuft...", 0, List.of(), LocalDateTime.now());
|
||||
try {
|
||||
File spreadsheet = findSpreadsheetFile();
|
||||
log.info("Starte Massenimport aus: {}", spreadsheet.getAbsolutePath());
|
||||
ProcessResult result = processRows(readSpreadsheet(spreadsheet));
|
||||
currentStatus = new ImportStatus(State.DONE, "IMPORT_DONE",
|
||||
"Import abgeschlossen. " + result.processed() + " Dokumente verarbeitet.",
|
||||
result.processed(), result.skippedFiles(), currentStatus.startedAt());
|
||||
} catch (NoSpreadsheetException e) {
|
||||
log.error("Massenimport fehlgeschlagen: keine Tabellendatei", e);
|
||||
currentStatus = new ImportStatus(State.FAILED, "IMPORT_FAILED_NO_SPREADSHEET",
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
} catch (Exception e) {
|
||||
log.error("Massenimport fehlgeschlagen", e);
|
||||
currentStatus = new ImportStatus(State.FAILED, "IMPORT_FAILED_INTERNAL",
|
||||
"Fehler: " + e.getMessage(), 0, List.of(), currentStatus.startedAt());
|
||||
}
|
||||
}
|
||||
|
||||
private static class NoSpreadsheetException extends RuntimeException {
|
||||
NoSpreadsheetException(String message) { super(message); }
|
||||
}
|
||||
|
||||
private File findSpreadsheetFile() throws IOException {
|
||||
try (Stream<Path> files = Files.list(Paths.get(importDir))) {
|
||||
return files
|
||||
.filter(p -> {
|
||||
String name = p.toString().toLowerCase();
|
||||
return name.endsWith(".ods") || name.endsWith(".xlsx") || name.endsWith(".xls");
|
||||
})
|
||||
.findFirst()
|
||||
.orElseThrow(() -> new NoSpreadsheetException(
|
||||
"Keine Tabellendatei (.ods/.xlsx/.xls) in " + importDir + " gefunden!"))
|
||||
.toFile();
|
||||
}
|
||||
}
|
||||
|
||||
// --- Spreadsheet reading (format-specific, produces neutral List<List<String>>) ---
|
||||
|
||||
private List<List<String>> readSpreadsheet(File file) throws Exception {
|
||||
String name = file.getName().toLowerCase();
|
||||
if (name.endsWith(".ods")) {
|
||||
return readOds(file);
|
||||
}
|
||||
return readXlsx(file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an ODS file by parsing its content.xml directly (no extra library needed).
|
||||
* ODS is a ZIP archive; content.xml holds the spreadsheet data as XML.
|
||||
*/
|
||||
List<List<String>> readOds(File file) throws Exception {
|
||||
List<List<String>> result = new ArrayList<>();
|
||||
|
||||
try (ZipFile zip = new ZipFile(file)) {
|
||||
var entry = zip.getEntry("content.xml");
|
||||
if (entry == null) throw new RuntimeException("Ungültige ODS-Datei: content.xml fehlt");
|
||||
|
||||
var factory = XxeSafeXmlParser.hardenedFactory();
|
||||
factory.setNamespaceAware(true);
|
||||
var builder = factory.newDocumentBuilder();
|
||||
var doc = builder.parse(zip.getInputStream(entry));
|
||||
|
||||
NodeList tables = doc.getElementsByTagNameNS(NS_TABLE, "table");
|
||||
if (tables.getLength() == 0) return result;
|
||||
|
||||
var table = (Element) tables.item(0);
|
||||
NodeList rows = table.getElementsByTagNameNS(NS_TABLE, "table-row");
|
||||
|
||||
for (int i = 0; i < rows.getLength(); i++) {
|
||||
var row = (Element) rows.item(i);
|
||||
List<String> rowData = new ArrayList<>();
|
||||
NodeList cells = row.getElementsByTagNameNS(NS_TABLE, "table-cell");
|
||||
|
||||
for (int j = 0; j < cells.getLength() && rowData.size() < MAX_COLS; j++) {
|
||||
var cell = (Element) cells.item(j);
|
||||
|
||||
// Read the display text (first <text:p>)
|
||||
String value = "";
|
||||
NodeList textNodes = cell.getElementsByTagNameNS(NS_TEXT, "p");
|
||||
if (textNodes.getLength() > 0) {
|
||||
value = textNodes.item(0).getTextContent().trim();
|
||||
}
|
||||
|
||||
// Expand number-columns-repeated (capped at MAX_COLS)
|
||||
String repeatAttr = cell.getAttributeNS(NS_TABLE, "number-columns-repeated");
|
||||
int repeat = repeatAttr.isEmpty() ? 1 : Integer.parseInt(repeatAttr);
|
||||
repeat = Math.min(repeat, MAX_COLS - rowData.size());
|
||||
|
||||
for (int r = 0; r < repeat; r++) {
|
||||
rowData.add(value);
|
||||
}
|
||||
}
|
||||
result.add(rowData);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Reads an XLSX/XLS file using Apache POI. Converts all cells to strings. */
|
||||
private List<List<String>> readXlsx(File file) throws Exception {
|
||||
List<List<String>> result = new ArrayList<>();
|
||||
try (FileInputStream fis = new FileInputStream(file);
|
||||
Workbook workbook = WorkbookFactory.create(fis)) {
|
||||
|
||||
Sheet sheet = workbook.getSheetAt(0);
|
||||
for (int i = 0; i <= sheet.getLastRowNum(); i++) {
|
||||
Row row = sheet.getRow(i);
|
||||
List<String> rowData = new ArrayList<>();
|
||||
if (row != null) {
|
||||
for (int j = 0; j < MAX_COLS; j++) {
|
||||
rowData.add(xlsxCellToString(row.getCell(j)));
|
||||
}
|
||||
}
|
||||
result.add(rowData);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String xlsxCellToString(Cell cell) {
|
||||
if (cell == null) return "";
|
||||
return switch (cell.getCellType()) {
|
||||
case STRING -> cell.getStringCellValue();
|
||||
case NUMERIC -> {
|
||||
if (DateUtil.isCellDateFormatted(cell)) {
|
||||
yield cell.getLocalDateTimeCellValue().toLocalDate().toString(); // ISO
|
||||
}
|
||||
yield String.valueOf((int) cell.getNumericCellValue());
|
||||
}
|
||||
case BOOLEAN -> String.valueOf(cell.getBooleanCellValue());
|
||||
default -> "";
|
||||
};
|
||||
}
|
||||
|
||||
// --- Import logic (works on neutral List<String> rows) ---
|
||||
|
||||
private ProcessResult processRows(List<List<String>> rows) {
|
||||
int processed = 0;
|
||||
List<SkippedFile> skippedFiles = new ArrayList<>();
|
||||
|
||||
for (int i = 1; i < rows.size(); i++) { // skip header row
|
||||
List<String> cells = rows.get(i);
|
||||
String index = getCell(cells, colIndex);
|
||||
if (index.isBlank()) continue;
|
||||
|
||||
String filename = index.contains(".") ? index : index + ".pdf";
|
||||
if (!isValidImportFilename(filename)) {
|
||||
log.warn("Skipping import row {}: filename rejected — {}", i, filename);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.INVALID_FILENAME_PATH_TRAVERSAL));
|
||||
continue;
|
||||
}
|
||||
Optional<File> fileOnDisk = findFileRecursive(filename);
|
||||
if (fileOnDisk.isEmpty()) {
|
||||
log.warn("Datei nicht gefunden, importiere nur Metadaten: {}", filename);
|
||||
}
|
||||
|
||||
if (fileOnDisk.isPresent()) {
|
||||
try {
|
||||
if (!isPdfMagicBytes(fileOnDisk.get())) {
|
||||
log.warn("Überspringe {}: Datei beginnt nicht mit %PDF-Signatur", filename);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.INVALID_PDF_SIGNATURE));
|
||||
continue;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Fehler beim Prüfen der Magic-Bytes für {}", filename, e);
|
||||
skippedFiles.add(new SkippedFile(filename, SkipReason.FILE_READ_ERROR));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
Optional<SkipReason> skipReason = importSingleDocument(cells, fileOnDisk, filename, index);
|
||||
if (skipReason.isPresent()) {
|
||||
skippedFiles.add(new SkippedFile(filename, skipReason.get()));
|
||||
} else {
|
||||
processed++;
|
||||
}
|
||||
}
|
||||
return new ProcessResult(processed, skippedFiles);
|
||||
}
|
||||
|
||||
private boolean isValidImportFilename(String filename) {
|
||||
if (filename == null || filename.isBlank()) return false;
|
||||
if (filename.contains("/")) return false;
|
||||
if (filename.contains("\\")) return false;
|
||||
if (filename.contains("∕")) return false; // U+2215 DIVISION SLASH
|
||||
if (filename.contains("/")) return false; // U+FF0F FULLWIDTH SOLIDUS
|
||||
if (filename.contains("⧵")) return false; // U+29F5 REVERSE SOLIDUS OPERATOR
|
||||
if (filename.contains("..")) return false;
|
||||
if (filename.equals(".")) return false;
|
||||
if (filename.contains("\0")) return false;
|
||||
// Paths.get() is safe here on Linux for all inputs that passed the checks above;
|
||||
// it may throw InvalidPathException for OS-specific illegal chars on Windows,
|
||||
// but those are not reachable in production.
|
||||
if (Paths.get(filename).isAbsolute()) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// package-private: Mockito spy in tests can override to inject IOException
|
||||
InputStream openFileStream(File file) throws IOException {
|
||||
return new FileInputStream(file);
|
||||
}
|
||||
|
||||
private boolean isPdfMagicBytes(File file) throws IOException {
|
||||
try (InputStream is = openFileStream(file)) {
|
||||
byte[] header = is.readNBytes(4);
|
||||
return header.length == 4
|
||||
&& header[0] == 0x25 // %
|
||||
&& header[1] == 0x50 // P
|
||||
&& header[2] == 0x44 // D
|
||||
&& header[3] == 0x46; // F
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports a single document row.
|
||||
*
|
||||
* @return empty Optional on success; an Optional containing the skip reason on failure/skip.
|
||||
*/
|
||||
@Transactional
|
||||
protected Optional<SkipReason> importSingleDocument(List<String> cells, Optional<File> file, String originalFilename, String index) {
|
||||
Optional<Document> existing = documentService.findByOriginalFilename(originalFilename);
|
||||
if (existing.isPresent() && existing.get().getStatus() != DocumentStatus.PLACEHOLDER) {
|
||||
log.info("Dokument {} existiert bereits, überspringe.", originalFilename);
|
||||
return Optional.of(SkipReason.ALREADY_EXISTS);
|
||||
}
|
||||
|
||||
String archiveBox = getCell(cells, colBox);
|
||||
String archiveFolder = getCell(cells, colFolder);
|
||||
String senderRaw = getCell(cells, colSender);
|
||||
String receiversRaw = getCell(cells, colReceivers);
|
||||
LocalDate date = parseDate(getCell(cells, colDate));
|
||||
String location = getCell(cells, colLocation);
|
||||
String tagRaw = getCell(cells, colTags);
|
||||
String summary = getCell(cells, colSummary);
|
||||
String transcription = getCell(cells, colTranscription);
|
||||
|
||||
String s3Key = null;
|
||||
String contentType = null;
|
||||
DocumentStatus status = DocumentStatus.PLACEHOLDER;
|
||||
|
||||
if (file.isPresent()) {
|
||||
try {
|
||||
contentType = Files.probeContentType(file.get().toPath());
|
||||
} catch (IOException e) {
|
||||
contentType = null;
|
||||
}
|
||||
if (contentType == null) contentType = "application/octet-stream";
|
||||
|
||||
s3Key = "documents/" + UUID.randomUUID() + "_" + file.get().getName();
|
||||
try {
|
||||
s3Client.putObject(PutObjectRequest.builder()
|
||||
.bucket(bucketName)
|
||||
.key(s3Key)
|
||||
.contentType(contentType)
|
||||
.build(),
|
||||
RequestBody.fromFile(file.get()));
|
||||
status = DocumentStatus.UPLOADED;
|
||||
} catch (Exception e) {
|
||||
log.error("S3 Upload Fehler für {}", file.get().getName(), e);
|
||||
return Optional.of(SkipReason.S3_UPLOAD_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
Person sender = senderRaw.isBlank() ? null : findOrCreatePerson(senderRaw);
|
||||
List<Person> receivers = PersonNameParser.parseReceivers(receiversRaw).stream()
|
||||
.map(this::findOrCreatePerson)
|
||||
.filter(Objects::nonNull)
|
||||
.toList();
|
||||
|
||||
Tag tag = null;
|
||||
if (!tagRaw.isBlank()) {
|
||||
tag = tagService.findOrCreate(tagRaw);
|
||||
}
|
||||
|
||||
Document doc = existing.orElse(Document.builder()
|
||||
.originalFilename(originalFilename)
|
||||
.build());
|
||||
|
||||
// Heuristic: mark as complete if at least one key field is present in the spreadsheet row
|
||||
boolean metadataComplete = date != null || !senderRaw.isBlank() || !receiversRaw.isBlank();
|
||||
|
||||
doc.setTitle(buildTitle(index, date, location));
|
||||
doc.setFilePath(s3Key);
|
||||
doc.setContentType(contentType);
|
||||
doc.setStatus(status);
|
||||
doc.setArchiveBox(archiveBox.isBlank() ? null : archiveBox);
|
||||
doc.setArchiveFolder(archiveFolder.isBlank() ? null : archiveFolder);
|
||||
doc.setDocumentDate(date);
|
||||
doc.setLocation(location.isBlank() ? null : location);
|
||||
doc.setSummary(summary.isBlank() ? null : summary);
|
||||
doc.setTranscription(transcription.isBlank() ? null : transcription);
|
||||
doc.setSender(sender);
|
||||
doc.getReceivers().addAll(receivers);
|
||||
if (tag != null) doc.getTags().add(tag);
|
||||
doc.setMetadataComplete(metadataComplete);
|
||||
|
||||
Document saved = documentService.save(doc);
|
||||
if (file.isPresent()) {
|
||||
thumbnailAsyncRunner.dispatchAfterCommit(saved.getId());
|
||||
}
|
||||
log.info("Importiert{}: {}", file.isEmpty() ? " (nur Metadaten)" : "", originalFilename);
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
private String getCell(List<String> cells, int col) {
|
||||
if (col >= cells.size()) return "";
|
||||
String val = cells.get(col);
|
||||
return val == null ? "" : val.trim();
|
||||
}
|
||||
|
||||
private LocalDate parseDate(String value) {
|
||||
if (value == null || value.isBlank()) return null;
|
||||
try {
|
||||
return LocalDate.parse(value.trim());
|
||||
} catch (DateTimeParseException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String buildTitle(String index, LocalDate date, String location) {
|
||||
StringBuilder sb = new StringBuilder(index);
|
||||
if (date != null) {
|
||||
sb.append(" \u2013 ").append(date.format(GERMAN_DATE));
|
||||
}
|
||||
if (location != null && !location.isBlank()) {
|
||||
sb.append(" \u2013 ").append(location);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private Person findOrCreatePerson(String rawName) {
|
||||
return personService.findOrCreateByAlias(rawName);
|
||||
}
|
||||
|
||||
private Optional<File> findFileRecursive(String filename) {
|
||||
File baseDir = new File(importDir);
|
||||
try (Stream<Path> walk = Files.walk(baseDir.toPath())) {
|
||||
Optional<Path> match = walk.filter(p -> !Files.isDirectory(p))
|
||||
.filter(p -> p.getFileName().toString().equals(filename))
|
||||
.findFirst();
|
||||
if (match.isEmpty()) return Optional.empty();
|
||||
File candidate = match.get().toFile();
|
||||
String baseDirCanonical = baseDir.getCanonicalPath();
|
||||
if (!candidate.getCanonicalPath().startsWith(baseDirCanonical + File.separator)) {
|
||||
throw DomainException.internal(ErrorCode.INTERNAL_ERROR, "Path escape detected: " + candidate);
|
||||
}
|
||||
return Optional.of(candidate);
|
||||
} catch (IOException e) {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
package org.raddatz.familienarchiv.importing;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
|
||||
class XxeSafeXmlParser {
|
||||
|
||||
private XxeSafeXmlParser() {}
|
||||
|
||||
static DocumentBuilderFactory hardenedFactory() throws ParserConfigurationException {
|
||||
var factory = DocumentBuilderFactory.newInstance();
|
||||
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
|
||||
factory.setFeature("http://xml.org/sax/features/external-general-entities", false);
|
||||
factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
|
||||
factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
|
||||
factory.setXIncludeAware(false);
|
||||
factory.setExpandEntityReferences(false);
|
||||
return factory;
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,8 @@ import org.raddatz.familienarchiv.security.Permission;
|
||||
import org.raddatz.familienarchiv.security.RequirePermission;
|
||||
import org.raddatz.familienarchiv.document.DocumentService;
|
||||
import org.raddatz.familienarchiv.document.DocumentVersionService;
|
||||
import org.raddatz.familienarchiv.importing.MassImportService;
|
||||
import org.raddatz.familienarchiv.importing.CanonicalImportOrchestrator;
|
||||
import org.raddatz.familienarchiv.importing.ImportStatus;
|
||||
import org.raddatz.familienarchiv.document.ThumbnailBackfillService;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
@@ -21,20 +22,20 @@ import lombok.RequiredArgsConstructor;
|
||||
@RequiredArgsConstructor
|
||||
public class AdminController {
|
||||
|
||||
private final MassImportService massImportService;
|
||||
private final CanonicalImportOrchestrator importOrchestrator;
|
||||
private final DocumentService documentService;
|
||||
private final DocumentVersionService documentVersionService;
|
||||
private final ThumbnailBackfillService thumbnailBackfillService;
|
||||
|
||||
@PostMapping("/trigger-import")
|
||||
public ResponseEntity<MassImportService.ImportStatus> triggerMassImport() {
|
||||
massImportService.runImportAsync();
|
||||
return ResponseEntity.accepted().body(massImportService.getStatus());
|
||||
public ResponseEntity<ImportStatus> triggerMassImport() {
|
||||
importOrchestrator.runImportAsync();
|
||||
return ResponseEntity.accepted().body(importOrchestrator.getStatus());
|
||||
}
|
||||
|
||||
@GetMapping("/import-status")
|
||||
public ResponseEntity<MassImportService.ImportStatus> importStatus() {
|
||||
return ResponseEntity.ok(massImportService.getStatus());
|
||||
public ResponseEntity<ImportStatus> importStatus() {
|
||||
return ResponseEntity.ok(importOrchestrator.getStatus());
|
||||
}
|
||||
|
||||
@PostMapping("/backfill-versions")
|
||||
|
||||
@@ -125,17 +125,10 @@ app:
|
||||
password: ${APP_ADMIN_PASSWORD:admin123}
|
||||
|
||||
import:
|
||||
col:
|
||||
index: 0
|
||||
box: 1
|
||||
folder: 2
|
||||
sender: 3
|
||||
receivers: 5
|
||||
date: 7
|
||||
location: 9
|
||||
tags: 10
|
||||
summary: 11
|
||||
transcription: 13
|
||||
# Directory holding the normalizer's committed canonical artifacts
|
||||
# (canonical-{documents,persons,tag-tree}.xlsx + canonical-persons-tree.json).
|
||||
# The loader maps columns by header name — no positional indices (see ADR-025).
|
||||
dir: ${IMPORT_DIR:/import}
|
||||
|
||||
ocr:
|
||||
sender-model:
|
||||
|
||||
Reference in New Issue
Block a user