import { createHash, randomUUID } from "node:crypto"; import { link, lstat, mkdir, open, readFile, readdir, rm, stat, unlink } from "node:fs/promises"; import path from "node:path"; import { canonicalJson, hashOrderedPairs, sha256Hex } from "../../shared/utils/ids.js"; import { isValidOcrResult, type OcrResult } from "./client.js"; import { composeCandidate, type CandidatePage } from "./composition.js"; import { classifyOcrPage } from "./detection.js"; const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu; const UUID_NAMESPACE_URL = Buffer.from("6ba7b8119dad11d180b400c04fd430c8", "hex"); interface StageInput { rootDirectory: string; versionId: string; createdAt: string; documents: Array<{ documentId: string; documentKey: string; bytes: Buffer; pages?: Array<{ page: number; text: string; rasterCoverage?: number; textSha256: string }>; requestedPages?: number[]; }>; } export function resolveArtifactPath(versionDirectory: string, relativePath: string): string { if (path.isAbsolute(relativePath)) throw new Error("Artifact path escapes version directory"); const root = path.resolve(versionDirectory); const resolved = path.resolve(root, relativePath); if (resolved === root || !resolved.startsWith(`${root}${path.sep}`)) throw new Error("Artifact path escapes version directory"); return resolved; } export async function stageOcrArtifacts(input: StageInput): Promise<{ versionDirectory: string; manifestPath: string; manifestSha256: string; originalManifestHash: string; }> { assertUuid(input.versionId); if (input.documents.length === 0 || Number.isNaN(Date.parse(input.createdAt))) throw new TypeError("Artifact manifest input is invalid"); const documentIds = new Set(input.documents.map(({ documentId }) => documentId)); const documentKeys = new Set(input.documents.map(({ documentKey }) => documentKey)); if (documentIds.size !== input.documents.length || documentKeys.size !== input.documents.length) throw new TypeError("Artifact documents must be unique"); await mkdir(input.rootDirectory, { recursive: true, mode: 0o700 }); const versionDirectory = path.join(path.resolve(input.rootDirectory), input.versionId); await mkdir(versionDirectory, { mode: 0o700 }); try { const documents = []; for (const document of [...input.documents].sort((left, right) => Buffer.compare(Buffer.from(left.documentKey), Buffer.from(right.documentKey)))) { const documentArtifactId = uuidV5(document.documentId); const relativeDirectory = path.posix.join("documents", documentArtifactId); const directory = resolveArtifactPath(versionDirectory, relativeDirectory); await mkdir(directory, { recursive: true, mode: 0o700 }); const originalPath = path.posix.join(relativeDirectory, "original.pdf"); await durableWrite(resolveArtifactPath(versionDirectory, originalPath), document.bytes); const nativePagesPath = path.posix.join(relativeDirectory, "native-pages.json"); const nativePages = { schemaVersion: "1", versionId: input.versionId, documentId: document.documentId, documentSha256: sha256Hex(document.bytes), requestedPages: document.requestedPages ?? [], pages: document.pages?.map((page) => ({ ...page, rasterCoverage: page.rasterCoverage ?? 0 })) ?? [] }; validateNativePages(nativePages); const nativeBytes = Buffer.from(canonicalJson(nativePages)); await durableWrite(resolveArtifactPath(versionDirectory, nativePagesPath), nativeBytes); documents.push({ documentId: document.documentId, documentKey: document.documentKey, documentArtifactId, originalPath, originalSha256: sha256Hex(document.bytes), nativePagesPath, nativePagesSha256: sha256Hex(nativeBytes) }); } const originalManifestHash = hashOrderedPairs(documents.map(({ documentKey, originalSha256 }) => [documentKey, originalSha256]))!; const manifest = { schemaVersion: "1", versionId: input.versionId, createdAt: input.createdAt, originalManifestHash, documents }; const serialized = canonicalJson(manifest); const manifestPath = path.join(versionDirectory, "manifest.json"); await durableWrite(manifestPath, Buffer.from(serialized)); await syncDirectory(versionDirectory); await syncDirectory(path.resolve(input.rootDirectory)); return { versionDirectory, manifestPath, manifestSha256: sha256Hex(serialized), originalManifestHash }; } catch (error) { await rm(versionDirectory, { recursive: true, force: true }); throw error; } } interface OcrResultArtifactInput { rootDirectory: string; versionId: string; documentId: string; result: OcrResult; } interface OcrResultReadInput extends Omit { jobId: string; documentSha256: string; pages: number[]; artifactSha256?: string; } export async function persistOcrResultArtifact(input: OcrResultArtifactInput): Promise<{ artifactPath: string; artifactSha256: string; result: OcrResult; }> { assertUuid(input.versionId); const pages = input.result.pages.map(({ page }) => page); if (!isValidOcrResult(input.result, input.result.jobId, { documentSha256: input.result.documentSha256, pages })) { throw resultIntegrityError(); } const resultSha256 = sha256Hex(canonicalJson(input.result)); const serialized = canonicalJson({ schemaVersion: "1", versionId: input.versionId, documentId: input.documentId, resultSha256, result: input.result }); const artifactPath = resultArtifactPath(input.rootDirectory, input.versionId, input.documentId); await publishImmutable(artifactPath, Buffer.from(serialized)); const artifactSha256 = sha256Hex(serialized); const result = await readOcrResultArtifact({ rootDirectory: input.rootDirectory, versionId: input.versionId, documentId: input.documentId, jobId: input.result.jobId, documentSha256: input.result.documentSha256, pages, artifactSha256 }); return { artifactPath, artifactSha256, result }; } export async function readOcrResultArtifact(input: OcrResultReadInput): Promise { assertUuid(input.versionId); const artifactPath = resultArtifactPath(input.rootDirectory, input.versionId, input.documentId); const file = await lstat(artifactPath).catch(() => { throw resultIntegrityError(); }); if (!file.isFile() || (file.mode & 0o777) !== 0o600) throw resultIntegrityError(); const bytes = await readFile(artifactPath); if (input.artifactSha256 && sha256Hex(bytes) !== input.artifactSha256) throw resultIntegrityError(); let envelope: unknown; try { envelope = JSON.parse(bytes.toString("utf8")); } catch { throw resultIntegrityError(); } if (!isRecord(envelope) || envelope.schemaVersion !== "1" || envelope.versionId !== input.versionId || envelope.documentId !== input.documentId) { throw new Error("OCR result artifact identity validation failed"); } if (typeof envelope.resultSha256 !== "string" || sha256Hex(canonicalJson(envelope.result)) !== envelope.resultSha256 || !isValidOcrResult(envelope.result, input.jobId, { documentSha256: input.documentSha256, pages: input.pages })) { throw resultIntegrityError(); } return envelope.result; } export interface CandidateArtifactPage extends CandidatePage { nativeTextSha256: string; ocrTextSha256: string | null; metrics: Record; } export interface ComposedCandidateArtifact { schemaVersion: "1"; versionId: string; candidateSha256: string; documents: Array<{ documentId: string; text: string; textSha256: string; pages: CandidateArtifactPage[] }>; } interface CandidateJobEvidence { documentId: string; remoteJobId: string | null; requestedPages: number[]; state: string; } export async function persistComposedCandidateArtifact(input: { rootDirectory: string; versionId: string; jobs: CandidateJobEvidence[]; }): Promise<{ artifactPath: string; artifactSha256: string; candidate: ComposedCandidateArtifact }> { assertUuid(input.versionId); const versionDirectory = path.join(path.resolve(input.rootDirectory), input.versionId); const manifest = await readPrivateJson(path.join(versionDirectory, "manifest.json"), undefined, "OCR manifest integrity validation failed"); if (!isRecord(manifest) || manifest.schemaVersion !== "1" || manifest.versionId !== input.versionId || !Array.isArray(manifest.documents)) { throw new Error("OCR manifest identity validation failed"); } const documents = []; for (const entry of manifest.documents) { if (!isManifestDocument(entry)) throw new Error("OCR manifest identity validation failed"); const native = await readPrivateJson(resolveArtifactPath(versionDirectory, entry.nativePagesPath), entry.nativePagesSha256, "OCR native page artifact integrity validation failed"); validateNativePages(native, input.versionId, entry.documentId, entry.originalSha256); const requestedPages = native.requestedPages as number[]; const job = input.jobs.find(({ documentId }) => documentId === entry.documentId); let result: OcrResult | undefined; if (requestedPages.length > 0) { if (!job || job.state !== "succeeded" || !job.remoteJobId || !sameNumbers(job.requestedPages, requestedPages)) throw new Error("OCR result artifact identity validation failed"); result = await readOcrResultArtifact({ rootDirectory: input.rootDirectory, versionId: input.versionId, documentId: entry.documentId, jobId: job.remoteJobId, documentSha256: entry.originalSha256, pages: requestedPages }); } else if (job) throw new Error("OCR result artifact identity validation failed"); const resultPages = new Map(result?.pages.map((page) => [page.page, page])); const evidence = (native.pages as Array<{ page: number; text: string; textSha256: string }>).map((page) => { const ocr = resultPages.get(page.page); if (!ocr) return { page: page.page, method: "native" as const, nativeText: page.text, rawOcrText: "", lines: [] }; const classification = classifyOcrPage({ inkCoverage: ocr.metrics.inkCoverage, metrics: ocr.metrics }); if (classification.method === "blocked") throw new Error(classification.errorCode); return { page: page.page, method: classification.method, nativeText: page.text, rawOcrText: ocr.text, lines: ocr.lines }; }); const composed = composeCandidate(evidence); documents.push({ documentId: entry.documentId, text: composed.text, textSha256: composed.textSha256, pages: composed.pages.map((page) => { const ocr = resultPages.get(page.page); return { ...page, nativeTextSha256: sha256Hex(page.nativeText), ocrTextSha256: ocr ? sha256Hex(ocr.text) : null, metrics: ocr?.metrics ?? {} }; }) }); } const candidate: ComposedCandidateArtifact = { schemaVersion: "1", versionId: input.versionId, candidateSha256: sha256Hex(canonicalJson(documents)), documents }; const serialized = canonicalJson(candidate); const artifactPath = path.join(versionDirectory, "candidate-pages.json"); await publishImmutable(artifactPath, Buffer.from(serialized)); const artifactSha256 = sha256Hex(serialized); return { artifactPath, artifactSha256, candidate: await readComposedCandidateArtifact({ ...input, artifactSha256 }) }; } export async function readComposedCandidateArtifact(input: { rootDirectory: string; versionId: string; artifactSha256?: string }): Promise { assertUuid(input.versionId); const value = await readPrivateJson(path.join(path.resolve(input.rootDirectory), input.versionId, "candidate-pages.json"), input.artifactSha256, "OCR candidate artifact integrity validation failed"); if (!isRecord(value) || value.schemaVersion !== "1" || value.versionId !== input.versionId || !Array.isArray(value.documents) || value.candidateSha256 !== sha256Hex(canonicalJson(value.documents))) throw new Error("OCR candidate artifact integrity validation failed"); return value as unknown as ComposedCandidateArtifact; } export interface ReviewedPagesArtifact { schemaVersion: "1"; versionId: string; sourceId: string; candidateSha256: string; reviewedTextSha256: string; reviewedBy: string; documents: Array<{ documentId: string; pages: Array<{ page: number; candidateText: string; ocr: { lines: Array<{ lineId: string; text: string; confidence: number; bbox: [number, number, number, number]; lineSha256: string }> } }> }>; } interface ReviewedPagesIdentity { rootDirectory: string; versionId: string; candidateSha256: string; reviewedTextSha256: string; artifactSha256?: string; } export async function persistReviewedPagesArtifact(input: Omit & { rootDirectory: string }): Promise<{ artifactPath: string; artifactSha256: string; created: boolean; reviewed: ReviewedPagesArtifact; }> { assertUuid(input.versionId); const versionDirectory = path.join(path.resolve(input.rootDirectory), input.versionId); const directory = await lstat(versionDirectory).catch(() => { throw new Error("OCR reviewed artifact directory not found"); }); if (!directory.isDirectory()) throw new Error("OCR reviewed artifact directory not found"); const reviewed: ReviewedPagesArtifact = { schemaVersion: "1", versionId: input.versionId, sourceId: input.sourceId, candidateSha256: input.candidateSha256, reviewedTextSha256: input.reviewedTextSha256, reviewedBy: input.reviewedBy, documents: input.documents }; validateReviewedPages(reviewed, input); const bytes = Buffer.from(canonicalJson(reviewed)); const artifactPath = path.join(versionDirectory, "reviewed-pages.json"); const created = await publishImmutable(artifactPath, bytes); const artifactSha256 = sha256Hex(bytes); try { return { artifactPath, artifactSha256, created, reviewed: await readReviewedPagesArtifact({ ...input, artifactSha256 }) }; } catch (error) { if (created) await unlink(artifactPath).catch(() => undefined); throw error; } } export async function readReviewedPagesArtifact(input: ReviewedPagesIdentity): Promise { assertUuid(input.versionId); const artifactPath = path.join(path.resolve(input.rootDirectory), input.versionId, "reviewed-pages.json"); const value = await readPrivateJson(artifactPath, input.artifactSha256, "OCR reviewed artifact integrity validation failed"); validateReviewedPages(value, input); return value as ReviewedPagesArtifact; } export async function removeReviewedPagesArtifact(rootDirectory: string, versionId: string): Promise { assertUuid(versionId); const target = path.join(path.resolve(rootDirectory), versionId, "reviewed-pages.json"); await unlink(target); await syncDirectory(path.dirname(target)); } export async function readOcrArtifactPageNumbers(input: { rootDirectory: string; versionId: string; documentId: string }): Promise { const { entry, native } = await readNativeDocument(input); validateNativePages(native, input.versionId, input.documentId, entry.originalSha256); return (native.pages as Array<{ page: number }>).map(({ page }) => page); } export async function persistReviewImageArtifacts(input: { rootDirectory: string; versionId: string; documentId: string; images: Array<{ page: number; bytes: Buffer; sha256: string }>; }): Promise<{ images: Array<{ page: number; artifactPath: string; sha256: string }> }> { assertUuid(input.versionId); const { versionDirectory, entry, native } = await readNativeDocument(input); validateNativePages(native, input.versionId, input.documentId, entry.originalSha256); const expectedPages = (native.pages as Array<{ page: number }>).map(({ page }) => page); if (!sameNumbers(input.images.map(({ page }) => page), expectedPages)) throw new Error("OCR review image identity validation failed"); const directory = resolveArtifactPath(versionDirectory, path.posix.join("documents", entry.documentArtifactId, "review-images")); await mkdir(directory, { recursive: true, mode: 0o700 }); const images = []; for (const image of input.images) { if (sha256Hex(image.bytes) !== image.sha256 || !image.bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex"))) throw new Error("OCR review image integrity validation failed"); const relativePath = path.posix.join("documents", entry.documentArtifactId, "review-images", `page-${String(image.page).padStart(4, "0")}.png`); const artifactPath = resolveArtifactPath(versionDirectory, relativePath); await publishImmutable(artifactPath, image.bytes); images.push({ page: image.page, relativePath, artifactPath, sha256: image.sha256, mimeType: "image/png" }); } const manifestPath = path.join(directory, "manifest.json"); await publishImmutable(manifestPath, Buffer.from(canonicalJson({ schemaVersion: "1", versionId: input.versionId, documentId: input.documentId, documentSha256: entry.originalSha256, images: images.map(({ artifactPath: _artifactPath, ...image }) => image) }))); return { images: images.map(({ page, artifactPath, sha256 }) => ({ page, artifactPath, sha256 })) }; } export async function readReviewImageArtifact(input: { rootDirectory: string; versionId: string; documentId: string; page: number; }): Promise<{ bytes: Buffer; sha256: string; mimeType: "image/png" }> { assertUuid(input.versionId); const { versionDirectory, entry } = await readNativeDocument(input); const directory = resolveArtifactPath(versionDirectory, path.posix.join("documents", entry.documentArtifactId, "review-images")); const manifest = await readPrivateJson(path.join(directory, "manifest.json"), undefined, "OCR review image integrity validation failed"); if (!isRecord(manifest) || manifest.schemaVersion !== "1" || manifest.versionId !== input.versionId || manifest.documentId !== input.documentId || manifest.documentSha256 !== entry.originalSha256 || !Array.isArray(manifest.images)) throw new Error("OCR review image identity validation failed"); const image = manifest.images.find((value) => isRecord(value) && value.page === input.page); if (!isRecord(image) || typeof image.relativePath !== "string" || typeof image.sha256 !== "string" || image.mimeType !== "image/png") throw new Error("OCR review image not found"); const artifactPath = resolveArtifactPath(versionDirectory, image.relativePath); const file = await lstat(artifactPath).catch(() => { throw new Error("OCR review image integrity validation failed"); }); const bytes = await readFile(artifactPath); if (!file.isFile() || (file.mode & 0o777) !== 0o600 || sha256Hex(bytes) !== image.sha256) throw new Error("OCR review image integrity validation failed"); return { bytes, sha256: image.sha256, mimeType: "image/png" }; } export async function sweepOrphanArtifacts(input: { rootDirectory: string; retainedVersionIds: ReadonlySet; olderThan: Date; }): Promise { for (const versionId of input.retainedVersionIds) assertUuid(versionId); let entries; try { entries = await readdir(input.rootDirectory, { withFileTypes: true }); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; } const removed: string[] = []; for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { if (!entry.isDirectory() || !UUID.test(entry.name) || input.retainedVersionIds.has(entry.name)) continue; const candidate = path.join(path.resolve(input.rootDirectory), entry.name); const current = await lstat(candidate); if (!current.isDirectory() || current.mtimeMs >= input.olderThan.getTime()) continue; await rm(candidate, { recursive: true, force: true }); removed.push(entry.name); } if (removed.length > 0) await syncDirectory(path.resolve(input.rootDirectory)); return removed; } async function durableWrite(target: string, bytes: Buffer): Promise { const temporary = `${target}.${randomUUID()}.tmp`; const handle = await open(temporary, "wx", 0o600); try { await handle.writeFile(bytes); await handle.chmod(0o600); await handle.sync(); } finally { await handle.close(); } try { await link(temporary, target); } finally { await unlink(temporary).catch(() => undefined); } await syncDirectory(path.dirname(target)); } async function publishImmutable(target: string, bytes: Buffer): Promise { try { await durableWrite(target, bytes); return true; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; const existing = await readFile(target); if (!existing.equals(bytes)) throw new Error("OCR result artifact conflicts with durable content"); return false; } } function resultArtifactPath(rootDirectory: string, versionId: string, documentId: string): string { const versionDirectory = path.join(path.resolve(rootDirectory), versionId); return resolveArtifactPath(versionDirectory, path.posix.join("documents", uuidV5(documentId), "ocr-result.json")); } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function isManifestDocument(value: unknown): value is { documentId: string; originalSha256: string; nativePagesPath: string; nativePagesSha256: string } { return isRecord(value) && typeof value.documentId === "string" && typeof value.originalSha256 === "string" && typeof value.nativePagesPath === "string" && typeof value.nativePagesSha256 === "string"; } async function readNativeDocument(input: { rootDirectory: string; versionId: string; documentId: string }) { assertUuid(input.versionId); const versionDirectory = path.join(path.resolve(input.rootDirectory), input.versionId); const manifest = await readPrivateJson(path.join(versionDirectory, "manifest.json"), undefined, "OCR manifest integrity validation failed"); if (!isRecord(manifest) || manifest.schemaVersion !== "1" || manifest.versionId !== input.versionId || !Array.isArray(manifest.documents)) throw new Error("OCR manifest identity validation failed"); const entry = manifest.documents.find((value) => isManifestDocument(value) && value.documentId === input.documentId); if (!entry || !isRecord(entry) || typeof entry.documentArtifactId !== "string") throw new Error("OCR review image not found"); const document = entry as { documentId: string; documentArtifactId: string; originalSha256: string; nativePagesPath: string; nativePagesSha256: string }; const native = await readPrivateJson(resolveArtifactPath(versionDirectory, document.nativePagesPath), document.nativePagesSha256, "OCR native page artifact integrity validation failed"); return { versionDirectory, entry: document, native }; } function validateNativePages(value: unknown, versionId?: string, documentId?: string, documentSha256?: string): asserts value is Record { if (!isRecord(value) || value.schemaVersion !== "1" || (versionId && value.versionId !== versionId) || (documentId && value.documentId !== documentId) || (documentSha256 && value.documentSha256 !== documentSha256) || !Array.isArray(value.pages) || !Array.isArray(value.requestedPages)) throw new Error("OCR native page artifact integrity validation failed"); const pages = value.pages as Array>; const requested = value.requestedPages as number[]; if (pages.some((page, index) => page.page !== index + 1 || typeof page.text !== "string" || page.textSha256 !== sha256Hex(page.text) || typeof page.rasterCoverage !== "number" || page.rasterCoverage < 0 || page.rasterCoverage > 1) || !sameNumbers(requested, [...new Set(requested)].sort((a, b) => a - b)) || requested.some((page) => page < 1 || page > pages.length)) { throw new Error("OCR native page artifact integrity validation failed"); } } function validateReviewedPages(value: unknown, identity: Pick): asserts value is ReviewedPagesArtifact { if (!isRecord(value) || value.schemaVersion !== "1" || value.versionId !== identity.versionId || value.candidateSha256 !== identity.candidateSha256 || value.reviewedTextSha256 !== identity.reviewedTextSha256 || typeof value.sourceId !== "string" || typeof value.reviewedBy !== "string" || !Array.isArray(value.documents)) throw new Error("OCR reviewed artifact identity validation failed"); const documents = value.documents as ReviewedPagesArtifact["documents"]; const pages = documents.flatMap((document) => Array.isArray(document.pages) ? document.pages : []); if (documents.some((document) => typeof document.documentId !== "string" || !Array.isArray(document.pages)) || pages.some((page) => !Number.isInteger(page.page) || page.page < 1 || typeof page.candidateText !== "string" || !isRecord(page.ocr) || !Array.isArray(page.ocr.lines) || page.ocr.lines.some((line) => !isRecord(line) || typeof line.lineId !== "string" || typeof line.text !== "string" || line.lineSha256 !== sha256Hex(line.text)) || (page.ocr.lines.length > 0 && page.candidateText !== page.ocr.lines.map(({ text }) => text).join("\n")))) { throw new Error("OCR reviewed artifact integrity validation failed"); } const reviewedText = pages.filter(({ candidateText }) => candidateText).map(({ candidateText }) => candidateText).join("\n\n"); if (sha256Hex(reviewedText) !== identity.reviewedTextSha256) throw new Error("OCR reviewed artifact integrity validation failed"); } async function readPrivateJson(target: string, expectedSha256: string | undefined, message: string): Promise { const file = await lstat(target).catch(() => { throw new Error(message); }); if (!file.isFile() || (file.mode & 0o777) !== 0o600) throw new Error(message); const bytes = await readFile(target); if (expectedSha256 && sha256Hex(bytes) !== expectedSha256) throw new Error(message); try { return JSON.parse(bytes.toString("utf8")); } catch { throw new Error(message); } } function sameNumbers(left: number[], right: number[]): boolean { return left.length === right.length && left.every((value, index) => value === right[index]); } function resultIntegrityError(): Error { return new Error("OCR result artifact integrity validation failed"); } async function syncDirectory(directory: string): Promise { const handle = await open(directory, "r"); try { await handle.sync(); } finally { await handle.close(); } } function uuidV5(value: string): string { const bytes = createHash("sha1").update(Buffer.concat([UUID_NAMESPACE_URL, Buffer.from(value)])).digest().subarray(0, 16); bytes[6] = (bytes[6]! & 0x0f) | 0x50; bytes[8] = (bytes[8]! & 0x3f) | 0x80; const hex = bytes.toString("hex"); return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; } function assertUuid(value: string): void { if (!UUID.test(value)) throw new TypeError("Version identity must be a UUID"); }