Paco POR-CORREO
5fef85cfb2
Wire the complete OCR review and indexing production pipeline: durable OCR result handoff before remote deletion (17a), native-page evidence and quality-gated candidate composition (17b), review images and restart-safe candidate loading (17c), transactional approval and rejection decisions (17d), reviewed-artifact indexing store with exact count verification (17e), and production approve-to-ready wiring with fail-closed OCR_INDEXING_UNAVAILABLE (17f). Activation remains a separately authorized operation; task 7.4 stays pending.
480 lines
26 KiB
TypeScript
480 lines
26 KiB
TypeScript
import { createHash, randomUUID } from "node:crypto";
|
|
import { link, lstat, mkdir, open, readFile, readdir, rm, stat, unlink } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { canonicalJson, hashOrderedPairs, sha256Hex } from "../../shared/utils/ids.js";
|
|
import { isValidOcrResult, type OcrResult } from "./client.js";
|
|
import { composeCandidate, type CandidatePage } from "./composition.js";
|
|
import { classifyOcrPage } from "./detection.js";
|
|
|
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
const UUID_NAMESPACE_URL = Buffer.from("6ba7b8119dad11d180b400c04fd430c8", "hex");
|
|
|
|
interface StageInput {
|
|
rootDirectory: string;
|
|
versionId: string;
|
|
createdAt: string;
|
|
documents: Array<{
|
|
documentId: string;
|
|
documentKey: string;
|
|
bytes: Buffer;
|
|
pages?: Array<{ page: number; text: string; rasterCoverage?: number; textSha256: string }>;
|
|
requestedPages?: number[];
|
|
}>;
|
|
}
|
|
|
|
export function resolveArtifactPath(versionDirectory: string, relativePath: string): string {
|
|
if (path.isAbsolute(relativePath)) throw new Error("Artifact path escapes version directory");
|
|
const root = path.resolve(versionDirectory);
|
|
const resolved = path.resolve(root, relativePath);
|
|
if (resolved === root || !resolved.startsWith(`${root}${path.sep}`)) throw new Error("Artifact path escapes version directory");
|
|
return resolved;
|
|
}
|
|
|
|
export async function stageOcrArtifacts(input: StageInput): Promise<{
|
|
versionDirectory: string;
|
|
manifestPath: string;
|
|
manifestSha256: string;
|
|
originalManifestHash: string;
|
|
}> {
|
|
assertUuid(input.versionId);
|
|
if (input.documents.length === 0 || Number.isNaN(Date.parse(input.createdAt))) throw new TypeError("Artifact manifest input is invalid");
|
|
const documentIds = new Set(input.documents.map(({ documentId }) => documentId));
|
|
const documentKeys = new Set(input.documents.map(({ documentKey }) => documentKey));
|
|
if (documentIds.size !== input.documents.length || documentKeys.size !== input.documents.length) throw new TypeError("Artifact documents must be unique");
|
|
|
|
await mkdir(input.rootDirectory, { recursive: true, mode: 0o700 });
|
|
const versionDirectory = path.join(path.resolve(input.rootDirectory), input.versionId);
|
|
await mkdir(versionDirectory, { mode: 0o700 });
|
|
try {
|
|
const documents = [];
|
|
for (const document of [...input.documents].sort((left, right) => Buffer.compare(Buffer.from(left.documentKey), Buffer.from(right.documentKey)))) {
|
|
const documentArtifactId = uuidV5(document.documentId);
|
|
const relativeDirectory = path.posix.join("documents", documentArtifactId);
|
|
const directory = resolveArtifactPath(versionDirectory, relativeDirectory);
|
|
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
const originalPath = path.posix.join(relativeDirectory, "original.pdf");
|
|
await durableWrite(resolveArtifactPath(versionDirectory, originalPath), document.bytes);
|
|
const nativePagesPath = path.posix.join(relativeDirectory, "native-pages.json");
|
|
const nativePages = {
|
|
schemaVersion: "1", versionId: input.versionId, documentId: document.documentId,
|
|
documentSha256: sha256Hex(document.bytes), requestedPages: document.requestedPages ?? [],
|
|
pages: document.pages?.map((page) => ({ ...page, rasterCoverage: page.rasterCoverage ?? 0 })) ?? []
|
|
};
|
|
validateNativePages(nativePages);
|
|
const nativeBytes = Buffer.from(canonicalJson(nativePages));
|
|
await durableWrite(resolveArtifactPath(versionDirectory, nativePagesPath), nativeBytes);
|
|
documents.push({
|
|
documentId: document.documentId,
|
|
documentKey: document.documentKey,
|
|
documentArtifactId,
|
|
originalPath,
|
|
originalSha256: sha256Hex(document.bytes),
|
|
nativePagesPath,
|
|
nativePagesSha256: sha256Hex(nativeBytes)
|
|
});
|
|
}
|
|
const originalManifestHash = hashOrderedPairs(documents.map(({ documentKey, originalSha256 }) => [documentKey, originalSha256]))!;
|
|
const manifest = { schemaVersion: "1", versionId: input.versionId, createdAt: input.createdAt, originalManifestHash, documents };
|
|
const serialized = canonicalJson(manifest);
|
|
const manifestPath = path.join(versionDirectory, "manifest.json");
|
|
await durableWrite(manifestPath, Buffer.from(serialized));
|
|
await syncDirectory(versionDirectory);
|
|
await syncDirectory(path.resolve(input.rootDirectory));
|
|
return { versionDirectory, manifestPath, manifestSha256: sha256Hex(serialized), originalManifestHash };
|
|
} catch (error) {
|
|
await rm(versionDirectory, { recursive: true, force: true });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
interface OcrResultArtifactInput {
|
|
rootDirectory: string;
|
|
versionId: string;
|
|
documentId: string;
|
|
result: OcrResult;
|
|
}
|
|
|
|
interface OcrResultReadInput extends Omit<OcrResultArtifactInput, "result"> {
|
|
jobId: string;
|
|
documentSha256: string;
|
|
pages: number[];
|
|
artifactSha256?: string;
|
|
}
|
|
|
|
export async function persistOcrResultArtifact(input: OcrResultArtifactInput): Promise<{
|
|
artifactPath: string;
|
|
artifactSha256: string;
|
|
result: OcrResult;
|
|
}> {
|
|
assertUuid(input.versionId);
|
|
const pages = input.result.pages.map(({ page }) => page);
|
|
if (!isValidOcrResult(input.result, input.result.jobId, { documentSha256: input.result.documentSha256, pages })) {
|
|
throw resultIntegrityError();
|
|
}
|
|
const resultSha256 = sha256Hex(canonicalJson(input.result));
|
|
const serialized = canonicalJson({
|
|
schemaVersion: "1",
|
|
versionId: input.versionId,
|
|
documentId: input.documentId,
|
|
resultSha256,
|
|
result: input.result
|
|
});
|
|
const artifactPath = resultArtifactPath(input.rootDirectory, input.versionId, input.documentId);
|
|
await publishImmutable(artifactPath, Buffer.from(serialized));
|
|
const artifactSha256 = sha256Hex(serialized);
|
|
const result = await readOcrResultArtifact({
|
|
rootDirectory: input.rootDirectory,
|
|
versionId: input.versionId,
|
|
documentId: input.documentId,
|
|
jobId: input.result.jobId,
|
|
documentSha256: input.result.documentSha256,
|
|
pages,
|
|
artifactSha256
|
|
});
|
|
return { artifactPath, artifactSha256, result };
|
|
}
|
|
|
|
export async function readOcrResultArtifact(input: OcrResultReadInput): Promise<OcrResult> {
|
|
assertUuid(input.versionId);
|
|
const artifactPath = resultArtifactPath(input.rootDirectory, input.versionId, input.documentId);
|
|
const file = await lstat(artifactPath).catch(() => { throw resultIntegrityError(); });
|
|
if (!file.isFile() || (file.mode & 0o777) !== 0o600) throw resultIntegrityError();
|
|
const bytes = await readFile(artifactPath);
|
|
if (input.artifactSha256 && sha256Hex(bytes) !== input.artifactSha256) throw resultIntegrityError();
|
|
let envelope: unknown;
|
|
try { envelope = JSON.parse(bytes.toString("utf8")); } catch { throw resultIntegrityError(); }
|
|
if (!isRecord(envelope) || envelope.schemaVersion !== "1" || envelope.versionId !== input.versionId || envelope.documentId !== input.documentId) {
|
|
throw new Error("OCR result artifact identity validation failed");
|
|
}
|
|
if (typeof envelope.resultSha256 !== "string" || sha256Hex(canonicalJson(envelope.result)) !== envelope.resultSha256
|
|
|| !isValidOcrResult(envelope.result, input.jobId, { documentSha256: input.documentSha256, pages: input.pages })) {
|
|
throw resultIntegrityError();
|
|
}
|
|
return envelope.result;
|
|
}
|
|
|
|
export interface CandidateArtifactPage extends CandidatePage {
|
|
nativeTextSha256: string;
|
|
ocrTextSha256: string | null;
|
|
metrics: Record<string, number>;
|
|
}
|
|
|
|
export interface ComposedCandidateArtifact {
|
|
schemaVersion: "1";
|
|
versionId: string;
|
|
candidateSha256: string;
|
|
documents: Array<{ documentId: string; text: string; textSha256: string; pages: CandidateArtifactPage[] }>;
|
|
}
|
|
|
|
interface CandidateJobEvidence {
|
|
documentId: string;
|
|
remoteJobId: string | null;
|
|
requestedPages: number[];
|
|
state: string;
|
|
}
|
|
|
|
export async function persistComposedCandidateArtifact(input: {
|
|
rootDirectory: string;
|
|
versionId: string;
|
|
jobs: CandidateJobEvidence[];
|
|
}): Promise<{ artifactPath: string; artifactSha256: string; candidate: ComposedCandidateArtifact }> {
|
|
assertUuid(input.versionId);
|
|
const versionDirectory = path.join(path.resolve(input.rootDirectory), input.versionId);
|
|
const manifest = await readPrivateJson(path.join(versionDirectory, "manifest.json"), undefined, "OCR manifest integrity validation failed");
|
|
if (!isRecord(manifest) || manifest.schemaVersion !== "1" || manifest.versionId !== input.versionId || !Array.isArray(manifest.documents)) {
|
|
throw new Error("OCR manifest identity validation failed");
|
|
}
|
|
const documents = [];
|
|
for (const entry of manifest.documents) {
|
|
if (!isManifestDocument(entry)) throw new Error("OCR manifest identity validation failed");
|
|
const native = await readPrivateJson(resolveArtifactPath(versionDirectory, entry.nativePagesPath), entry.nativePagesSha256, "OCR native page artifact integrity validation failed");
|
|
validateNativePages(native, input.versionId, entry.documentId, entry.originalSha256);
|
|
const requestedPages = native.requestedPages as number[];
|
|
const job = input.jobs.find(({ documentId }) => documentId === entry.documentId);
|
|
let result: OcrResult | undefined;
|
|
if (requestedPages.length > 0) {
|
|
if (!job || job.state !== "succeeded" || !job.remoteJobId || !sameNumbers(job.requestedPages, requestedPages)) throw new Error("OCR result artifact identity validation failed");
|
|
result = await readOcrResultArtifact({ rootDirectory: input.rootDirectory, versionId: input.versionId, documentId: entry.documentId,
|
|
jobId: job.remoteJobId, documentSha256: entry.originalSha256, pages: requestedPages });
|
|
} else if (job) throw new Error("OCR result artifact identity validation failed");
|
|
const resultPages = new Map(result?.pages.map((page) => [page.page, page]));
|
|
const evidence = (native.pages as Array<{ page: number; text: string; textSha256: string }>).map((page) => {
|
|
const ocr = resultPages.get(page.page);
|
|
if (!ocr) return { page: page.page, method: "native" as const, nativeText: page.text, rawOcrText: "", lines: [] };
|
|
const classification = classifyOcrPage({ inkCoverage: ocr.metrics.inkCoverage, metrics: ocr.metrics });
|
|
if (classification.method === "blocked") throw new Error(classification.errorCode);
|
|
return { page: page.page, method: classification.method, nativeText: page.text, rawOcrText: ocr.text, lines: ocr.lines };
|
|
});
|
|
const composed = composeCandidate(evidence);
|
|
documents.push({ documentId: entry.documentId, text: composed.text, textSha256: composed.textSha256,
|
|
pages: composed.pages.map((page) => { const ocr = resultPages.get(page.page); return { ...page,
|
|
nativeTextSha256: sha256Hex(page.nativeText), ocrTextSha256: ocr ? sha256Hex(ocr.text) : null,
|
|
metrics: ocr?.metrics ?? {} }; }) });
|
|
}
|
|
const candidate: ComposedCandidateArtifact = { schemaVersion: "1", versionId: input.versionId,
|
|
candidateSha256: sha256Hex(canonicalJson(documents)), documents };
|
|
const serialized = canonicalJson(candidate);
|
|
const artifactPath = path.join(versionDirectory, "candidate-pages.json");
|
|
await publishImmutable(artifactPath, Buffer.from(serialized));
|
|
const artifactSha256 = sha256Hex(serialized);
|
|
return { artifactPath, artifactSha256, candidate: await readComposedCandidateArtifact({ ...input, artifactSha256 }) };
|
|
}
|
|
|
|
export async function readComposedCandidateArtifact(input: { rootDirectory: string; versionId: string; artifactSha256?: string }): Promise<ComposedCandidateArtifact> {
|
|
assertUuid(input.versionId);
|
|
const value = await readPrivateJson(path.join(path.resolve(input.rootDirectory), input.versionId, "candidate-pages.json"), input.artifactSha256, "OCR candidate artifact integrity validation failed");
|
|
if (!isRecord(value) || value.schemaVersion !== "1" || value.versionId !== input.versionId || !Array.isArray(value.documents)
|
|
|| value.candidateSha256 !== sha256Hex(canonicalJson(value.documents))) throw new Error("OCR candidate artifact integrity validation failed");
|
|
return value as unknown as ComposedCandidateArtifact;
|
|
}
|
|
|
|
export interface ReviewedPagesArtifact {
|
|
schemaVersion: "1";
|
|
versionId: string;
|
|
sourceId: string;
|
|
candidateSha256: string;
|
|
reviewedTextSha256: string;
|
|
reviewedBy: string;
|
|
documents: Array<{ documentId: string; pages: Array<{ page: number; candidateText: string;
|
|
ocr: { lines: Array<{ lineId: string; text: string; confidence: number; bbox: [number, number, number, number]; lineSha256: string }> } }> }>;
|
|
}
|
|
|
|
interface ReviewedPagesIdentity {
|
|
rootDirectory: string;
|
|
versionId: string;
|
|
candidateSha256: string;
|
|
reviewedTextSha256: string;
|
|
artifactSha256?: string;
|
|
}
|
|
|
|
export async function persistReviewedPagesArtifact(input: Omit<ReviewedPagesArtifact, "schemaVersion"> & { rootDirectory: string }): Promise<{
|
|
artifactPath: string; artifactSha256: string; created: boolean; reviewed: ReviewedPagesArtifact;
|
|
}> {
|
|
assertUuid(input.versionId);
|
|
const versionDirectory = path.join(path.resolve(input.rootDirectory), input.versionId);
|
|
const directory = await lstat(versionDirectory).catch(() => { throw new Error("OCR reviewed artifact directory not found"); });
|
|
if (!directory.isDirectory()) throw new Error("OCR reviewed artifact directory not found");
|
|
const reviewed: ReviewedPagesArtifact = { schemaVersion: "1", versionId: input.versionId, sourceId: input.sourceId,
|
|
candidateSha256: input.candidateSha256, reviewedTextSha256: input.reviewedTextSha256,
|
|
reviewedBy: input.reviewedBy, documents: input.documents };
|
|
validateReviewedPages(reviewed, input);
|
|
const bytes = Buffer.from(canonicalJson(reviewed));
|
|
const artifactPath = path.join(versionDirectory, "reviewed-pages.json");
|
|
const created = await publishImmutable(artifactPath, bytes);
|
|
const artifactSha256 = sha256Hex(bytes);
|
|
try {
|
|
return { artifactPath, artifactSha256, created, reviewed: await readReviewedPagesArtifact({ ...input, artifactSha256 }) };
|
|
} catch (error) {
|
|
if (created) await unlink(artifactPath).catch(() => undefined);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export async function readReviewedPagesArtifact(input: ReviewedPagesIdentity): Promise<ReviewedPagesArtifact> {
|
|
assertUuid(input.versionId);
|
|
const artifactPath = path.join(path.resolve(input.rootDirectory), input.versionId, "reviewed-pages.json");
|
|
const value = await readPrivateJson(artifactPath, input.artifactSha256, "OCR reviewed artifact integrity validation failed");
|
|
validateReviewedPages(value, input);
|
|
return value as ReviewedPagesArtifact;
|
|
}
|
|
|
|
export async function removeReviewedPagesArtifact(rootDirectory: string, versionId: string): Promise<void> {
|
|
assertUuid(versionId);
|
|
const target = path.join(path.resolve(rootDirectory), versionId, "reviewed-pages.json");
|
|
await unlink(target);
|
|
await syncDirectory(path.dirname(target));
|
|
}
|
|
|
|
export async function readOcrArtifactPageNumbers(input: { rootDirectory: string; versionId: string; documentId: string }): Promise<number[]> {
|
|
const { entry, native } = await readNativeDocument(input);
|
|
validateNativePages(native, input.versionId, input.documentId, entry.originalSha256);
|
|
return (native.pages as Array<{ page: number }>).map(({ page }) => page);
|
|
}
|
|
|
|
export async function persistReviewImageArtifacts(input: {
|
|
rootDirectory: string; versionId: string; documentId: string;
|
|
images: Array<{ page: number; bytes: Buffer; sha256: string }>;
|
|
}): Promise<{ images: Array<{ page: number; artifactPath: string; sha256: string }> }> {
|
|
assertUuid(input.versionId);
|
|
const { versionDirectory, entry, native } = await readNativeDocument(input);
|
|
validateNativePages(native, input.versionId, input.documentId, entry.originalSha256);
|
|
const expectedPages = (native.pages as Array<{ page: number }>).map(({ page }) => page);
|
|
if (!sameNumbers(input.images.map(({ page }) => page), expectedPages)) throw new Error("OCR review image identity validation failed");
|
|
const directory = resolveArtifactPath(versionDirectory, path.posix.join("documents", entry.documentArtifactId, "review-images"));
|
|
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
const images = [];
|
|
for (const image of input.images) {
|
|
if (sha256Hex(image.bytes) !== image.sha256 || !image.bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex"))) throw new Error("OCR review image integrity validation failed");
|
|
const relativePath = path.posix.join("documents", entry.documentArtifactId, "review-images", `page-${String(image.page).padStart(4, "0")}.png`);
|
|
const artifactPath = resolveArtifactPath(versionDirectory, relativePath);
|
|
await publishImmutable(artifactPath, image.bytes);
|
|
images.push({ page: image.page, relativePath, artifactPath, sha256: image.sha256, mimeType: "image/png" });
|
|
}
|
|
const manifestPath = path.join(directory, "manifest.json");
|
|
await publishImmutable(manifestPath, Buffer.from(canonicalJson({ schemaVersion: "1", versionId: input.versionId, documentId: input.documentId,
|
|
documentSha256: entry.originalSha256, images: images.map(({ artifactPath: _artifactPath, ...image }) => image) })));
|
|
return { images: images.map(({ page, artifactPath, sha256 }) => ({ page, artifactPath, sha256 })) };
|
|
}
|
|
|
|
export async function readReviewImageArtifact(input: {
|
|
rootDirectory: string; versionId: string; documentId: string; page: number;
|
|
}): Promise<{ bytes: Buffer; sha256: string; mimeType: "image/png" }> {
|
|
assertUuid(input.versionId);
|
|
const { versionDirectory, entry } = await readNativeDocument(input);
|
|
const directory = resolveArtifactPath(versionDirectory, path.posix.join("documents", entry.documentArtifactId, "review-images"));
|
|
const manifest = await readPrivateJson(path.join(directory, "manifest.json"), undefined, "OCR review image integrity validation failed");
|
|
if (!isRecord(manifest) || manifest.schemaVersion !== "1" || manifest.versionId !== input.versionId || manifest.documentId !== input.documentId
|
|
|| manifest.documentSha256 !== entry.originalSha256 || !Array.isArray(manifest.images)) throw new Error("OCR review image identity validation failed");
|
|
const image = manifest.images.find((value) => isRecord(value) && value.page === input.page);
|
|
if (!isRecord(image) || typeof image.relativePath !== "string" || typeof image.sha256 !== "string" || image.mimeType !== "image/png") throw new Error("OCR review image not found");
|
|
const artifactPath = resolveArtifactPath(versionDirectory, image.relativePath);
|
|
const file = await lstat(artifactPath).catch(() => { throw new Error("OCR review image integrity validation failed"); });
|
|
const bytes = await readFile(artifactPath);
|
|
if (!file.isFile() || (file.mode & 0o777) !== 0o600 || sha256Hex(bytes) !== image.sha256) throw new Error("OCR review image integrity validation failed");
|
|
return { bytes, sha256: image.sha256, mimeType: "image/png" };
|
|
}
|
|
|
|
export async function sweepOrphanArtifacts(input: {
|
|
rootDirectory: string;
|
|
retainedVersionIds: ReadonlySet<string>;
|
|
olderThan: Date;
|
|
}): Promise<string[]> {
|
|
for (const versionId of input.retainedVersionIds) assertUuid(versionId);
|
|
let entries;
|
|
try {
|
|
entries = await readdir(input.rootDirectory, { withFileTypes: true });
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
throw error;
|
|
}
|
|
const removed: string[] = [];
|
|
for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
|
|
if (!entry.isDirectory() || !UUID.test(entry.name) || input.retainedVersionIds.has(entry.name)) continue;
|
|
const candidate = path.join(path.resolve(input.rootDirectory), entry.name);
|
|
const current = await lstat(candidate);
|
|
if (!current.isDirectory() || current.mtimeMs >= input.olderThan.getTime()) continue;
|
|
await rm(candidate, { recursive: true, force: true });
|
|
removed.push(entry.name);
|
|
}
|
|
if (removed.length > 0) await syncDirectory(path.resolve(input.rootDirectory));
|
|
return removed;
|
|
}
|
|
|
|
async function durableWrite(target: string, bytes: Buffer): Promise<void> {
|
|
const temporary = `${target}.${randomUUID()}.tmp`;
|
|
const handle = await open(temporary, "wx", 0o600);
|
|
try {
|
|
await handle.writeFile(bytes);
|
|
await handle.chmod(0o600);
|
|
await handle.sync();
|
|
} finally {
|
|
await handle.close();
|
|
}
|
|
try {
|
|
await link(temporary, target);
|
|
} finally {
|
|
await unlink(temporary).catch(() => undefined);
|
|
}
|
|
await syncDirectory(path.dirname(target));
|
|
}
|
|
|
|
async function publishImmutable(target: string, bytes: Buffer): Promise<boolean> {
|
|
try {
|
|
await durableWrite(target, bytes);
|
|
return true;
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
|
const existing = await readFile(target);
|
|
if (!existing.equals(bytes)) throw new Error("OCR result artifact conflicts with durable content");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function resultArtifactPath(rootDirectory: string, versionId: string, documentId: string): string {
|
|
const versionDirectory = path.join(path.resolve(rootDirectory), versionId);
|
|
return resolveArtifactPath(versionDirectory, path.posix.join("documents", uuidV5(documentId), "ocr-result.json"));
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function isManifestDocument(value: unknown): value is { documentId: string; originalSha256: string; nativePagesPath: string; nativePagesSha256: string } {
|
|
return isRecord(value) && typeof value.documentId === "string" && typeof value.originalSha256 === "string"
|
|
&& typeof value.nativePagesPath === "string" && typeof value.nativePagesSha256 === "string";
|
|
}
|
|
|
|
async function readNativeDocument(input: { rootDirectory: string; versionId: string; documentId: string }) {
|
|
assertUuid(input.versionId);
|
|
const versionDirectory = path.join(path.resolve(input.rootDirectory), input.versionId);
|
|
const manifest = await readPrivateJson(path.join(versionDirectory, "manifest.json"), undefined, "OCR manifest integrity validation failed");
|
|
if (!isRecord(manifest) || manifest.schemaVersion !== "1" || manifest.versionId !== input.versionId || !Array.isArray(manifest.documents)) throw new Error("OCR manifest identity validation failed");
|
|
const entry = manifest.documents.find((value) => isManifestDocument(value) && value.documentId === input.documentId);
|
|
if (!entry || !isRecord(entry) || typeof entry.documentArtifactId !== "string") throw new Error("OCR review image not found");
|
|
const document = entry as { documentId: string; documentArtifactId: string; originalSha256: string; nativePagesPath: string; nativePagesSha256: string };
|
|
const native = await readPrivateJson(resolveArtifactPath(versionDirectory, document.nativePagesPath), document.nativePagesSha256, "OCR native page artifact integrity validation failed");
|
|
return { versionDirectory, entry: document, native };
|
|
}
|
|
|
|
function validateNativePages(value: unknown, versionId?: string, documentId?: string, documentSha256?: string): asserts value is Record<string, unknown> {
|
|
if (!isRecord(value) || value.schemaVersion !== "1" || (versionId && value.versionId !== versionId)
|
|
|| (documentId && value.documentId !== documentId) || (documentSha256 && value.documentSha256 !== documentSha256)
|
|
|| !Array.isArray(value.pages) || !Array.isArray(value.requestedPages)) throw new Error("OCR native page artifact integrity validation failed");
|
|
const pages = value.pages as Array<Record<string, unknown>>;
|
|
const requested = value.requestedPages as number[];
|
|
if (pages.some((page, index) => page.page !== index + 1 || typeof page.text !== "string" || page.textSha256 !== sha256Hex(page.text)
|
|
|| typeof page.rasterCoverage !== "number" || page.rasterCoverage < 0 || page.rasterCoverage > 1)
|
|
|| !sameNumbers(requested, [...new Set(requested)].sort((a, b) => a - b)) || requested.some((page) => page < 1 || page > pages.length)) {
|
|
throw new Error("OCR native page artifact integrity validation failed");
|
|
}
|
|
}
|
|
|
|
function validateReviewedPages(value: unknown, identity: Pick<ReviewedPagesIdentity, "versionId" | "candidateSha256" | "reviewedTextSha256">): asserts value is ReviewedPagesArtifact {
|
|
if (!isRecord(value) || value.schemaVersion !== "1" || value.versionId !== identity.versionId || value.candidateSha256 !== identity.candidateSha256
|
|
|| value.reviewedTextSha256 !== identity.reviewedTextSha256 || typeof value.sourceId !== "string" || typeof value.reviewedBy !== "string"
|
|
|| !Array.isArray(value.documents)) throw new Error("OCR reviewed artifact identity validation failed");
|
|
const documents = value.documents as ReviewedPagesArtifact["documents"];
|
|
const pages = documents.flatMap((document) => Array.isArray(document.pages) ? document.pages : []);
|
|
if (documents.some((document) => typeof document.documentId !== "string" || !Array.isArray(document.pages))
|
|
|| pages.some((page) => !Number.isInteger(page.page) || page.page < 1 || typeof page.candidateText !== "string" || !isRecord(page.ocr)
|
|
|| !Array.isArray(page.ocr.lines) || page.ocr.lines.some((line) => !isRecord(line) || typeof line.lineId !== "string"
|
|
|| typeof line.text !== "string" || line.lineSha256 !== sha256Hex(line.text))
|
|
|| (page.ocr.lines.length > 0 && page.candidateText !== page.ocr.lines.map(({ text }) => text).join("\n")))) {
|
|
throw new Error("OCR reviewed artifact integrity validation failed");
|
|
}
|
|
const reviewedText = pages.filter(({ candidateText }) => candidateText).map(({ candidateText }) => candidateText).join("\n\n");
|
|
if (sha256Hex(reviewedText) !== identity.reviewedTextSha256) throw new Error("OCR reviewed artifact integrity validation failed");
|
|
}
|
|
|
|
async function readPrivateJson(target: string, expectedSha256: string | undefined, message: string): Promise<unknown> {
|
|
const file = await lstat(target).catch(() => { throw new Error(message); });
|
|
if (!file.isFile() || (file.mode & 0o777) !== 0o600) throw new Error(message);
|
|
const bytes = await readFile(target);
|
|
if (expectedSha256 && sha256Hex(bytes) !== expectedSha256) throw new Error(message);
|
|
try { return JSON.parse(bytes.toString("utf8")); } catch { throw new Error(message); }
|
|
}
|
|
|
|
function sameNumbers(left: number[], right: number[]): boolean {
|
|
return left.length === right.length && left.every((value, index) => value === right[index]);
|
|
}
|
|
|
|
function resultIntegrityError(): Error {
|
|
return new Error("OCR result artifact integrity validation failed");
|
|
}
|
|
|
|
async function syncDirectory(directory: string): Promise<void> {
|
|
const handle = await open(directory, "r");
|
|
try { await handle.sync(); } finally { await handle.close(); }
|
|
}
|
|
|
|
function uuidV5(value: string): string {
|
|
const bytes = createHash("sha1").update(Buffer.concat([UUID_NAMESPACE_URL, Buffer.from(value)])).digest().subarray(0, 16);
|
|
bytes[6] = (bytes[6]! & 0x0f) | 0x50;
|
|
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
|
const hex = bytes.toString("hex");
|
|
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
}
|
|
|
|
function assertUuid(value: string): void {
|
|
if (!UUID.test(value)) throw new TypeError("Version identity must be a UUID");
|
|
}
|