139 lines
8.5 KiB
TypeScript
139 lines
8.5 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { access, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import test from "node:test";
|
|
import { buildOcrIdentity, type OcrResult } from "../../src/modules/ocr/client.js";
|
|
import { persistComposedCandidateArtifact, persistOcrResultArtifact, persistReviewImageArtifacts,
|
|
readOcrQualityReportArtifact, stageOcrArtifacts } from "../../src/modules/ocr/artifacts.js";
|
|
import { sha256Hex } from "../../src/shared/utils/ids.js";
|
|
import { prioritizeReviewPages } from "../../src/modules/ocr/review.js";
|
|
|
|
const png = Buffer.from("89504e470d0a1a0a01020304", "hex");
|
|
|
|
function fixturePng(page: number): Buffer {
|
|
return Buffer.concat([png, Buffer.from([page])]);
|
|
}
|
|
|
|
function result(documentSha256: string, pages: OcrResult["pages"]): OcrResult {
|
|
const identity = buildOcrIdentity({ documentSha256, pages: pages.map(({ page }) => page), idempotencyKey: "quality-job-key" });
|
|
return {
|
|
schemaVersion: "1", jobId: "quality-job", ...identity,
|
|
engine: { name: "paddleocr", version: "3.4.0", runtime: "paddlepaddle-3.2.2", device: "cpu", configVersion: "ocr-v2", dpi: 200 },
|
|
pages
|
|
};
|
|
}
|
|
|
|
async function stageQualityVersion(rootDirectory: string, versionId: string, quality: "warning" | "blocked") {
|
|
const documentId = `doc:${quality}`;
|
|
const original = Buffer.from(`%PDF-${quality}`);
|
|
const text = quality === "warning" ? "safe diagnostic fixture with enough non whitespace characters 123456789" : "short";
|
|
await stageOcrArtifacts({ rootDirectory, versionId, createdAt: "2026-09-22T12:00:00.000Z", documents: [{
|
|
documentId, documentKey: `${quality}.pdf`, bytes: original, requestedPages: [1],
|
|
pages: [{ page: 1, text: "weak native", rasterCoverage: 1, textSha256: sha256Hex("weak native") }]
|
|
}] });
|
|
await persistOcrResultArtifact({ rootDirectory, versionId, documentId, result: result(sha256Hex(original), [{
|
|
page: 1, width: 100, height: 100, processingMs: 1, text,
|
|
metrics: { lineCount: 1, nonWhitespaceCharacters: quality === "warning" ? 60 : 5, inkCoverage: 0.4,
|
|
medianConfidence: quality === "warning" ? 0.99 : 0.7, p10Confidence: 0.339,
|
|
lowConfidenceLineRatio: quality === "warning" ? 0.1 : 0.3 },
|
|
lines: [{ lineId: "p1-l1", text, confidence: 0.9, bbox: [1, 2, 30, 10] }]
|
|
}]) });
|
|
await persistReviewImageArtifacts({ rootDirectory, versionId, documentId,
|
|
images: [{ page: 1, bytes: png, sha256: sha256Hex(png) }] });
|
|
return { documentId, jobs: [{ documentId, remoteJobId: "quality-job", requestedPages: [1], state: "succeeded" }] };
|
|
}
|
|
|
|
test("quality report is private, immutable, text-free, and chained into a warning candidate", async (context) => {
|
|
const rootDirectory = await mkdtemp(path.join(os.tmpdir(), "rag-quality-report-"));
|
|
const versionId = "12121212-1212-4121-8121-121212121212";
|
|
context.after(() => rm(rootDirectory, { recursive: true, force: true }));
|
|
const input = await stageQualityVersion(rootDirectory, versionId, "warning");
|
|
|
|
const persisted = await persistComposedCandidateArtifact({ rootDirectory, versionId, jobs: input.jobs });
|
|
assert.ok(persisted.artifactPath);
|
|
const reportPath = path.join(rootDirectory, versionId, "quality-report.json");
|
|
const report = await readOcrQualityReportArtifact({ rootDirectory, versionId, artifactSha256: persisted.qualityReportSha256 });
|
|
assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
|
|
assert.equal((await stat(persisted.artifactPath)).mode & 0o777, 0o600);
|
|
assert.equal(report.pages[0]?.qualityOutcome, "warning");
|
|
assert.deepEqual(report.pages[0]?.warnings, ["LOW_P10_CONFIDENCE"]);
|
|
assert.equal(persisted.candidate.qualityReportSha256, persisted.qualityReportSha256);
|
|
assert.equal(persisted.candidate.documents[0]?.pages[0]?.qualityReportSha256, persisted.qualityReportSha256);
|
|
const serialized = await readFile(reportPath, "utf8");
|
|
assert.doesNotMatch(serialized, /safe diagnostic fixture|weak native|quality-job-key/u);
|
|
const repeated = await persistComposedCandidateArtifact({ rootDirectory, versionId, jobs: input.jobs });
|
|
assert.equal(repeated.qualityReportSha256, persisted.qualityReportSha256);
|
|
await writeFile(path.join(rootDirectory, versionId, "manifest.json"), "{}", { mode: 0o600 });
|
|
await assert.rejects(readOcrQualityReportArtifact({ rootDirectory, versionId,
|
|
artifactSha256: persisted.qualityReportSha256 }), /quality report integrity validation failed/u);
|
|
});
|
|
|
|
test("blocked pages retain every ordered reason and publish no candidate artifact", async (context) => {
|
|
const rootDirectory = await mkdtemp(path.join(os.tmpdir(), "rag-quality-blocked-"));
|
|
const versionId = "34343434-3434-4343-8343-343434343434";
|
|
context.after(() => rm(rootDirectory, { recursive: true, force: true }));
|
|
const input = await stageQualityVersion(rootDirectory, versionId, "blocked");
|
|
|
|
const persisted = await persistComposedCandidateArtifact({ rootDirectory, versionId, jobs: input.jobs });
|
|
assert.equal(persisted.artifactPath, null);
|
|
assert.deepEqual(persisted.candidate.documents[0]?.pages[0]?.blockingReasons,
|
|
["INSUFFICIENT_TEXT", "LOW_MEDIAN_CONFIDENCE", "EXCESS_LOW_CONFIDENCE_LINES"]);
|
|
assert.equal(persisted.candidate.documents[0]?.pages[0]?.primaryBlockingReason, "INSUFFICIENT_TEXT");
|
|
await access(path.join(rootDirectory, versionId, "quality-report.json"));
|
|
await assert.rejects(access(path.join(rootDirectory, versionId, "candidate-pages.json")));
|
|
});
|
|
|
|
test("review ordering places warned pages first and then pages with ambiguous tokens", () => {
|
|
const ordered = prioritizeReviewPages([
|
|
{ page: 1, qualityOutcome: "accepted" as const, risks: [] },
|
|
{ page: 2, qualityOutcome: "accepted" as const, risks: ["FATo7"] },
|
|
{ page: 3, qualityOutcome: "warning" as const, risks: [] }
|
|
]);
|
|
assert.deepEqual(ordered.map(({ page }) => page), [3, 2, 1]);
|
|
});
|
|
|
|
test("self-contained 25-page fixture retains p10 warnings, every review PNG, and a reviewable candidate", async (context) => {
|
|
const rootDirectory = await mkdtemp(path.join(os.tmpdir(), "rag-quality-25-pages-"));
|
|
const versionId = "56565656-5656-4565-8565-565656565656";
|
|
const documentId = "doc:twenty-five-pages";
|
|
const pages = Array.from({ length: 25 }, (_, index) => index + 1);
|
|
const warningPages = new Set([2, 20, 21]);
|
|
const original = Buffer.from("%PDF-self-contained-twenty-five-page-fixture");
|
|
context.after(() => rm(rootDirectory, { recursive: true, force: true }));
|
|
|
|
await stageOcrArtifacts({ rootDirectory, versionId, createdAt: "2026-09-22T12:00:00.000Z", documents: [{
|
|
documentId, documentKey: "twenty-five-pages.pdf", bytes: original, requestedPages: pages,
|
|
pages: pages.map((page) => {
|
|
const text = `native fixture page ${page}`;
|
|
return { page, text, rasterCoverage: 1, textSha256: sha256Hex(text) };
|
|
})
|
|
}] });
|
|
await persistOcrResultArtifact({ rootDirectory, versionId, documentId, result: result(sha256Hex(original), pages.map((page) => {
|
|
const text = `self-contained OCR fixture page ${page} has enough text for human review`;
|
|
const p10Confidence = warningPages.has(page) ? 0.339 : 0.95;
|
|
return {
|
|
page, width: 100, height: 100, processingMs: 1, text,
|
|
metrics: { lineCount: 1, nonWhitespaceCharacters: text.replace(/\s/gu, "").length, inkCoverage: 0.4,
|
|
medianConfidence: 0.99, p10Confidence, lowConfidenceLineRatio: 0.1 },
|
|
lines: [{ lineId: `p${page}-l1`, text, confidence: 0.99, bbox: [1, 2, 30, 10] }]
|
|
};
|
|
})) });
|
|
const images = await persistReviewImageArtifacts({ rootDirectory, versionId, documentId,
|
|
images: pages.map((page) => {
|
|
const bytes = fixturePng(page);
|
|
return { page, bytes, sha256: sha256Hex(bytes) };
|
|
}) });
|
|
|
|
const persisted = await persistComposedCandidateArtifact({ rootDirectory, versionId,
|
|
jobs: [{ documentId, remoteJobId: "quality-job", requestedPages: pages, state: "succeeded" }] });
|
|
const report = await readOcrQualityReportArtifact({ rootDirectory, versionId, artifactSha256: persisted.qualityReportSha256 });
|
|
|
|
assert.ok(persisted.artifactPath);
|
|
assert.equal(images.images.length, 25);
|
|
assert.equal(report.pages.length, 25);
|
|
assert.deepEqual(report.pages.filter(({ qualityOutcome }) => qualityOutcome === "warning").map(({ page }) => page), [2, 20, 21]);
|
|
assert.ok(report.pages.every(({ qualityOutcome }) => qualityOutcome === "accepted" || qualityOutcome === "warning"));
|
|
assert.equal(persisted.candidate.documents[0]?.pages.length, 25);
|
|
assert.deepEqual(persisted.candidate.documents[0]?.pages.filter(({ warnings }) => warnings.includes("LOW_P10_CONFIDENCE")).map(({ page }) => page), [2, 20, 21]);
|
|
});
|