187 lines
11 KiB
TypeScript
187 lines
11 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
import { createApp } from "../../src/app.js";
|
|
import { env } from "../../src/config/env.js";
|
|
import { OcrDispatcher } from "../../src/modules/ocr/dispatcher.js";
|
|
import { OcrReadyIndexingService } from "../../src/modules/ocr/indexing.js";
|
|
import { OcrReviewService, type OcrReviewCandidate } from "../../src/modules/ocr/review.js";
|
|
import { sha256Hex } from "../../src/shared/utils/ids.js";
|
|
|
|
const token = "local-e2e-token";
|
|
const scanVersion = "11111111-1111-4111-8111-111111111111";
|
|
const mixedVersion = "22222222-2222-4222-8222-222222222222";
|
|
|
|
function headers(json = false): Record<string, string> {
|
|
return { authorization: `Bearer ${token}`, ...(json ? { "content-type": "application/json" } : {}) };
|
|
}
|
|
|
|
function candidate(): OcrReviewCandidate {
|
|
const text = "CBGO4a";
|
|
return {
|
|
versionId: scanVersion,
|
|
sourceId: "src:scan",
|
|
state: "review_required",
|
|
candidateSha256: "candidate-hash",
|
|
baseActiveVersionId: "previous-active",
|
|
currentActiveVersionId: "previous-active",
|
|
activateRequested: true,
|
|
processingFingerprint: "fingerprint",
|
|
metadataHash: "metadata",
|
|
documents: [{
|
|
documentId: "doc:scan",
|
|
pages: [{
|
|
page: 1,
|
|
imageUrl: `/ingestions/${scanVersion}/documents/doc:scan/pages/1/image`,
|
|
nativeText: "",
|
|
ocr: { text, lines: [{ lineId: "line-1", text, confidence: 0.72, bbox: [1, 2, 30, 12], lineSha256: sha256Hex(text) }] },
|
|
candidateText: text,
|
|
differences: ["native text is empty"],
|
|
risks: [text]
|
|
}]
|
|
}]
|
|
};
|
|
}
|
|
|
|
test("local HTTP flow keeps native ingestion synchronous and stops approved OCR at the indexing boundary", async (context) => {
|
|
const previous = { lifecycle: env.knowledgeLifecycleEnforced, enabled: env.ocrIngestEnabled, token: env.lifecycleAdminToken };
|
|
Object.assign(env, { knowledgeLifecycleEnforced: true, ocrIngestEnabled: true, lifecycleAdminToken: token });
|
|
context.after(() => Object.assign(env, {
|
|
knowledgeLifecycleEnforced: previous.lifecycle,
|
|
ocrIngestEnabled: previous.enabled,
|
|
lifecycleAdminToken: previous.token
|
|
}));
|
|
let reviewCandidate = candidate();
|
|
let reviewedText = "";
|
|
const states = new Map([[scanVersion, "review_required"], [mixedVersion, "review_required"]]);
|
|
const ingestService = {
|
|
async ingest(input: { sourceRef: string }) {
|
|
if (input.sourceRef === "native.pdf") {
|
|
return { accepted: true, sourceId: "src:native", versionId: "native-active", versionNumber: 1, state: "active", filesDiscovered: 1, documentsProcessed: 1, chunksStored: 1, activated: true, noOp: false, collectionName: "rag" };
|
|
}
|
|
const versionId = input.sourceRef === "mixed.pdf" ? mixedVersion : scanVersion;
|
|
return { accepted: true, sourceId: `src:${input.sourceRef}`, versionId, versionNumber: 2, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false };
|
|
},
|
|
async cleanup() { return { deleted: 0 }; }
|
|
};
|
|
const catalog = {
|
|
async claimNextOcrJob() { return undefined; },
|
|
async getIngestionStatus(versionId: string) {
|
|
const state = states.get(versionId);
|
|
if (!state) return undefined;
|
|
const mixed = versionId === mixedVersion;
|
|
return {
|
|
sourceId: mixed ? "src:mixed.pdf" : "src:scan.pdf", versionId, state, phase: state,
|
|
activated: state === "active",
|
|
documents: mixed
|
|
? [{ documentId: "doc:mixed", state: "ocr_complete", completedPages: 2, totalPages: 2, pages: [
|
|
{ page: 1, method: "native", state: "native_complete" }, { page: 2, method: "ocr", state: "ocr_complete" }
|
|
] }]
|
|
: [{ documentId: "doc:scan", state: "ocr_complete", completedPages: 1, totalPages: 1, pages: [{ page: 1, method: "ocr", state: "ocr_complete" }] }],
|
|
error: null, statusUrl: `/ingestions/${versionId}`, reviewUrl: state === "review_required" ? `/ingestions/${versionId}/review` : null
|
|
};
|
|
}
|
|
};
|
|
const review = new OcrReviewService({
|
|
async loadCandidate(versionId) { return versionId === scanVersion ? reviewCandidate : undefined; },
|
|
async commitApproval(input) {
|
|
reviewedText = input.reviewedText;
|
|
reviewCandidate = { ...reviewCandidate, state: "indexing" };
|
|
states.set(scanVersion, "indexing");
|
|
},
|
|
async commitRejection() { throw new Error("rejection is outside this flow"); }
|
|
});
|
|
let indexingCalls = 0;
|
|
const indexing = new OcrReadyIndexingService({
|
|
async findReusableVersion() { return undefined; },
|
|
async indexReviewed(value) { indexingCalls += 1; assert.equal(value.reviewedText, "CBG04a"); return 1; },
|
|
async markReady(versionId) { states.set(versionId, "ready"); },
|
|
async settleReusable() { throw new Error("candidate is not reusable"); },
|
|
async activateVersion(_sourceId, versionId, expected) {
|
|
assert.equal(expected, "previous-active");
|
|
states.set(versionId, "active");
|
|
return versionId;
|
|
}
|
|
} as never);
|
|
const server = createApp({ ingestService: ingestService as never, catalog: catalog as never, ocrClient: {} as never, reviewService: review, indexingService: indexing, startReconciler: false }).listen(0);
|
|
context.after(() => server.close());
|
|
const address = server.address();
|
|
assert.ok(address && typeof address === "object");
|
|
const base = `http://127.0.0.1:${address.port}`;
|
|
const ingest = (sourceRef: string) => fetch(`${base}/ingest`, { method: "POST", headers: headers(true), body: JSON.stringify({ sourceType: "file", sourceRef }) });
|
|
|
|
assert.equal((await ingest("native.pdf")).status, 201);
|
|
const accepted = await ingest("scanned.pdf");
|
|
assert.equal(accepted.status, 202);
|
|
assert.deepEqual(await accepted.json(), {
|
|
accepted: true, sourceId: "src:scanned.pdf", versionId: scanVersion, versionNumber: 2,
|
|
state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${scanVersion}`, reviewUrl: null, activated: false
|
|
});
|
|
const reviewResponse = await fetch(`${base}/ingestions/${scanVersion}/review`, { headers: headers() });
|
|
assert.equal(reviewResponse.status, 200);
|
|
assert.deepEqual((await reviewResponse.json() as OcrReviewCandidate).documents[0]?.pages[0]?.risks, ["CBGO4a"]);
|
|
const approval = await fetch(`${base}/ingestions/${scanVersion}/approve`, {
|
|
method: "POST", headers: headers(true), body: JSON.stringify({
|
|
candidateSha256: "candidate-hash", expectedActiveVersionId: "previous-active", reviewedBy: "local-reviewer",
|
|
corrections: [{ documentId: "doc:scan", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("CBGO4a"), replacementText: "CBG04a" }]
|
|
})
|
|
});
|
|
assert.equal(approval.status, 200);
|
|
assert.deepEqual(await approval.json(), { versionId: scanVersion, state: "ready", activated: false });
|
|
assert.equal(reviewedText, "CBG04a");
|
|
assert.equal(indexingCalls, 1);
|
|
assert.equal((await fetch(`${base}/ingestions/${scanVersion}`, { headers: headers() }).then((response) => response.json()) as { state: string }).state, "ready");
|
|
|
|
assert.equal((await ingest("mixed.pdf")).status, 202);
|
|
const mixed = await fetch(`${base}/ingestions/${mixedVersion}`, { headers: headers() }).then((response) => response.json()) as { documents: Array<{ pages: Array<{ method: string }> }> };
|
|
assert.deepEqual(mixed.documents[0]?.pages.map(({ method }) => method), ["native", "ocr"]);
|
|
});
|
|
|
|
test("resends reuse identity, OCR exhaustion fails closed, and catalog absence returns 503", async (context) => {
|
|
const previous = { lifecycle: env.knowledgeLifecycleEnforced, enabled: env.ocrIngestEnabled, token: env.lifecycleAdminToken };
|
|
Object.assign(env, { knowledgeLifecycleEnforced: true, ocrIngestEnabled: true, lifecycleAdminToken: token });
|
|
context.after(() => Object.assign(env, {
|
|
knowledgeLifecycleEnforced: previous.lifecycle,
|
|
ocrIngestEnabled: previous.enabled,
|
|
lifecycleAdminToken: previous.token
|
|
}));
|
|
const identities = new Map<string, string>();
|
|
let creations = 0;
|
|
const ingestService = {
|
|
async ingest(input: { sourceRef: string }) {
|
|
let versionId = identities.get(input.sourceRef);
|
|
if (!versionId) { versionId = input.sourceRef === "scan.pdf" ? scanVersion : mixedVersion; identities.set(input.sourceRef, versionId); creations += 1; }
|
|
return { accepted: true, sourceId: `src:${input.sourceRef}`, versionId, versionNumber: 1, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false };
|
|
},
|
|
async cleanup() { return { deleted: 0 }; }
|
|
};
|
|
const server = createApp({ ingestService: ingestService as never, catalog: undefined, startReconciler: false }).listen(0);
|
|
context.after(() => server.close());
|
|
const address = server.address();
|
|
assert.ok(address && typeof address === "object");
|
|
const base = `http://127.0.0.1:${address.port}`;
|
|
const resend = () => fetch(`${base}/ingest`, { method: "POST", headers: headers(true), body: JSON.stringify({ sourceType: "file", sourceRef: "scan.pdf" }) }).then((response) => response.json()) as Promise<{ versionId: string }>;
|
|
assert.deepEqual([(await resend()).versionId, (await resend()).versionId], [scanVersion, scanVersion]);
|
|
assert.equal(creations, 1);
|
|
|
|
const calls: string[] = [];
|
|
let candidateState = "indexing";
|
|
const dispatcher = new OcrDispatcher({
|
|
async claimNextOcrJob() { return { jobId: "job", versionId: scanVersion, documentId: "doc", remoteJobId: null, remoteIdempotencyKey: "stable-key", state: "running", requestedPages: [1], completedPages: 0, configVersion: "ocr-v2", attemptCount: 1, heartbeatAt: null, leaseExpiresAt: null, nextAttemptAt: null, errorCode: null, errorDetail: null }; },
|
|
async claimOcrJob() { return undefined; }, async recoverExpiredOcrLeases() { return []; }, async setOcrRemoteJob() { calls.push("remote"); },
|
|
async requeueOcrJob() { calls.push("requeue"); }, async completeOcrJob() { calls.push("complete"); return false; },
|
|
async failOcrJob() { calls.push("job-failed"); }, async markReviewRequired() { calls.push("review"); },
|
|
async markFailed() { candidateState = "failed"; calls.push("version-failed"); }
|
|
}, { async submit() { throw new Error("OCR connection unavailable after bounded retries"); } } as never,
|
|
async () => ({ bytes: Buffer.from("pdf"), documentSha256: sha256Hex("pdf") }),
|
|
async (_job, result) => result, async () => undefined);
|
|
assert.equal(await dispatcher.runOnce(), "failed");
|
|
assert.deepEqual(calls, ["job-failed", "version-failed"]);
|
|
assert.equal(candidateState, "failed");
|
|
|
|
const unavailableStatus = await fetch(`${base}/ingestions/${scanVersion}`, { headers: headers() });
|
|
assert.equal(unavailableStatus.status, 503);
|
|
assert.equal((await unavailableStatus.json() as { code: string }).code, "CATALOG_UNAVAILABLE");
|
|
const unavailableRetrieval = await fetch(`${base}/retrieve`, { method: "POST", headers: headers(true), body: JSON.stringify({ query: "prior active", mode: "documental", intent: "specific" }) });
|
|
assert.equal(unavailableRetrieval.status, 503);
|
|
assert.equal((await unavailableRetrieval.json() as { code: string }).code, "CATALOG_UNAVAILABLE");
|
|
});
|