import assert from "node:assert/strict"; import test from "node:test"; import { createApp } from "../../src/app.js"; import { env } from "../../src/config/env.js"; import { OcrDispatcher } from "../../src/modules/ocr/dispatcher.js"; import { OcrIndexingService } from "../../src/modules/ocr/indexing.js"; import { OcrReviewService, type OcrReviewCandidate } from "../../src/modules/ocr/review.js"; import { sha256Hex } from "../../src/shared/utils/ids.js"; const token = "local-e2e-token"; const scanVersion = "11111111-1111-4111-8111-111111111111"; const mixedVersion = "22222222-2222-4222-8222-222222222222"; function headers(json = false): Record { return { authorization: `Bearer ${token}`, ...(json ? { "content-type": "application/json" } : {}) }; } function candidate(): OcrReviewCandidate { const text = "CBGO4a"; return { versionId: scanVersion, sourceId: "src:scan", state: "review_required", candidateSha256: "candidate-hash", baseActiveVersionId: "previous-active", currentActiveVersionId: "previous-active", activateRequested: true, processingFingerprint: "fingerprint", metadataHash: "metadata", documents: [{ documentId: "doc:scan", pages: [{ page: 1, imageUrl: `/ingestions/${scanVersion}/documents/doc:scan/pages/1/image`, nativeText: "", ocr: { text, lines: [{ lineId: "line-1", text, confidence: 0.72, bbox: [1, 2, 30, 12], lineSha256: sha256Hex(text) }] }, candidateText: text, differences: ["native text is empty"], risks: [text] }] }] }; } test("local HTTP flow keeps native ingestion synchronous and activates only reviewed OCR content", async (context) => { const previous = { lifecycle: env.knowledgeLifecycleEnforced, enabled: env.ocrIngestEnabled, token: env.lifecycleAdminToken }; Object.assign(env, { knowledgeLifecycleEnforced: true, ocrIngestEnabled: true, lifecycleAdminToken: token }); context.after(() => Object.assign(env, { knowledgeLifecycleEnforced: previous.lifecycle, ocrIngestEnabled: previous.enabled, lifecycleAdminToken: previous.token })); let reviewCandidate = candidate(); let reviewedText = ""; const states = new Map([[scanVersion, "review_required"], [mixedVersion, "review_required"]]); const ingestService = { async ingest(input: { sourceRef: string }) { if (input.sourceRef === "native.pdf") { return { accepted: true, sourceId: "src:native", versionId: "native-active", versionNumber: 1, state: "active", filesDiscovered: 1, documentsProcessed: 1, chunksStored: 1, activated: true, noOp: false, collectionName: "rag" }; } const versionId = input.sourceRef === "mixed.pdf" ? mixedVersion : scanVersion; return { accepted: true, sourceId: `src:${input.sourceRef}`, versionId, versionNumber: 2, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false }; }, async cleanup() { return { deleted: 0 }; } }; const catalog = { async claimNextOcrJob() { return undefined; }, async getIngestionStatus(versionId: string) { const state = states.get(versionId); if (!state) return undefined; const mixed = versionId === mixedVersion; return { sourceId: mixed ? "src:mixed.pdf" : "src:scan.pdf", versionId, state, phase: state, activated: state === "active", documents: mixed ? [{ documentId: "doc:mixed", state: "ocr_complete", completedPages: 2, totalPages: 2, pages: [ { page: 1, method: "native", state: "native_complete" }, { page: 2, method: "ocr", state: "ocr_complete" } ] }] : [{ documentId: "doc:scan", state: "ocr_complete", completedPages: 1, totalPages: 1, pages: [{ page: 1, method: "ocr", state: "ocr_complete" }] }], error: null, statusUrl: `/ingestions/${versionId}`, reviewUrl: state === "review_required" ? `/ingestions/${versionId}/review` : null }; } }; const review = new OcrReviewService({ async loadCandidate(versionId) { return versionId === scanVersion ? reviewCandidate : undefined; }, async commitApproval(input) { reviewedText = input.reviewedText; reviewCandidate = { ...reviewCandidate, state: "indexing" }; states.set(scanVersion, "indexing"); }, async commitRejection() { throw new Error("rejection is outside this flow"); } }); const indexing = new OcrIndexingService({ async findReusableVersion() { return undefined; }, async indexReviewed(value) { assert.equal(value.reviewedText, "CBG04a"); return 1; }, async markReady(versionId) { states.set(versionId, "ready"); }, async settleReusable() { throw new Error("candidate is not reusable"); }, async activateVersion(_sourceId, versionId, expected) { assert.equal(expected, "previous-active"); states.set(versionId, "active"); return versionId; } }); const server = createApp({ ingestService: ingestService as never, catalog: catalog as never, ocrClient: {} as never, reviewService: review, indexingService: indexing, startReconciler: false }).listen(0); context.after(() => server.close()); const address = server.address(); assert.ok(address && typeof address === "object"); const base = `http://127.0.0.1:${address.port}`; const ingest = (sourceRef: string) => fetch(`${base}/ingest`, { method: "POST", headers: headers(true), body: JSON.stringify({ sourceType: "file", sourceRef }) }); assert.equal((await ingest("native.pdf")).status, 201); const accepted = await ingest("scanned.pdf"); assert.equal(accepted.status, 202); assert.deepEqual(await accepted.json(), { accepted: true, sourceId: "src:scanned.pdf", versionId: scanVersion, versionNumber: 2, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${scanVersion}`, reviewUrl: null, activated: false }); const reviewResponse = await fetch(`${base}/ingestions/${scanVersion}/review`, { headers: headers() }); assert.equal(reviewResponse.status, 200); assert.deepEqual((await reviewResponse.json() as OcrReviewCandidate).documents[0]?.pages[0]?.risks, ["CBGO4a"]); const approval = await fetch(`${base}/ingestions/${scanVersion}/approve`, { method: "POST", headers: headers(true), body: JSON.stringify({ candidateSha256: "candidate-hash", expectedActiveVersionId: "previous-active", reviewedBy: "local-reviewer", corrections: [{ documentId: "doc:scan", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("CBGO4a"), replacementText: "CBG04a" }] }) }); assert.equal(approval.status, 200); assert.deepEqual(await approval.json(), { versionId: scanVersion, state: "active", activated: true, activatedVersionId: scanVersion }); assert.equal(reviewedText, "CBG04a"); assert.equal((await fetch(`${base}/ingestions/${scanVersion}`, { headers: headers() }).then((response) => response.json()) as { state: string }).state, "active"); assert.equal((await ingest("mixed.pdf")).status, 202); const mixed = await fetch(`${base}/ingestions/${mixedVersion}`, { headers: headers() }).then((response) => response.json()) as { documents: Array<{ pages: Array<{ method: string }> }> }; assert.deepEqual(mixed.documents[0]?.pages.map(({ method }) => method), ["native", "ocr"]); }); test("resends reuse identity, OCR exhaustion fails closed, and catalog absence returns 503", async (context) => { const previous = { lifecycle: env.knowledgeLifecycleEnforced, enabled: env.ocrIngestEnabled, token: env.lifecycleAdminToken }; Object.assign(env, { knowledgeLifecycleEnforced: true, ocrIngestEnabled: true, lifecycleAdminToken: token }); context.after(() => Object.assign(env, { knowledgeLifecycleEnforced: previous.lifecycle, ocrIngestEnabled: previous.enabled, lifecycleAdminToken: previous.token })); const identities = new Map(); let creations = 0; const ingestService = { async ingest(input: { sourceRef: string }) { let versionId = identities.get(input.sourceRef); if (!versionId) { versionId = input.sourceRef === "scan.pdf" ? scanVersion : mixedVersion; identities.set(input.sourceRef, versionId); creations += 1; } return { accepted: true, sourceId: `src:${input.sourceRef}`, versionId, versionNumber: 1, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false }; }, async cleanup() { return { deleted: 0 }; } }; const server = createApp({ ingestService: ingestService as never, catalog: undefined, startReconciler: false }).listen(0); context.after(() => server.close()); const address = server.address(); assert.ok(address && typeof address === "object"); const base = `http://127.0.0.1:${address.port}`; const resend = () => fetch(`${base}/ingest`, { method: "POST", headers: headers(true), body: JSON.stringify({ sourceType: "file", sourceRef: "scan.pdf" }) }).then((response) => response.json()) as Promise<{ versionId: string }>; assert.deepEqual([(await resend()).versionId, (await resend()).versionId], [scanVersion, scanVersion]); assert.equal(creations, 1); const calls: string[] = []; let candidateState = "indexing"; const dispatcher = new OcrDispatcher({ async claimNextOcrJob() { return { jobId: "job", versionId: scanVersion, documentId: "doc", remoteJobId: null, remoteIdempotencyKey: "stable-key", state: "running", requestedPages: [1], completedPages: 0, configVersion: "ocr-v1", attemptCount: 1, heartbeatAt: null, leaseExpiresAt: null, nextAttemptAt: null, errorCode: null, errorDetail: null }; }, async claimOcrJob() { return undefined; }, async recoverExpiredOcrLeases() { return []; }, async setOcrRemoteJob() { calls.push("remote"); }, async requeueOcrJob() { calls.push("requeue"); }, async completeOcrJob() { calls.push("complete"); return false; }, async failOcrJob() { calls.push("job-failed"); }, async markReviewRequired() { calls.push("review"); }, async markFailed() { candidateState = "failed"; calls.push("version-failed"); } }, { async submit() { throw new Error("OCR connection unavailable after bounded retries"); } } as never, async () => ({ bytes: Buffer.from("pdf"), documentSha256: sha256Hex("pdf") })); assert.equal(await dispatcher.runOnce(), "failed"); assert.deepEqual(calls, ["job-failed", "version-failed"]); assert.equal(candidateState, "failed"); const unavailableStatus = await fetch(`${base}/ingestions/${scanVersion}`, { headers: headers() }); assert.equal(unavailableStatus.status, 503); assert.equal((await unavailableStatus.json() as { code: string }).code, "CATALOG_UNAVAILABLE"); const unavailableRetrieval = await fetch(`${base}/retrieve`, { method: "POST", headers: headers(true), body: JSON.stringify({ query: "prior active", mode: "documental", intent: "specific" }) }); assert.equal(unavailableRetrieval.status, 503); assert.equal((await unavailableRetrieval.json() as { code: string }).code, "CATALOG_UNAVAILABLE"); });