From 53c4e0a62b3cca3a3fcc2276dd0d5e7e6d7fe513 Mon Sep 17 00:00:00 2001 From: Paco POR-CORREO Date: Tue, 15 Sep 2026 15:48:45 +0200 Subject: [PATCH] feat(ocr): complete reviewed ingestion workflow --- Dockerfile | 7 +- docs/HISTORIAL_SESIONES.md | 64 ++- .../ocr-ingest-integration/apply-progress.md | 161 ++++++- .../changes/ocr-ingest-integration/tasks.md | 28 +- public/playground/app.js | 125 ++++++ public/playground/index.html | 29 ++ public/playground/styles.css | 17 + src/api/openapi.ts | 135 +++++- src/app.ts | 136 +++++- src/config/env.ts | 10 +- src/modules/catalog/reconciler.ts | 13 +- src/modules/catalog/repository.ts | 308 ++++++++++++-- src/modules/ingest/service.ts | 192 ++++++++- src/modules/ocr/client.ts | 5 +- src/modules/ocr/dispatcher.ts | 108 +++++ src/modules/ocr/indexing.ts | 67 +++ src/modules/ocr/retention.ts | 45 ++ src/modules/ocr/review.ts | 143 +++++++ tests/ocr/contracts-deploy.test.ts | 71 ++++ tests/ocr/dispatcher.test.ts | 397 ++++++++++++++++++ tests/ocr/e2e.test.ts | 183 ++++++++ tests/ocr/retention.test.ts | 134 ++++++ tests/ocr/review.test.ts | 185 ++++++++ 23 files changed, 2484 insertions(+), 79 deletions(-) create mode 100644 src/modules/ocr/dispatcher.ts create mode 100644 src/modules/ocr/indexing.ts create mode 100644 src/modules/ocr/retention.ts create mode 100644 src/modules/ocr/review.ts create mode 100644 tests/ocr/contracts-deploy.test.ts create mode 100644 tests/ocr/dispatcher.test.ts create mode 100644 tests/ocr/e2e.test.ts create mode 100644 tests/ocr/retention.test.ts create mode 100644 tests/ocr/review.test.ts diff --git a/Dockerfile b/Dockerfile index 98012ae..ea3dc14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,11 +9,14 @@ RUN npm run build FROM node:22-bookworm-slim AS runtime WORKDIR /app -ENV NODE_ENV=production +ENV NODE_ENV=production OCR_ARTIFACT_ROOT=/data/ingestions COPY package.json package-lock.json ./ RUN npm ci --omit=dev COPY --from=build /app/dist ./dist COPY --from=build /app/migrations ./migrations COPY public ./public +RUN install -d -o node -g node -m 700 /data/ingestions +VOLUME ["/data/ingestions"] +USER node EXPOSE 3000 -CMD ["node", "dist/server.js"] +CMD ["sh", "-c", "node dist/modules/catalog/migrations.js && exec node dist/server.js"] diff --git a/docs/HISTORIAL_SESIONES.md b/docs/HISTORIAL_SESIONES.md index 68c3a86..1a4b2aa 100644 --- a/docs/HISTORIAL_SESIONES.md +++ b/docs/HISTORIAL_SESIONES.md @@ -2,14 +2,64 @@ **Proyecto:** Workspace de tools IA para empresas **Modulo:** RAG -**Ultima actualizacion:** 2026-09-14 -**Ultima modificacion por:** Subagente OCR Unit 7 Client +**Ultima actualizacion:** 2026-09-15 +**Ultima modificacion por:** Subagente Implement Unit 13 Local E2E **Estado:** Activo --- ## Registro de sesion +### 2026-09-15 - Subagente Implement Unit 13 Local E2E +**Agent:** Subagente Implement Unit 13 Local E2E · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5af65977ffehYI4eK2l177YAL` +**Responsibility:** Implement only OCR Unit 13 tasks 7.1–7.3 under strict TDD, preserving Units 8–12 and excluding production acceptance task 7.4 and Git delivery work. +**Work:** Added deterministic localhost E2E coverage for native, scanned review/approval/activation, mixed documents, resend identity, OCR-down fail-closed behavior, and catalog-down `503` responses. No production code changed. +**Validation:** The new requirement tests passed immediately as 2/2 baseline/characterization evidence; canonical Node passed 78/78, offline Python passed 13/13, and check/build/whitespace/cleanup/process gates passed. +**Accounting:** 183 functional plus 52 metadata changed lines, 235 total, within the 400-line Unit 13 budget. Tasks 7.1–7.3 are complete; production task 7.4 remains unchecked and untouched. +**Files:** `tests/ocr/e2e.test.ts`, OpenSpec tasks/progress, and this history. No external or production service was contacted; `ocr-service/.venv` was preserved. + +--- + +### 2026-09-15 - Subagente Implement Unit 12 Contracts Deploy +**Agent:** Subagente Implement Unit 12 Contracts Deploy · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b09a3a3ffetN0oeikOd4f33P` +**Responsibility:** Implement only OCR Unit 12 tasks 6.3–6.4 under strict TDD, preserving Units 8–11 and excluding all 7.x E2E, production, and Git delivery work. +**Work:** Added accurate authenticated OCR ingestion/status/review/correction/decision OpenAPI contracts, durable private OCR deployment defaults, and a non-root migration-aware RAG container with `/data/ingestions` volume wiring. +**Validation:** Genuine initial RED was 0/3 and triangulation produced a second RED; focused GREEN passed 3/3, relevant regressions 45/45, canonical Node 76/76, check/build/whitespace gates, sanitized Docker build/inspection, cleanup, and process checks passed. +**Accounting:** 219 functional plus 50 metadata changed lines, 269 total, within the 400-line Unit 12 budget. Tasks 6.3–6.4 are complete; all 7.x tasks remain pending. +**Files:** `src/api/openapi.ts`, `src/config/env.ts`, `Dockerfile`, `tests/ocr/contracts-deploy.test.ts`, OpenSpec tasks/progress, and this history. The authoritative lifecycle/OCR contract remained read-only. + +--- + +### 2026-09-15 - Subagente Implement Unit 11 Retention +**Agent:** Subagente Implement Unit 11 Retention · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b1c9293ffeDc3y3lJLgJ8UHF` +**Responsibility:** Implement only OCR Unit 11 tasks 6.1–6.2 under strict TDD, preserving Units 8–10 and excluding OpenAPI, deployment, E2E, production, and Git delivery work. +**Work:** Added state-aware OCR retention with exact TTL selection, review expiry, active-safe CAS deletion, restart resumption, reconciler execution, and flag-off candidate API isolation while preserving native synchronous ingestion. +**Validation:** Genuine missing-module and activation-race RED evidence reached focused GREEN 6/6; relevant regressions passed 28/28; canonical Node passed 73/73; check, build, tracked/untracked whitespace, temporary-directory cleanup, and process checks passed. +**Accounting:** 277 functional plus 50 metadata changed lines, 327 total, within the 400-line Unit 11 budget. Tasks 6.1–6.2 are complete; tasks 6.3–6.4 and 7.x remain pending. +**Files:** `src/app.ts`, `src/modules/catalog/{repository,reconciler}.ts`, `src/modules/ocr/retention.ts`, `tests/ocr/{dispatcher,review,retention}.test.ts`, OpenSpec tasks/progress, and this history. + +--- + +### 2026-09-15 - Subagente Implement Unit 10 Review UI +**Agent:** Subagente Implement Unit 10 Review UI · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b3af43bffewaqPFGRUOsi6Qt` +**Responsibility:** Implement only OCR Unit 10 task 5.4 under strict TDD, preserving Units 8–9 and excluding retention, deploy, and E2E work. +**Work:** Added authenticated, state-safe rejection with durable audit completion and zero indexing/activation side effects. Extended the existing static playground with authenticated candidate inspection, protected images, corrections, approval, and rejection controls. +**Validation:** Genuine RED was 5/7; focused GREEN/refactor passed 7/7, the localhost rejection harness passed 1/1, canonical Node passed 67/67, and check/build/whitespace gates passed. No live external dependency or persistent process was used. +**Accounting:** 271 functional plus 42 metadata changed lines, 313 total, within the 400-line Unit 10 budget. Task 5.4 is complete; tasks 6.x/7.x remain pending. +**Files:** `src/app.ts`, `src/modules/ocr/review.ts`, `tests/ocr/review.test.ts`, `public/playground/{index.html,app.js,styles.css}`, OpenSpec tasks/progress, and this history. + +--- + +### 2026-09-15 - Subagente Implement Unit 9 Core +**Agent:** Subagente Implement Unit 9 Core · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b4d20b9ffewoOAG5oiIu25fk` +**Responsibility:** Implement only OCR Unit 9 tasks 5.1–5.3 under strict TDD, preserving Unit 8 and excluding rejection/UI work. +**Work:** Added authenticated review/approval routes, auditable review and atomic correction boundaries, stale-write conflicts, and new/reusable indexing activation CAS behavior. Rejected candidates are refused by indexing without embeddings. +**Validation:** RED failed on the missing indexing module; focused GREEN passed 5/5, canonical Node passed 65/65, and check/build/tracked plus untracked whitespace gates passed. No external service or persistent process was used. +**Accounting:** 341 functional plus 38 metadata changed lines, 379 total. Unit 10 task 5.4 remains pending. +**Files:** `src/app.ts`, `src/modules/ocr/{review,indexing}.ts`, `tests/ocr/review.test.ts`, OpenSpec tasks/progress, and this history. + +--- + ### 2026-09-14 - Agente RAG 2 - Diagnostico de resultados vacios de subagentes **Modelo:** openai/gpt-5.6-sol @@ -32,6 +82,16 @@ **Archivos modificados:** - `docs/HISTORIAL_SESIONES.md` + +--- +### 2026-09-15 - Subagente Recover OCR Unit 8 +**Agent:** Subagente Recover OCR Unit 8 · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b7b4242ffekQxMZTYXOvPWGA` +**Responsibility:** Recover only Unit 8 tasks 4.1, 4.4, and 4.5 under strict TDD; no Unit 9 or Git delivery work. +**Work:** Completed OCR runtime wiring, exact lease recovery, deterministic pending reuse, mixed-document progress, upload/status HTTP behavior, and fail-closed retrieval evidence. +**Validation:** Persisted evidence records focused dispatcher 9/9, focused repository OCR 6/6, canonical Node 60/60, and clean check/build/whitespace gates; curl returned 202 then authenticated 200. The initially leaked harness child was detected and terminated; temporary files and port were rechecked clean. +**Evidence chain:** Native failed/interrupted revision `sha256:182cc73954a8518d03bd6c3ef5f4141b352aa9b69fca4177447d76deaddad223` preserved an 815-line candidate. The maintainer-selected `auto-chain` / `feature-branch-chain` preflight reset at revision `sha256:c0db82cf3db8ae790d34e4c310b998f97bb36864c97c3aa84e4c87ab77d07c2a` preserved that candidate and bounded only the additional recovery work. Fresh passed evidence revision: `sha256:e424afe7b2f8efd547dcfd0baf64ec1b46ac344f39afd87259560a094d59163f`. +**Accounting:** Relative to committed HEAD `ac046e3`, the complete Unit 8 review slice is 1,045 functional changed lines plus 41 required metadata lines, for 1,086 total; it is not within 400 lines and has no `size:exception`. Only the post-reset recovery delta—350 functional plus 41 metadata lines, 391 total—fits the separate 400-line recovery objective. +**Files:** `src/app.ts`, `src/config/env.ts`, `src/modules/{catalog,ingest,ocr}/`, `tests/ocr/dispatcher.test.ts`, OpenSpec tasks/progress, and this history. - `/home/pancho/Documentos/Empresa/IA/herramientas/docs/gentle-ai/SEGUIMIENTO_DESCUBRIMIENTOS_MEJORAS_GENTLE_AI.md` --- diff --git a/openspec/changes/ocr-ingest-integration/apply-progress.md b/openspec/changes/ocr-ingest-integration/apply-progress.md index 570f0ee..476b49b 100644 --- a/openspec/changes/ocr-ingest-integration/apply-progress.md +++ b/openspec/changes/ocr-ingest-integration/apply-progress.md @@ -3,9 +3,9 @@ ## Current State - **Mode:** Strict TDD -- **Delivery:** Feature-branch-chain; Units 1–7 implemented; the maintainer approved `size:exception` for Unit 7's cohesive 506 functional authored lines and 585 total native-accounting lines -- **Completed tasks:** 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 2.5, 3.1, 3.2, 3.3, 3.4, 4.2, 4.3 -- **Overall task progress:** 15/30 complete +- **Delivery:** Feature-branch-chain; Units 1–13 implemented as bounded slices; production acceptance remains pending +- **Completed tasks:** 1.1–1.4, 2.1–2.5, 3.1–3.4, 4.1–4.5, 5.1–5.4, 6.1–6.4, 7.1–7.3 +- **Overall task progress:** 29/30 complete ## Unit 1: Migration @@ -271,3 +271,158 @@ None — the migration follows the proposal, specifications, design, and closed - Corrective work unit `unit-7-size-exception-validation` uses attempt token `sha256:0109cfd8d3b37a5672c65a37700269b89cb80e4e357a39f68a920779a1715c39` and must settle against the failed revision through the parent. - Production and test files remain unchanged for the corrective rerun; only approval/reset evidence and fresh validation metadata may change. - Task 4.1 remains pending, while tasks 4.2 and 4.3 remain complete. Unit 8 was not started. + +## Unit 8: Dispatcher and Runtime Orchestration +- Completed multi-document progress, uniqueness-race reuse, exact lease recovery, persisted-key dispatch, default runtime wiring, authenticated status, and fail-closed retrieval; no activation or embeddings occur on OCR failure. + +### Evidence Chain and Review Accounting +- **Complete review slice:** Relative to committed HEAD `ac046e3`, Unit 8 contains 1,045 functional changed lines plus 41 required metadata lines, for 1,086 total changed lines. The complete slice is not within 400 lines, and no `size:exception` was approved or recorded for it. +- **Interrupted preserved candidate:** Native failed/interrupted revision `sha256:182cc73954a8518d03bd6c3ef5f4141b352aa9b69fca4177447d76deaddad223` contained 815 changed lines. +- **Auto-chain reset:** The maintainer-selected preflight strategy was `auto-chain` with `feature-branch-chain`. Native reset revision `sha256:c0db82cf3db8ae790d34e4c310b998f97bb36864c97c3aa84e4c87ab77d07c2a` preserved the interrupted candidate and scoped the next bounded recovery objective only to additional changes; it did not approve the complete Unit 8 slice as a size exception. +- **Recovery delta:** After that reset, the bounded recovery added 350 functional changed lines plus 41 required metadata lines, for 391 total changed lines within the separate 400-line recovery objective. +- **Passed recovery:** Fresh passed evidence revision `sha256:e424afe7b2f8efd547dcfd0baf64ec1b46ac344f39afd87259560a094d59163f` closes the interrupted → auto-chain reset → passed recovery chain. + +### TDD Cycle Evidence +| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR | +|---|---|---|---|---|---|---|---| +| 4.1 | `tests/ocr/{dispatcher,client}.test.ts` | HTTP/integration | Recovered baseline 7/7 passed; not labeled RED | New scenarios produced genuine 5/9 RED; catalog `/retrieve` already failed closed | Unit 8 9/9; client 6/6 | 201/202/status, OCR failure, 503 retrieval, retries/integrity | Consolidated persisted-key assertion; 9/9 | +| 4.4 | `tests/ocr/dispatcher.test.ts` | Unit/integration | Recovered baseline 7/7 | Exact recovery failed by claiming unrelated work | 9/9 | Live/new queue separation plus periodic dispatch | Exact-job dispatch helper; 9/9 | +| 4.5 | `tests/ocr/dispatcher.test.ts` | HTTP/filesystem | Recovered baseline 7/7 | Runtime wiring RED was 8/10 | 9/9 | Multipart upload, durable reload, duplicate race, mixed documents | Shared queue drain; 9/9 | + +### Work Unit Evidence +| Evidence | Result | +|---|---| +| Focused test | `NODE_ENV=test npx --no-install tsx --test tests/ocr/dispatcher.test.ts` — 9 passed, 0 failed. | +| Focused repository OCR test | `NODE_ENV=test npx --no-install tsx --test tests/catalog/repository-ocr.test.ts` — 6 passed, 0 failed. | +| Canonical Node test | `npm test` — 60 passed, 0 failed. | +| Runtime harness | Bounded localhost curl — ingest `202/ocr_queued`; authenticated status `200/ocr_queued`; leaked child from initial shell cleanup was detected, terminated, and port/temp cleanup reverified. | +| Rollback boundary | Revert Unit 8 deltas in `src/{app,config/env}.ts`, catalog/ingest/client modules, `src/modules/ocr/dispatcher.ts`, and its test; preserve Units 1–7. | + +## Unit 9: Review and Indexing Core +- Added authenticated review/approval routes with injectable service boundaries, immutable correction preparation, stale/duplicate conflict guards, and new/reusable activation CAS behavior. Rejection transition and playground UI remain Unit 10. + +### TDD Cycle Evidence +| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR | +|---|---|---|---|---|---|---|---| +| 5.1 | `tests/ocr/review.test.ts` | HTTP/unit | App suites 9/9 and 20/20 | Missing `indexing.js`; exit 1 | 5/5 | Unauthorized/premature, candidate/line/active staleness, duplicate targets, new/reusable races, rejected no-index | Assertions remained 5/5 | +| 5.2 | `tests/ocr/review.test.ts` | HTTP/unit | 29/29 | Tests preceded `review.ts` | 5/5 | Audit view plus two corrections and four conflict paths | Clone prevents failed commits mutating loaded candidates; 5/5 | +| 5.3 | `tests/ocr/review.test.ts` | Unit | N/A (new file) | Tests preceded `indexing.ts` | 5/5 | New/reusable true/false, both CAS races, rejected no-index | Named CAS error mapping; 5/5 | + +### Work Unit Evidence +| Evidence | Result | +|---|---| +| Focused test | `NODE_ENV=test npx --no-install tsx --test tests/ocr/review.test.ts` — exit 0; 5 passed, 0 failed. | +| Runtime harness | Same runner with `--test-name-pattern "review HTTP rejects"` — exit 0; 1 passed; real localhost Express requests returned authenticated boundary outcomes without external services. | +| Rollback boundary | Remove `src/modules/ocr/{review,indexing}.ts` and `tests/ocr/review.test.ts`; revert only the Unit 9 route additions in `src/app.ts` and Unit 9 metadata. Preserve Units 1–8. | + +### Validation and Boundary +- Canonical `NODE_ENV=test npm test` passed 65/65; `npm run check`, `npm run build`, tracked whitespace, and no-index whitespace checks passed. +- Unit 9 adds 341 functional changed lines and 38 metadata changed lines, 379 total, within the 400-line budget. No live PostgreSQL, OpenAI, Qdrant, or OCR process was used; no test process remained. +- Start: Unit 8 dispatch/status is complete. End: review/correction and indexing/CAS ports plus authenticated routes are green. Unit 10 task 5.4 remains unchecked. + +## Unit 10: Rejection and Review UI +- Added authenticated rejection with current candidate-hash and state guards, required audit fields, durable store completion before success, and no indexing or activation call. +- Extended the existing static playground with authenticated candidate loading, protected page images, native/raw/candidate comparisons, line confidence/boxes, corrections, risks, approval, and rejection controls. + +### TDD Cycle Evidence +| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR | +|---|---|---|---|---|---|---|---| +| 5.4 | `tests/ocr/review.test.ts` | HTTP integration/static HTTP | Existing review suite passed 5/5 before edits. | Genuine RED passed only 5/7: rejection returned 404 instead of authenticated 401 and the playground lacked the review contract. A malformed-body edge then returned 409 instead of 400. | Focused suite passed 7/7 after minimal rejection and UI implementation. | Unauthorized, malformed, stale-hash, successful, repeated-state, durable-status, and static audit-field paths exercise distinct outcomes. | Shared UI decision-state helper extracted; focused suite remained 7/7. | + +### Work Unit Evidence +| Evidence | Result | +|---|---| +| Focused test command and exact result | `NODE_ENV=test npx --no-install tsx --test tests/ocr/review.test.ts` — exit 0; 7 passed, 0 failed. | +| Runtime harness command/scenario and exact result | `NODE_ENV=test npx --no-install tsx --test --test-name-pattern "authenticated rejection" tests/ocr/review.test.ts` — exit 0; 1 passed, 0 failed. Real localhost Express requests returned 401/400/409/200, persisted the rejected audit state through the injected store, retrieved rejected status, and made zero indexing calls. | +| Rollback boundary | Revert Unit 10 deltas in `src/app.ts`, `src/modules/ocr/review.ts`, `tests/ocr/review.test.ts`, and `public/playground/{index.html,app.js,styles.css}` plus Unit 10 metadata; preserve Units 1–9. | + +### Validation and Boundary +- Canonical `NODE_ENV=test npm test` passed 67/67; `npm run check` and `npm run build` exited 0. +- Previously observed tracked `git diff --check` exited 0; no-index whitespace checks for Unit 10's untracked `src/modules/ocr/review.ts` and `tests/ocr/review.test.ts` exited 0. +- Unit 10 adds 271 functional and 42 metadata changed lines, 313 total, below the 400-line budget. No live database, embedding, vector, OCR, or external service was used; no persistent process was started. +- Start: Unit 9 review/indexing ports and approval routes are green. End: rejection isolation and the static review UI are green. Retention, deploy, OpenAPI, and E2E tasks remain untouched. +- No specification or design deviation. + +## Unit 11: Retention and Flag-Off Isolation + +### Implementation Summary +- Added a retention boundary that expires 30-day reviews before deletion, retains failed/rejected artifacts for 7 days, and retains superseded OCR artifacts for 30 days. +- Added exact state/artifact CAS claims, active-pointer checks, activation exclusion after a deletion claim, and restart-safe `retention_deleting→retention_deleted` completion. +- Wired retention into the existing exclusive reconciler cycle and made status/review/decision APIs return `404` without store access while OCR is disabled; native synchronous ingestion remains unchanged. + +### TDD Cycle Evidence +| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR | +|---|---|---|---|---|---|---|---| +| 6.1 | `tests/ocr/retention.test.ts` | Repository/HTTP/filesystem integration | Existing repository, dispatcher, and review suites passed 22/22. | Genuine missing-module RED failed before `retention.ts`; activation-race RED then failed 0/1 with “Missing expected rejection.” | Focused suite passed 6/6. | Exact TTL SQL, active pointer/state guards, flag-off `201/404`, and deletion-claim activation conflict exercise independent paths. | No production refactor was needed; two test-side expectations were corrected during GREEN without weakening behavior. | +| 6.2 | `tests/ocr/retention.test.ts` | Filesystem/reconciler integration | Same 22/22 safety net. | The missing-module RED preceded all production retention code. | Focused suite passed 6/6; relevant regression set passed 28/28. | Review expiry, terminal deletion, active preservation, simulated restart, repeat no-op, and exclusive reconciler execution cover distinct paths. | Named store boundary keeps filesystem behavior isolated; final focused suite remained 6/6. | + +### Work Unit Evidence +| Evidence | Result | +|---|---| +| Focused test command and exact result | `NODE_ENV=test npx --no-install tsx --test tests/ocr/retention.test.ts` — exit 0; 6 passed, 0 failed. | +| Runtime harness command/scenario and exact result | The focused suite used real temporary directories and localhost Express requests: interrupted deletion removed only the claimed directory, resumed to `retention_deleted`, repeated as a no-op, preserved the active directory, returned native `201`, and returned `404` for all disabled OCR candidate APIs without store access. | +| Rollback boundary | Remove `src/modules/ocr/retention.ts` and `tests/ocr/retention.test.ts`; revert only Unit 11 seams in `src/app.ts`, catalog repository/reconciler, and explicit enabled-state setup in dispatcher/review tests. Preserve Units 1–10. | + +### Validation and Boundary +- Relevant regression suites passed 28/28 after explicitly enabling OCR in pre-existing enabled-path tests; the initial 19/22 run exposed only that test-fixture assumption. +- Canonical `NODE_ENV=test npm test` passed 73/73; `npm run check`, `npm run build`, tracked whitespace, and no-index whitespace checks for all seven intended untracked files passed. +- Temporary `rag-retention-*` directories were removed and no `tsx --test` or Node test process remained. No live database, OCR, embedding, vector, or external service was used; `ocr-service/.venv` was preserved. +- Unit 11 adds 277 functional changed lines and 50 metadata changed lines, 327 total, below the 400-line budget. +- Start: Unit 10 rejection/UI is complete. End: tasks 6.1–6.2 are complete. OpenAPI, deployment/configuration, E2E, production, commit, push, and PR work remain untouched. +- No specification or design deviation. + +## Unit 12: Contracts and Deployment Wiring + +### Implementation Summary +- Documented OCR `202` ingestion variants and authenticated status, review, correction/approval, and rejection routes with behavior-matched success and error responses. +- Added progress, review evidence, optimistic correction, decision, and structured error schemas while preserving the legacy-disabled `202` variants. +- Added private durable artifact defaults and OCR limits/timeouts; the root image now declares `/data/ingestions`, runs as `node`, and applies catalog migrations before startup without embedding secrets. + +### TDD Cycle Evidence +| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR | +|---|---|---|---|---|---|---|---| +| 6.3 | `tests/ocr/contracts-deploy.test.ts` | OpenAPI contract | Existing lifecycle/dispatcher/review/retention suites passed 42/42. | Genuine initial RED was 0/3: OCR `202` referenced the native schema and OCR routes/schemas were absent. | Focused suite passed 3/3. | A second RED required legacy-disabled `202` variants and accurate help authentication; final suite remained 3/3. | Shared schema references kept nested contracts reviewable; no behavior changed. | +| 6.4 | `tests/ocr/contracts-deploy.test.ts` | Config/static container | Same 42/42 safety net. | Initial RED found the old relative artifact root, absent limits/timeouts, volume, non-root user, and migration-aware command. | Focused suite passed 3/3; sanitized Docker build and image inspection passed. | A second RED required explicit runtime artifact-root wiring; final suite remained 3/3. | No further refactor needed. | + +### Work Unit Evidence +| Evidence | Result | +|---|---| +| Focused test command and exact result | `NODE_ENV=test npx --no-install tsx --test tests/ocr/contracts-deploy.test.ts` — exit 0; 3 passed, 0 failed. | +| Relevant regression command and exact result | Lifecycle, dispatcher, review, retention, and contract suites — exit 0; 45 passed, 0 failed. | +| Runtime harness | N/A — Unit 12 is static-only by task forecast. A sanitized-context Docker build succeeded and inspection proved user `node`, durable volume, production artifact root, and migration-before-server command. | +| Rollback boundary | Remove `tests/ocr/contracts-deploy.test.ts` and revert only Unit 12 deltas in `src/api/openapi.ts`, `src/config/env.ts`, and `Dockerfile`; preserve Units 1–11. | + +### Validation and Boundary +- Canonical `NODE_ENV=test npm test` passed 76/76; `npm run check`, `npm run build`, tracked whitespace, and all intended-untracked whitespace checks passed. +- Docker build `rag-service:unit12-contract` succeeded from a sanitized 430.31 kB context; inspection matched the deployment contract. The image and temporary context were removed, no test process remained, and `ocr-service/.venv` was preserved. +- Unit 12 adds 219 functional lines and 50 metadata lines, 269 total, below the 400-line budget. +- Start: Unit 11 retention is complete. End: tasks 6.3–6.4 are complete. Unit 13 / all 7.x E2E and production work remains untouched. +- `docs/CONTRATO_CICLO_VIDA_Y_OCR.md` was read as authoritative and remained unchanged. No specification or design deviation. + +## Unit 13: Local Deterministic E2E and Complete Gate + +### Implementation Summary +- Added a bounded localhost HTTP suite covering native `201`, scanned `202 → review → approve → active`, and exact mixed native/OCR page reporting. +- Added resend identity reuse, fail-closed OCR exhaustion, and catalog-unavailable `503` coverage without external services. +- Made no production-code changes; the new scenarios passed immediately as baseline/characterization evidence, so no RED was fabricated. + +### TDD Cycle Evidence +| Task | Test File | Layer | Safety Net | RED / Baseline | GREEN | TRIANGULATE | REFACTOR | +|---|---|---|---|---|---|---|---| +| 7.1 | `tests/ocr/e2e.test.ts` | Local HTTP E2E | Existing OCR suites passed 28/28. | Baseline/characterization: the test was written first and passed 2/2 immediately; no implementation gap existed at the local fake boundary. | Focused suite passed 2/2. | Native, scanned, corrected approval, post-activation status, and mixed page methods exercise distinct paths. | Test-only state naming improved; focused suite remained 2/2. | +| 7.2 | `tests/ocr/e2e.test.ts` | Local HTTP/dispatcher E2E | Same 28/28 safety net. | Baseline/characterization: resend, OCR-down, and catalog-down assertions passed immediately; no production code was written. | Focused suite passed 2/2. | Duplicate resend, terminal dispatch failure, status `503`, and retrieval `503` cover independent outcomes. | None needed beyond the shared test harness. | +| 7.3 | Gate commands below | Complete gate | Focused E2E passed 2/2. | N/A — verification-only task with no production behavior to implement. | All required gates passed. | Node type/build/test and offline Python tests provide independent stack evidence. | N/A — no production change. | + +### Work Unit Evidence +| Evidence | Result | +|---|---| +| Focused test | `NODE_ENV=test npx --no-install tsx --test tests/ocr/e2e.test.ts` — exit 0; 2 passed, 0 failed. | +| Runtime harness | The focused suite started bounded ephemeral localhost Express servers and exercised real HTTP requests through ingest, status, review, approval, and retrieval routes; all servers closed through test cleanup. | +| Complete gate | `npm run check`, `npm run build`, and `NODE_ENV=test npm test` exited 0; canonical Node passed 78/78. `PYTHONDONTWRITEBYTECODE=1 ocr-service/.venv/bin/python -m pytest ocr-service/tests -q` passed 13/13 offline with one dependency deprecation warning. | +| Rollback boundary | Remove `tests/ocr/e2e.test.ts` and revert only tasks 7.1–7.3 plus this Unit 13 progress/history metadata; preserve Units 1–12. | + +### Validation and Boundary +- Unit 13 adds 183 functional lines. Required tasks/progress/history metadata adds 52 changed lines, for 235 total, below the 400-line budget. +- Tracked and complete intended-untracked whitespace checks passed. No matching `tsx --test`, E2E Node, or OCR Uvicorn process remained after the corrected self-excluding process check. +- Intended untracked inventory: `src/modules/ocr/{dispatcher,indexing,retention,review}.ts`; `tests/ocr/{contracts-deploy,dispatcher,e2e,retention,review}.test.ts`. +- Tasks 7.1–7.3 are complete. Task 7.4 remains unchecked and untouched; no production flag, service, database, vector store, embedding provider, OCR endpoint, commit, push, or PR was used. diff --git a/openspec/changes/ocr-ingest-integration/tasks.md b/openspec/changes/ocr-ingest-integration/tasks.md index c7fb89d..f1a2351 100644 --- a/openspec/changes/ocr-ingest-integration/tasks.md +++ b/openspec/changes/ocr-ingest-integration/tasks.md @@ -51,29 +51,29 @@ Tracker feature/ocr-ingest-integration is draft/no-merge and sole main target. P ## 4 Orchestration (Units 7–8; O1–O4) -- [ ] 4.1 RED HTTP: flag-off 201; OCR 202/status; OCR-down fail-closed; catalog-down 503; retries/integrity +- [x] 4.1 RED HTTP: flag-off 201; OCR 202/status; OCR-down fail-closed; catalog-down 503; retries/integrity - [x] 4.2 GREEN: `src/modules/ocr/client.ts` retries and integrity - [x] 4.3 GREEN: `src/modules/ocr/artifacts.ts` originals, manifest, sweep -- [ ] 4.4 GREEN: `src/modules/ocr/dispatcher.ts` leases; `src/modules/catalog/reconciler.ts` -- [ ] 4.5 GREEN: `src/modules/ingest/service.ts` branch; `src/app.ts` upload/status +- [x] 4.4 GREEN: `src/modules/ocr/dispatcher.ts` leases; `src/modules/catalog/reconciler.ts` +- [x] 4.5 GREEN: `src/modules/ingest/service.ts` branch; `src/app.ts` upload/status ## 5 Review (Units 9–10; V1–V4) -- [ ] 5.1 RED: auth/premature access; atomic corrections/stale 409; new/reusable activation race; reject/no embeddings -- [ ] 5.2 GREEN: `src/modules/ocr/review.ts` view, corrections, conflicts -- [ ] 5.3 GREEN: `src/modules/ocr/indexing.ts` CAS; new/reusable activate_requested=true activates, false stays ready -- [ ] 5.4 GREEN: rejection and `public/playground/` review UI +- [x] 5.1 RED: auth/premature access; atomic corrections/stale 409; new/reusable activation race; reject/no embeddings +- [x] 5.2 GREEN: `src/modules/ocr/review.ts` view, corrections, conflicts +- [x] 5.3 GREEN: `src/modules/ocr/indexing.ts` CAS; new/reusable activate_requested=true activates, false stays ready +- [x] 5.4 GREEN: rejection and `public/playground/` review UI ## 6 Retention/Deploy (Units 11–12; O5/V5) -- [ ] 6.1 RED: TTL/CAS/resume/active protection; flag-off native-only and candidate invisibility -- [ ] 6.2 GREEN: `src/modules/ocr/retention.ts` -- [ ] 6.3 GREEN: `src/api/openapi.ts` OCR/status/review contracts -- [ ] 6.4 GREEN: `src/config/env.ts`, `Dockerfile`; follow `docs/CONTRATO_CICLO_VIDA_Y_OCR.md` (read-only) +- [x] 6.1 RED: TTL/CAS/resume/active protection; flag-off native-only and candidate invisibility +- [x] 6.2 GREEN: `src/modules/ocr/retention.ts` +- [x] 6.3 GREEN: `src/api/openapi.ts` OCR/status/review contracts +- [x] 6.4 GREEN: `src/config/env.ts`, `Dockerfile`; follow `docs/CONTRATO_CICLO_VIDA_Y_OCR.md` (read-only) ## 7 E2E (Unit 13; V6) -- [ ] 7.1 E2E: native 201; scanned 202→review→approve→active; mixed -- [ ] 7.2 E2E: resend, OCR-down fail-closed, catalog-down 503 -- [ ] 7.3 Gate: check, build, test, offline pytest +- [x] 7.1 E2E: native 201; scanned 202→review→approve→active; mixed +- [x] 7.2 E2E: resend, OCR-down fail-closed, catalog-down 503 +- [x] 7.3 Gate: check, build, test, offline pytest - [ ] 7.4 Production: flag-off verify→enable→FacturaTech 34 entries; CBG04a/FAT07/DSAU08/NSAV06 diff --git a/public/playground/app.js b/public/playground/app.js index 188dc14..7316de6 100644 --- a/public/playground/app.js +++ b/public/playground/app.js @@ -10,6 +10,9 @@ const manualLogButton = document.getElementById("manualLogButton"); const presetDocs = document.getElementById("presetDocs"); const presetRagDocs = document.getElementById("presetRagDocs"); const presetCode = document.getElementById("presetCode"); +const loadReviewButton = document.getElementById("loadReviewButton"); +const approveReviewButton = document.getElementById("approveReviewButton"); +const rejectReviewButton = document.getElementById("rejectReviewButton"); const healthResult = document.getElementById("healthResult"); const ingestResult = document.getElementById("ingestResult"); @@ -24,6 +27,12 @@ const contextStatusText = document.getElementById("contextStatusText"); const contextScopeText = document.getElementById("contextScopeText"); const logsResult = document.getElementById("logsResult"); const logCounterValue = document.getElementById("logCounterValue"); +const reviewCandidate = document.getElementById("reviewCandidate"); +const reviewResult = document.getElementById("reviewResult"); +const reviewVersionId = document.getElementById("reviewVersionId"); +const reviewToken = document.getElementById("reviewToken"); +const reviewedBy = document.getElementById("reviewedBy"); +const rejectionReason = document.getElementById("rejectionReason"); const ingestSourceType = document.getElementById("ingestSourceType"); const ingestScopeMode = document.getElementById("ingestScopeMode"); @@ -69,6 +78,8 @@ let lastBootstrapMeta = null; let chatHistory = []; let availableScopes = []; let lastInteraction = null; +let loadedReview = null; +let reviewImageUrls = []; let currentUploadType = null; // 'file' o 'folder' @@ -179,6 +190,72 @@ function request(url, payload, method = "POST") { }); } +async function authorizedReviewRequest(path, payload) { + const response = await fetch(`/ingestions/${encodeURIComponent(reviewVersionId.value.trim())}${path}`, { + method: payload ? "POST" : "GET", + headers: { Authorization: `Bearer ${reviewToken.value}`, ...(payload ? { "Content-Type": "application/json" } : {}) }, + body: payload ? JSON.stringify(payload) : undefined + }); + const data = await response.json(); + if (!response.ok) throw new Error(`${data.code || response.status}: ${data.error || "Review request failed"}`); + return data; +} + +function reviewText(name, value) { + const element = document.createElement("pre"); + element.textContent = `${name}\n${Array.isArray(value) ? value.join("\n") : value || "(empty)"}`; + return element; +} + +async function renderReview(candidate) { + for (const url of reviewImageUrls) URL.revokeObjectURL(url); + reviewImageUrls = []; + reviewCandidate.replaceChildren(); + const identity = document.createElement("p"); + identity.textContent = `Candidate SHA-256: ${candidate.candidateSha256} · Base active version: ${candidate.baseActiveVersionId || "none"}`; + reviewCandidate.append(identity); + for (const document of candidate.documents) for (const page of document.pages) { + const card = document.createElement("article"); + card.className = "review-page"; + const title = document.createElement("h3"); + title.textContent = `${document.documentId} · Page ${page.page}`; + const image = document.createElement("img"); + image.className = "review-image"; + image.alt = `Source page ${page.page}`; + card.append(title, image, reviewText("Native text", page.nativeText), reviewText("Raw OCR", page.ocr.text), reviewText("Candidate text", page.candidateText), reviewText("Differences", page.differences), reviewText("Risks", page.risks)); + for (const line of page.ocr.lines) { + const label = document.createElement("label"); + label.className = "review-line"; + const detail = document.createElement("span"); + detail.textContent = `${line.lineId} · confidence ${line.confidence} · bbox ${line.bbox.join(", ")}`; + const input = document.createElement("textarea"); + input.value = line.text; + Object.assign(input.dataset, { original: line.text, documentId: document.documentId, page: String(page.page), lineId: line.lineId, expectedLineSha256: line.lineSha256 }); + label.append(detail, input); + card.append(label); + } + reviewCandidate.append(card); + void fetch(page.imageUrl, { headers: { Authorization: `Bearer ${reviewToken.value}` } }).then(async (response) => { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const url = URL.createObjectURL(await response.blob()); + reviewImageUrls.push(url); + image.src = url; + }).catch((error) => { image.alt = `Protected image unavailable: ${error}`; }); + } +} + +function reviewDecisionBase() { + if (!loadedReview) throw new Error("Load a current candidate first"); + const reviewer = reviewedBy.value.trim(); + if (!reviewer) throw new Error("Reviewer identity is required"); + return { candidateSha256: loadedReview.candidateSha256, reviewedBy: reviewer }; +} + +function setReviewDecisionEnabled(enabled) { + approveReviewButton.disabled = !enabled; + rejectReviewButton.disabled = !enabled; +} + function renderBootstrapContext() { if (!lastBootstrapContext) { bootstrapContextResult.textContent = "Aun no hay bootstrap cargado."; @@ -468,6 +545,54 @@ ingestButton.addEventListener("click", async () => { } }); +loadReviewButton.addEventListener("click", async () => { + reviewResult.textContent = "Loading authenticated review..."; + try { + loadedReview = await authorizedReviewRequest("/review"); + await renderReview(loadedReview); + setReviewDecisionEnabled(true); + reviewResult.textContent = `Loaded review_required version ${loadedReview.versionId}.`; + } catch (error) { + loadedReview = null; + setReviewDecisionEnabled(false); + reviewCandidate.textContent = "No candidate loaded."; + reviewResult.textContent = String(error); + } +}); + +approveReviewButton.addEventListener("click", async () => { + reviewResult.textContent = "Submitting reviewed text..."; + try { + const corrections = [...reviewCandidate.querySelectorAll(".review-line textarea")] + .filter((input) => input.value !== input.dataset.original) + .map((input) => ({ + documentId: input.dataset.documentId, + page: Number(input.dataset.page), + lineId: input.dataset.lineId, + expectedLineSha256: input.dataset.expectedLineSha256, + replacementText: input.value + })); + const result = await authorizedReviewRequest("/approve", { ...reviewDecisionBase(), expectedActiveVersionId: loadedReview.baseActiveVersionId, corrections }); + reviewResult.textContent = format(result); + setReviewDecisionEnabled(false); + } catch (error) { + reviewResult.textContent = String(error); + } +}); + +rejectReviewButton.addEventListener("click", async () => { + reviewResult.textContent = "Rejecting candidate..."; + try { + const reason = rejectionReason.value.trim(); + if (!reason) throw new Error("Rejection reason is required"); + const result = await authorizedReviewRequest("/reject", { ...reviewDecisionBase(), reason }); + reviewResult.textContent = format(result); + setReviewDecisionEnabled(false); + } catch (error) { + reviewResult.textContent = String(error); + } +}); + cleanupButton.addEventListener("click", async () => { if (!cleanupScopeSelect.value) { cleanupResult.textContent = "Error: Debes seleccionar un scope primero."; diff --git a/public/playground/index.html b/public/playground/index.html index 628fbcd..976ca01 100644 --- a/public/playground/index.html +++ b/public/playground/index.html @@ -29,11 +29,40 @@
+
+
+
+

OCR Candidate Review

+

Inspect every page and submit an authenticated approval or rejection. OCR content remains unavailable to retrieval until approval completes.

+
+ + + + +
+
+ + + +
+
No candidate loaded.
+
No review request submitted.
+
+
+

Ingesta

diff --git a/public/playground/styles.css b/public/playground/styles.css index 37b35c2..44987e9 100644 --- a/public/playground/styles.css +++ b/public/playground/styles.css @@ -182,6 +182,23 @@ button.secondary { border: 1px solid var(--border); } +button.danger { background: var(--danger); color: #0b1020; } +button:disabled { cursor: not-allowed; opacity: 0.45; } + +.review-candidate { display: grid; gap: 16px; margin: 20px 0; } +.review-page { + display: grid; + gap: 14px; + padding: 18px; + border: 1px solid var(--border); + border-radius: 16px; + background: #0b1020; +} +.review-page pre { margin: 0; min-height: 0; } +.review-image { max-width: 100%; max-height: 520px; object-fit: contain; justify-self: start; } +.review-line { display: grid; gap: 6px; } +.review-line span { color: var(--muted); font-size: 12px; } + .actions, .checkbox { display: flex; gap: 12px; diff --git a/src/api/openapi.ts b/src/api/openapi.ts index 8824635..f809cd5 100644 --- a/src/api/openapi.ts +++ b/src/api/openapi.ts @@ -30,6 +30,7 @@ export const openApiDocument = { { name: "Discovery", description: "API contract and browser playground." }, { name: "Status", description: "Service capabilities and available resources." }, { name: "Ingestion", description: "Knowledge ingestion and cleanup." }, + { name: "OCR Review", description: "Authenticated OCR progress, review, correction, and decisions." }, { name: "Lifecycle", description: "Knowledge source versions, activation, rollback, and purge." }, { name: "Retrieval", description: "Context retrieval and model-backed answers." }, { name: "Evaluation", description: "Evaluation log capture and review." } @@ -222,7 +223,7 @@ export const openApiDocument = { }, responses: { "201": jsonResponse("Lifecycle ingestion completed and committed.", ref("IngestResponse")), - "202": jsonResponse("Legacy ingestion completed and accepted while lifecycle enforcement is disabled.", ref("IngestResponse")), + "202": jsonResponse("OCR ingestion accepted, or legacy ingestion accepted while lifecycle enforcement is disabled.", { oneOf: [ref("OcrAccepted"), ref("IngestResponse")] }), "400": jsonResponse("Invalid lifecycle activation request.", ref("Error")), "409": jsonResponse("Concurrent active version change or reusable version in progress.", ref("Error")), "422": jsonResponse("Empty or unsupported source.", ref("Error")), @@ -245,7 +246,7 @@ export const openApiDocument = { }, responses: { "201": jsonResponse("Lifecycle upload ingested successfully.", ref("UploadIngestResponse")), - "202": jsonResponse("Legacy upload ingested successfully while lifecycle enforcement is disabled.", ref("UploadIngestResponse")), + "202": jsonResponse("OCR upload accepted, or legacy upload accepted while lifecycle enforcement is disabled.", { oneOf: [ref("OcrUploadAccepted"), ref("UploadIngestResponse")] }), "400": jsonResponse("The file field is missing.", ref("Error"), { ok: false, error: "Missing file upload" }), "409": jsonResponse("Concurrent active version change or reusable version in progress.", ref("Error")), "422": jsonResponse("Empty or unsupported source.", ref("Error")), @@ -254,6 +255,60 @@ export const openApiDocument = { } } }, + "/ingestions/{versionId}": { + get: { + tags: ["OCR Review"], summary: "Get OCR ingestion progress", security: [{ bearerAuth: [] }], + parameters: [{ name: "versionId", in: "path", required: true, schema: { type: "string", format: "uuid" } }], + responses: { + "200": jsonResponse("Current document and page progress.", ref("IngestionStatus")), + "401": jsonResponse("Missing or invalid lifecycle admin token.", ref("Error")), + "404": jsonResponse("OCR is disabled or the ingestion does not exist.", ref("Error")), + "503": serverError, "500": serverError + } + } + }, + "/ingestions/{versionId}/review": { + get: { + tags: ["OCR Review"], summary: "Inspect an OCR candidate", security: [{ bearerAuth: [] }], + parameters: [{ name: "versionId", in: "path", required: true, schema: { type: "string", format: "uuid" } }], + responses: { + "200": jsonResponse("Review candidate with page evidence.", ref("OcrReview")), + "401": jsonResponse("Missing or invalid lifecycle admin token.", ref("Error")), + "404": jsonResponse("OCR is disabled or the candidate does not exist.", ref("Error")), + "409": jsonResponse("The version is not awaiting review.", ref("Error")), + "503": serverError, "500": serverError + } + } + }, + "/ingestions/{versionId}/approve": { + post: { + tags: ["OCR Review"], summary: "Correct and approve an OCR candidate", security: [{ bearerAuth: [] }], + parameters: [{ name: "versionId", in: "path", required: true, schema: { type: "string", format: "uuid" } }], + requestBody: { required: true, content: jsonContent(ref("OcrApprovalRequest")) }, + responses: { + "200": jsonResponse("Candidate indexed and settled according to activation intent.", ref("OcrDecisionResponse")), + "401": jsonResponse("Missing or invalid lifecycle admin token.", ref("Error")), + "404": jsonResponse("OCR is disabled or the candidate does not exist.", ref("Error")), + "409": jsonResponse("Candidate, correction, state, or active-version precondition conflict.", ref("Error")), + "503": serverError, "500": serverError + } + } + }, + "/ingestions/{versionId}/reject": { + post: { + tags: ["OCR Review"], summary: "Reject an OCR candidate", security: [{ bearerAuth: [] }], + parameters: [{ name: "versionId", in: "path", required: true, schema: { type: "string", format: "uuid" } }], + requestBody: { required: true, content: jsonContent(ref("OcrRejectionRequest")) }, + responses: { + "200": jsonResponse("Candidate rejected without indexing or activation.", ref("OcrDecisionResponse")), + "400": jsonResponse("Required rejection fields are missing.", ref("Error")), + "401": jsonResponse("Missing or invalid lifecycle admin token.", ref("Error")), + "404": jsonResponse("OCR is disabled or the candidate does not exist.", ref("Error")), + "409": jsonResponse("Candidate hash or review state conflict.", ref("Error")), + "503": serverError, "500": serverError + } + } + }, "/cleanup": { post: { tags: ["Ingestion"], @@ -402,7 +457,8 @@ export const openApiDocument = { required: ["ok", "error"], properties: { ok: { type: "boolean", const: false }, - error: { type: "string" } + error: { type: "string" }, + code: { type: "string" } } }, Scope: { @@ -490,6 +546,77 @@ export const openApiDocument = { } ] }, + OcrPhase: { + type: "string", + enum: ["native_extracting", "ocr_queued", "ocr_running", "review_required", "indexing", "ready", "active", "failed", "rejected"] + }, + OcrAccepted: { + type: "object", + required: ["accepted", "sourceId", "versionId", "versionNumber", "state", "phase", "statusUrl", "reviewUrl", "activated"], + properties: { + accepted: { type: "boolean", const: true }, sourceId: { type: "string" }, versionId: { type: "string", format: "uuid" }, + versionNumber: { type: "integer" }, state: { type: "string", const: "indexing" }, phase: { type: "string", const: "ocr_queued" }, + statusUrl: { type: "string" }, reviewUrl: { type: "null" }, activated: { type: "boolean", const: false } + } + }, + OcrUploadAccepted: { + allOf: [ref("OcrAccepted"), { type: "object", required: ["uploadedResource"], properties: { uploadedResource: { type: "string" } } }] + }, + IngestionStatus: { + type: "object", + required: ["sourceId", "versionId", "state", "phase", "activated", "documents", "error", "statusUrl", "reviewUrl"], + properties: { + sourceId: { type: "string" }, versionId: { type: "string", format: "uuid" }, state: ref("SourceVersionState"), phase: ref("OcrPhase"), activated: { type: "boolean" }, + documents: { type: "array", items: ref("OcrProgressDocument") }, + error: { oneOf: [ref("OcrStatusError"), { type: "null" }] }, statusUrl: { type: "string" }, reviewUrl: { type: ["string", "null"] } + } + }, + OcrProgressDocument: { + type: "object", required: ["documentId", "state", "completedPages", "totalPages", "pages"], + properties: { + documentId: { type: "string" }, state: { type: "string", enum: ["native_complete", "ocr_queued", "ocr_running", "ocr_complete", "failed"] }, + completedPages: { type: "integer" }, totalPages: { type: "integer" }, pages: { type: "array", items: ref("OcrProgressPage") } + } + }, + OcrProgressPage: { + type: "object", required: ["page", "method", "state"], + properties: { page: { type: "integer" }, method: { type: "string", enum: ["native", "ocr", "blank"] }, state: { type: "string", enum: ["pending", "native_complete", "ocr_queued", "ocr_running", "ocr_complete", "blank", "failed"] }, errorCode: { type: "string" } } + }, + OcrStatusError: { type: "object", required: ["code", "message", "retryable"], properties: { code: { type: "string" }, message: { type: "string" }, retryable: { type: "boolean" } } }, + OcrLine: { + type: "object", required: ["lineId", "text", "confidence", "bbox", "lineSha256"], + properties: { lineId: { type: "string" }, text: { type: "string" }, confidence: { type: "number" }, bbox: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4 }, lineSha256: { type: "string", pattern: "^[a-f0-9]{64}$" } } + }, + OcrReview: { + type: "object", + required: ["versionId", "sourceId", "state", "candidateSha256", "baseActiveVersionId", "currentActiveVersionId", "activateRequested", "processingFingerprint", "metadataHash", "documents"], + properties: { + versionId: { type: "string", format: "uuid" }, sourceId: { type: "string" }, state: { type: "string", const: "review_required" }, candidateSha256: { type: "string", pattern: "^[a-f0-9]{64}$" }, + baseActiveVersionId: { type: ["string", "null"], format: "uuid" }, currentActiveVersionId: { type: ["string", "null"], format: "uuid" }, activateRequested: { type: "boolean" }, processingFingerprint: { type: "string" }, metadataHash: { type: "string" }, + documents: { type: "array", items: ref("OcrReviewDocument") } + } + }, + OcrReviewDocument: { type: "object", required: ["documentId", "pages"], properties: { documentId: { type: "string" }, pages: { type: "array", items: ref("OcrReviewPage") } } }, + OcrReviewPage: { + type: "object", required: ["page", "imageUrl", "nativeText", "ocr", "candidateText", "differences", "risks"], + properties: { page: { type: "integer" }, imageUrl: { type: "string" }, nativeText: { type: "string" }, ocr: { type: "object", required: ["text", "lines"], properties: { text: { type: "string" }, lines: { type: "array", items: ref("OcrLine") } } }, candidateText: { type: "string" }, differences: { type: "array", items: { type: "string" } }, risks: { type: "array", items: { type: "string" } } } + }, + OcrCorrection: { + type: "object", required: ["documentId", "page", "lineId", "expectedLineSha256", "replacementText"], + properties: { documentId: { type: "string" }, page: { type: "integer" }, lineId: { type: "string" }, expectedLineSha256: { type: "string", pattern: "^[a-f0-9]{64}$" }, replacementText: { type: "string" } } + }, + OcrApprovalRequest: { + type: "object", required: ["candidateSha256", "expectedActiveVersionId", "reviewedBy", "corrections"], + properties: { candidateSha256: { type: "string", pattern: "^[a-f0-9]{64}$" }, expectedActiveVersionId: { type: ["string", "null"], format: "uuid" }, reviewedBy: { type: "string" }, corrections: { type: "array", items: ref("OcrCorrection") } } + }, + OcrRejectionRequest: { + type: "object", required: ["candidateSha256", "reviewedBy", "reason"], + properties: { candidateSha256: { type: "string", pattern: "^[a-f0-9]{64}$" }, reviewedBy: { type: "string" }, reason: { type: "string" } } + }, + OcrDecisionResponse: { + type: "object", required: ["versionId", "state", "activated"], + properties: { versionId: { type: "string", format: "uuid" }, state: { type: "string", enum: ["ready", "active", "rejected"] }, activated: { type: "boolean" }, activatedVersionId: { type: "string", format: "uuid" }, errorCode: { type: "string", enum: ["DUPLICATE_REUSABLE_VERSION"] } } + }, CleanupRequest: { type: "object", required: ["scope"], @@ -877,7 +1004,7 @@ export function buildApiHelp() { openapiPurpose: "Consult this contract for complete parameters, request bodies, responses, errors, and examples for every endpoint.", playground: "/playground" }, - authentication: "None. The current API is publicly accessible; authentication is planned separately.", + authentication: "Bearer authentication is required for lifecycle administration and all OCR candidate routes; other routes remain public.", endpoints }; } diff --git a/src/app.ts b/src/app.ts index aa2ca1e..bf71d9b 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,10 +1,10 @@ import express from "express"; import multer from "multer"; import type { Request } from "express"; -import { writeFile, unlink, mkdtemp, rm } from "node:fs/promises"; +import { readFile, unlink, mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { timingSafeEqual } from "node:crypto"; +import { randomUUID, timingSafeEqual } from "node:crypto"; import { fileURLToPath } from "node:url"; import AdmZip from "adm-zip"; import { buildApiHelp, openApiDocument } from "./api/openapi.js"; @@ -20,28 +20,66 @@ import { QdrantVectorStoreClient } from "./modules/vectorstore/client.js"; import { getCatalogPool } from "./modules/catalog/client.js"; import { KnowledgeLifecycleReconciler } from "./modules/catalog/reconciler.js"; import { CatalogError, CatalogRepository } from "./modules/catalog/repository.js"; +import { OcrClient } from "./modules/ocr/client.js"; +import { OcrDispatcher } from "./modules/ocr/dispatcher.js"; +import { resolveArtifactPath } from "./modules/ocr/artifacts.js"; +import type { OcrReviewService } from "./modules/ocr/review.js"; +import type { OcrIndexingService } from "./modules/ocr/indexing.js"; +import { OcrRetentionService } from "./modules/ocr/retention.js"; import type { ChatMessage, ChunkMode, RetrieveIntent, RetrieveScope } from "./shared/types/rag.js"; +import { sha256Hex } from "./shared/utils/ids.js"; type UploadRequest = Request & { file?: Express.Multer.File; }; -export function createApp() { +interface AppOptions { + ingestService?: Pick; + catalog?: CatalogRepository; + ocrClient?: OcrClient; + reviewService?: Pick; + indexingService?: Pick; + startReconciler?: boolean; +} + +export function createApp(options: AppOptions = {}) { const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const publicDir = path.resolve(__dirname, "../public"); const app = express(); - const upload = multer({ storage: multer.memoryStorage() }); + const upload = multer({ + storage: multer.diskStorage({ + destination: os.tmpdir(), + filename: (_request, file, callback) => callback(null, `${randomUUID()}${path.extname(file.originalname).toLowerCase()}`) + }), + limits: { fileSize: 50 * 1024 * 1024 } + }); const embeddingProvider = new OpenRouterEmbeddingProvider(); const vectorStore = new QdrantVectorStoreClient(); const catalogPool = getCatalogPool(); - const catalog = catalogPool ? new CatalogRepository(catalogPool) : undefined; - const reconciler = new KnowledgeLifecycleReconciler(catalog, vectorStore); + const catalog = Object.prototype.hasOwnProperty.call(options, "catalog") + ? options.catalog + : catalogPool ? new CatalogRepository(catalogPool) : undefined; + const ocr = { enabled: env.ocrIngestEnabled, artifactRoot: path.resolve(env.ocrArtifactRoot) }; + const ocrClient = ocr.enabled ? options.ocrClient ?? new OcrClient({ baseUrl: env.ocrServiceUrl, token: env.ocrInternalToken }) : undefined; + const ocrDispatcher = catalog && ocrClient ? new OcrDispatcher(catalog, ocrClient, async (job) => { + const versionDirectory = path.join(ocr.artifactRoot, job.versionId); + const manifest = JSON.parse(await readFile(path.join(versionDirectory, "manifest.json"), "utf8")) as { + documents?: Array<{ documentId: string; originalPath: string; originalSha256: string }>; + }; + const document = manifest.documents?.find(({ documentId }) => documentId === job.documentId); + if (!document) throw new Error("OCR artifact manifest does not contain the queued document"); + const bytes = await readFile(resolveArtifactPath(versionDirectory, document.originalPath)); + if (sha256Hex(bytes) !== document.originalSha256) throw new Error("OCR artifact integrity validation failed"); + return { bytes, documentSha256: document.originalSha256 }; + }) : undefined; + const retention = catalog ? new OcrRetentionService(catalog, ocr.artifactRoot) : undefined; + const reconciler = new KnowledgeLifecycleReconciler(catalog, vectorStore, ocrDispatcher, retention); const evaluationLogs = new EvaluationLogService(embeddingProvider); - const ingestService = new IngestService(embeddingProvider, vectorStore, catalog); + const ingestService = options.ingestService ?? new IngestService(embeddingProvider, vectorStore, catalog, ocr); const retrieveService = new RetrieveService(embeddingProvider, vectorStore, catalog); const answerService = new AnswerService(retrieveService); - reconciler.start(); + if (options.startReconciler !== false) reconciler.start(); function sendError(res: express.Response, error: unknown, fallback: string) { const upstreamUnavailable = error instanceof Error && /(ECONNREFUSED|ETIMEDOUT|timeout|connection|database|postgres|qdrant)/i.test(error.message); @@ -54,6 +92,10 @@ export function createApp() { res.status(statusCode).json(body); } + function dispatchAcceptedOcr(result: Awaited>): void { + if ("phase" in result) void ocrDispatcher?.dispatchAvailable(); + } + function requireLifecycleAdmin(req: express.Request, res: express.Response): boolean { const header = req.header("authorization") ?? ""; const token = header.startsWith("Bearer ") ? header.slice("Bearer ".length) : ""; @@ -71,6 +113,12 @@ export function createApp() { return true; } + function requireOcrEnabled(res: express.Response): boolean { + if (ocr.enabled) return true; + res.status(404).json({ ok: false, error: "Not found" }); + return false; + } + function requireCatalog(res: express.Response): CatalogRepository | undefined { if (!catalog) { res.status(503).json({ ok: false, error: "Knowledge catalog is not configured", code: "CATALOG_UNAVAILABLE" }); @@ -201,6 +249,66 @@ export function createApp() { } }); + app.get("/ingestions/:versionId", async (req, res) => { + if (!requireOcrEnabled(res)) return; + if (!requireLifecycleAdmin(req, res)) return; + const repository = requireCatalog(res); + if (!repository) return; + try { + const status = await repository.getIngestionStatus(String(req.params.versionId)); + if (!status) { + res.status(404).json({ ok: false, error: "Ingestion not found", code: "INGESTION_NOT_FOUND" }); + return; + } + res.json(status); + } catch (error) { + sendError(res, error, "Unknown ingestion status error"); + } + }); + + app.get("/ingestions/:versionId/review", async (req, res) => { + if (!requireOcrEnabled(res)) return; + if (!requireLifecycleAdmin(req, res)) return; + if (!options.reviewService) { + res.status(503).json({ ok: false, error: "OCR review service is not configured", code: "OCR_REVIEW_UNAVAILABLE" }); + return; + } + try { + res.json(await options.reviewService.view(String(req.params.versionId))); + } catch (error) { + sendError(res, error, "Unknown OCR review error"); + } + }); + + app.post("/ingestions/:versionId/approve", async (req, res) => { + if (!requireOcrEnabled(res)) return; + if (!requireLifecycleAdmin(req, res)) return; + if (!options.reviewService || !options.indexingService) { + res.status(503).json({ ok: false, error: "OCR approval service is not configured", code: "OCR_REVIEW_UNAVAILABLE" }); + return; + } + try { + const approved = await options.reviewService.approve(String(req.params.versionId), req.body); + res.json(await options.indexingService.index(approved)); + } catch (error) { + sendError(res, error, "Unknown OCR approval error"); + } + }); + + app.post("/ingestions/:versionId/reject", async (req, res) => { + if (!requireOcrEnabled(res)) return; + if (!requireLifecycleAdmin(req, res)) return; + if (!options.reviewService) { + res.status(503).json({ ok: false, error: "OCR review service is not configured", code: "OCR_REVIEW_UNAVAILABLE" }); + return; + } + try { + res.json(await options.reviewService.reject(String(req.params.versionId), req.body)); + } catch (error) { + sendError(res, error, "Unknown OCR rejection error"); + } + }); + app.get("/models/answer", async (_req, res) => { try { const models = await answerService.listAvailableAnswerModels(); @@ -226,7 +334,8 @@ export function createApp() { app.post("/ingest", async (req, res) => { try { const result = await ingestService.ingest(req.body); - res.status(env.knowledgeLifecycleEnforced ? 201 : 202).json(result); + dispatchAcceptedOcr(result); + res.status("phase" in result ? 202 : env.knowledgeLifecycleEnforced ? 201 : 202).json(result); } catch (error) { sendError(res, error, "Unknown ingest error"); } @@ -322,7 +431,8 @@ export function createApp() { activate: req.body.activate === undefined ? true : Boolean(req.body.activate), expectedActiveVersionId: req.body.expectedActiveVersionId === undefined ? null : req.body.expectedActiveVersionId }); - res.status(201).json(result); + dispatchAcceptedOcr(result); + res.status("phase" in result ? 202 : 201).json(result); return; } res.status(202).json({ @@ -371,8 +481,7 @@ export function createApp() { const isZipFolder = req.body.isZipFolder === "true"; const tempDirBase = await os.tmpdir(); - tempFilePath = path.join(tempDirBase, `${Date.now()}-${req.file.originalname}`); - await writeFile(tempFilePath, req.file.buffer); + tempFilePath = req.file.path; const tags = typeof req.body.tags === "string" ? req.body.tags.split(",").map((entry: string) => entry.trim()).filter(Boolean) @@ -408,7 +517,8 @@ export function createApp() { }); } - res.status(env.knowledgeLifecycleEnforced ? 201 : 202).json({ + dispatchAcceptedOcr(result); + res.status("phase" in result ? 202 : env.knowledgeLifecycleEnforced ? 201 : 202).json({ ...result, uploadedResource: req.file.originalname }); diff --git a/src/config/env.ts b/src/config/env.ts index 5cca087..b2d3beb 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -52,5 +52,13 @@ export const env = { knowledgeLifecycleEnforced: booleanEnv("KNOWLEDGE_LIFECYCLE_ENFORCED", false), ingestWritesEnabled: booleanEnv("INGEST_WRITES_ENABLED", true), lifecycleReconcileIntervalMs: Number(process.env.LIFECYCLE_RECONCILE_INTERVAL_MS ?? 300000), - lifecycleIndexingStaleTimeoutMs: Number(process.env.LIFECYCLE_INDEXING_STALE_TIMEOUT_MS ?? 1800000) + lifecycleIndexingStaleTimeoutMs: Number(process.env.LIFECYCLE_INDEXING_STALE_TIMEOUT_MS ?? 1800000), + ocrIngestEnabled: booleanEnv("OCR_INGEST_ENABLED", false), + ocrServiceUrl: process.env.OCR_SERVICE_URL ?? "http://ocr-service:8000", + ocrInternalToken: process.env.OCR_INTERNAL_TOKEN ?? "", + ocrArtifactRoot: process.env.OCR_ARTIFACT_ROOT ?? "/data/ingestions", + ocrMaxUploadBytes: Number(process.env.OCR_MAX_UPLOAD_BYTES ?? 50 * 1024 * 1024), + ocrMaxPages: Number(process.env.OCR_MAX_PAGES ?? 100), + ocrPageTimeoutMs: Number(process.env.OCR_PAGE_TIMEOUT_MS ?? 60_000), + ocrTotalTimeoutMs: Number(process.env.OCR_TOTAL_TIMEOUT_MS ?? 15 * 60_000) } as const; diff --git a/src/modules/catalog/reconciler.ts b/src/modules/catalog/reconciler.ts index 49e5883..4b6cc34 100644 --- a/src/modules/catalog/reconciler.ts +++ b/src/modules/catalog/reconciler.ts @@ -1,6 +1,8 @@ import { env } from "../../config/env.js"; import type { VectorStoreClient } from "../vectorstore/client.js"; import type { CatalogRepository } from "./repository.js"; +import type { OcrDispatcher } from "../ocr/dispatcher.js"; +import type { OcrRetentionService } from "../ocr/retention.js"; export interface ReconcilerStatus { ok: boolean; @@ -9,6 +11,8 @@ export interface ReconcilerStatus { inconsistentSources: string[]; orphanedVersionsRecovered?: number; orphanedVersionsFailed?: number; + ocrLeasesRecovered?: number; + ocrRetentionDeleted?: number; invariantViolations?: string[]; } @@ -18,7 +22,9 @@ export class KnowledgeLifecycleReconciler { constructor( private readonly catalog: CatalogRepository | undefined, - private readonly vectorStore: VectorStoreClient + private readonly vectorStore: VectorStoreClient, + private readonly ocrDispatcher?: Pick, + private readonly ocrRetention?: Pick ) {} getStatus(): ReconcilerStatus { @@ -49,6 +55,9 @@ export class KnowledgeLifecycleReconciler { const activeVersions = await catalog.resolveActiveVersions(); const inconsistentSources: string[] = []; const invariantViolations = await catalog.validateActiveInvariant(); + const ocrLeasesRecovered = await this.ocrDispatcher?.recoverExpiredLeases() ?? 0; + await this.ocrDispatcher?.dispatchAvailable(); + const retention = await this.ocrRetention?.runOnce(); const orphanRecovery = await this.recoverOrphanedVersions(catalog); for (const version of activeVersions) { @@ -71,6 +80,8 @@ export class KnowledgeLifecycleReconciler { inconsistentSources, orphanedVersionsRecovered: orphanRecovery.ready, orphanedVersionsFailed: orphanRecovery.failed, + ocrLeasesRecovered, + ocrRetentionDeleted: (retention?.deleted ?? 0) + (retention?.resumed ?? 0), invariantViolations }; }); diff --git a/src/modules/catalog/repository.ts b/src/modules/catalog/repository.ts index 7653836..811ad46 100644 --- a/src/modules/catalog/repository.ts +++ b/src/modules/catalog/repository.ts @@ -1,7 +1,10 @@ import type { AvailableScope, ChunkMode, IngestSourceInput, SourceType, SourceVersionState, RetrieveScope } from "../../shared/types/rag.js"; +import { sha256Hex } from "../../shared/utils/ids.js"; import { withCatalogAdvisoryLock, withCatalogAdvisorySharedLocks, withCatalogTryAdvisoryLock, withTransaction, type PgPool, type PgPoolClient } from "./client.js"; import { CatalogError } from "./errors.js"; import { normalizeExpectedActiveVersion } from "./lifecycle.js"; +import type { OcrResult } from "../ocr/client.js"; +import type { OcrRetentionCandidate } from "../ocr/retention.js"; export interface CatalogDocumentInput { documentId: string; @@ -12,9 +15,26 @@ export interface CatalogDocumentInput { mimeType: string; title: string; chunkCount: number; + extractionMethod?: "native" | "ocr"; + artifactManifestPath?: string; +} + +export interface OcrPageInput { + documentId: string; + page: number; + extractionMethod: "native" | "ocr"; + nativeTextHash: string; +} + +export interface OcrJobInput { + documentId: string; + remoteIdempotencyKey: string; + requestedPages: number[]; + configVersion: "ocr-v1"; } export interface CreateVersionInput { + versionId?: string; sourceId: string; sourceType: SourceType; sourceRef: string; @@ -31,6 +51,8 @@ export interface CreateVersionInput { expectedDocumentCount: number; expectedPointCount: number; documents: CatalogDocumentInput[]; + ocrPages?: OcrPageInput[]; + ocrJobs?: OcrJobInput[]; } export interface CatalogVersionRow { @@ -79,6 +101,7 @@ export interface PendingOcrIdentity { export interface OcrJobRow { jobId: string; + versionId: string; documentId: string; remoteJobId: string | null; remoteIdempotencyKey: string; @@ -116,10 +139,12 @@ type VersionDbRow = { expected_point_count: string | number; verified_point_count: string | number; qdrant_collection: string; + artifact_state?: "none" | "present" | "retention_deleting" | "retention_deleted"; }; type OcrJobDbRow = { ocr_job_id: string; + ocr_version_id: string; ocr_document_id: string; ocr_remote_job_id: string | null; ocr_remote_idempotency_key: string; @@ -136,7 +161,7 @@ type OcrJobDbRow = { }; const OCR_JOB_COLUMNS = [ - ["job_id", "ocr_job_id"], ["document_id", "ocr_document_id"], + ["job_id", "ocr_job_id"], ["version_id", "ocr_version_id"], ["document_id", "ocr_document_id"], ["remote_job_id", "ocr_remote_job_id"], ["remote_idempotency_key", "ocr_remote_idempotency_key"], ["state", "ocr_state"], ["requested_pages", "ocr_requested_pages"], ["completed_pages", "ocr_completed_pages"], ["config_version", "ocr_config_version"], @@ -173,6 +198,7 @@ function toVersion(row: VersionDbRow): CatalogVersionRow { function toOcrJob(row: OcrJobDbRow): OcrJobRow { return { jobId: row.ocr_job_id, + versionId: row.ocr_version_id, documentId: row.ocr_document_id, remoteJobId: row.ocr_remote_job_id, remoteIdempotencyKey: row.ocr_remote_idempotency_key, @@ -189,6 +215,14 @@ function toOcrJob(row: OcrJobDbRow): OcrJobRow { }; } +function ingestionPhase(state: SourceVersionState, jobs: Array): string { + if (state !== "indexing") return state; + if (jobs.some((job) => job === "failed")) return "failed"; + if (jobs.some((job) => job === "running")) return "ocr_running"; + if (jobs.some((job) => job === "queued")) return "ocr_queued"; + return "indexing"; +} + export class CatalogRepository { constructor(private readonly pool: PgPool) {} @@ -338,6 +372,19 @@ export class CatalogRepository { }); } + async claimOcrJob(jobId: string, leaseMs: number): Promise { + const result = await this.pool.query( + `UPDATE rag_ocr_jobs + SET state = 'running', attempt_count = attempt_count + 1, + started_at = COALESCE(started_at, now()), heartbeat_at = now(), + lease_expires_at = now() + ($2::text || ' milliseconds')::interval + WHERE job_id = $1 AND state = 'queued' AND (next_attempt_at IS NULL OR next_attempt_at <= now()) + RETURNING ${ocrJobColumns()}`, + [jobId, String(leaseMs)] + ); + return result.rows[0] ? toOcrJob(result.rows[0]) : undefined; + } + async recoverExpiredOcrLeases(): Promise { const result = await this.pool.query( `UPDATE rag_ocr_jobs @@ -348,6 +395,142 @@ export class CatalogRepository { return result.rows.map(toOcrJob); } + async setOcrRemoteJob(jobId: string, remoteJobId: string, leaseMs: number): Promise { + const result = await this.pool.query( + `UPDATE rag_ocr_jobs SET remote_job_id = $2, heartbeat_at = now(), + lease_expires_at = now() + ($3::text || ' milliseconds')::interval + WHERE job_id = $1 AND state = 'running'`, + [jobId, remoteJobId, String(leaseMs)] + ); + if (result.rowCount !== 1) throw new CatalogError("OCR job lease was lost", 409, "OCR_LEASE_LOST"); + } + + async requeueOcrJob(jobId: string, code: string, detail: string, delayMs: number): Promise { + await this.pool.query( + `UPDATE rag_ocr_jobs SET state = 'queued', heartbeat_at = NULL, lease_expires_at = NULL, + next_attempt_at = now() + ($4::text || ' milliseconds')::interval, error_code = $2, error_detail = $3 + WHERE job_id = $1 AND state = 'running'`, + [jobId, code, detail.slice(0, 2000), String(delayMs)] + ); + } + + async completeOcrJob(jobId: string, result: OcrResult): Promise { + return withTransaction(this.pool, async (client) => { + const identity = await client.query<{ version_id: string; document_id: string }>( + "SELECT version_id, document_id FROM rag_ocr_jobs WHERE job_id = $1 AND state = 'running' FOR UPDATE", + [jobId] + ); + if (!identity.rowCount) throw new CatalogError("OCR job lease was lost", 409, "OCR_LEASE_LOST"); + const { version_id: versionId, document_id: documentId } = identity.rows[0]; + for (const page of result.pages) { + await client.query( + `UPDATE rag_document_pages SET ocr_text_hash = $4, candidate_text_hash = $4, metrics = $5 + WHERE version_id = $1 AND document_id = $2 AND page_number = $3 AND extraction_method = 'ocr'`, + [versionId, documentId, page.page, sha256Hex(page.text), page.metrics] + ); + } + await client.query( + `UPDATE rag_ocr_jobs SET state = 'succeeded', completed_pages = cardinality(requested_pages), + heartbeat_at = now(), lease_expires_at = NULL, next_attempt_at = NULL, + error_code = NULL, error_detail = NULL, completed_at = now() + WHERE job_id = $1`, + [jobId] + ); + const pending = await client.query( + "SELECT 1 FROM rag_ocr_jobs WHERE version_id = $1 AND state <> 'succeeded' LIMIT 1", + [versionId] + ); + return pending.rowCount === 0; + }); + } + + async failOcrJob(jobId: string, code: string, detail: string): Promise { + await this.pool.query( + `UPDATE rag_ocr_jobs SET state = 'failed', lease_expires_at = NULL, next_attempt_at = NULL, + error_code = $2, error_detail = $3, completed_at = now() WHERE job_id = $1 AND state = 'running'`, + [jobId, code, detail.slice(0, 2000)] + ); + } + + async getIngestionStatus(versionId: string): Promise | undefined> { + const version = await this.pool.query<{ + source_id: string; + state: SourceVersionState; + error_code: string | null; + error_detail: string | null; + }>("SELECT source_id, state, error_code, error_detail FROM rag_source_versions WHERE version_id = $1", [versionId]); + if (!version.rowCount) return undefined; + const documents = await this.pool.query<{ + document_id: string; + index_state: "pending" | "indexing" | "ready" | "failed"; + job_state: OcrJobRow["state"] | null; + completed_pages: string | number | null; + requested_pages: number[] | null; + }>( + `SELECT d.document_id, d.index_state, j.state AS job_state, j.completed_pages, j.requested_pages + FROM rag_version_documents d LEFT JOIN rag_ocr_jobs j + ON j.version_id = d.version_id AND j.document_id = d.document_id + WHERE d.version_id = $1 ORDER BY d.document_id`, + [versionId] + ); + const pages = await this.pool.query<{ + document_id: string; + page_number: string | number; + extraction_method: "native" | "ocr" | "blank"; + blocked_reason: string | null; + }>( + "SELECT document_id, page_number, extraction_method, blocked_reason FROM rag_document_pages WHERE version_id = $1 ORDER BY document_id, page_number", + [versionId] + ); + const row = version.rows[0]; + const phase = ingestionPhase(row.state, documents.rows.map(({ job_state }) => job_state)); + return { + sourceId: row.source_id, + versionId, + state: row.state, + phase, + activated: row.state === "active", + documents: documents.rows.map((document) => { + const documentPages = pages.rows.filter((page) => page.document_id === document.document_id); + const state = document.job_state === "failed" || document.index_state === "failed" + ? "failed" + : document.job_state === "succeeded" + ? "ocr_complete" + : document.job_state === "running" + ? "ocr_running" + : document.job_state === "queued" + ? "ocr_queued" + : "native_complete"; + return { + documentId: document.document_id, + state, + completedPages: documentPages.filter((page) => page.extraction_method !== "ocr" && !page.blocked_reason).length + + Number(document.completed_pages ?? 0), + totalPages: documentPages.length, + pages: documentPages.map((page) => ({ + page: Number(page.page_number), + method: page.extraction_method, + state: page.blocked_reason + ? "failed" + : page.extraction_method === "native" + ? "native_complete" + : page.extraction_method === "blank" + ? "blank" + : document.job_state === "succeeded" + ? "ocr_complete" + : document.job_state === "running" + ? "ocr_running" + : "ocr_queued", + ...(page.blocked_reason ? { errorCode: page.blocked_reason } : {}) + })) + }; + }), + error: row.error_code ? { code: row.error_code, message: row.error_detail ?? row.error_code, retryable: false } : null, + statusUrl: `/ingestions/${versionId}`, + reviewUrl: row.state === "review_required" ? `/ingestions/${versionId}/review` : null + }; + } + async markReviewRequired(versionId: string): Promise { const result = await this.pool.query( `UPDATE rag_source_versions SET state = 'review_required' @@ -379,7 +562,8 @@ export class CatalogRepository { async rejectOcrVersion(versionId: string, reviewedBy: string, reason: string): Promise { const result = await this.pool.query( `UPDATE rag_source_versions - SET state = 'rejected', reviewed_at = now(), reviewed_by = $2, error_code = 'OCR_REJECTED', error_detail = $3 + SET state = 'rejected', reviewed_at = now(), reviewed_by = $2, error_code = 'OCR_REJECTED', error_detail = $3, + retention_due_at = now() + interval '7 days' WHERE version_id = $1 AND state = 'review_required'`, [versionId, reviewedBy, reason.slice(0, 2000)] ); @@ -414,17 +598,19 @@ export class CatalogRepository { [input.sourceId] ); const versionNumber = Number(versionNumberResult.rows[0].next_version_number); - const versionResult = await client.query( - `INSERT INTO rag_source_versions( - source_id, version_number, previous_version_id, state, original_manifest_hash, - source_content_hash, processing_fingerprint, metadata_hash, tags, activate_requested, - base_active_version_id, embedding_provider, embedding_model, embedding_dimensions, - qdrant_collection, expected_document_count, expected_point_count - ) VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) - RETURNING *`, - [ - input.sourceId, - versionNumber, + const versionResult = await client.query( + `INSERT INTO rag_source_versions( + version_id, source_id, version_number, previous_version_id, state, original_manifest_hash, + source_content_hash, processing_fingerprint, metadata_hash, tags, activate_requested, + base_active_version_id, embedding_provider, embedding_model, embedding_dimensions, + qdrant_collection, expected_document_count, expected_point_count, artifact_state + ) VALUES (COALESCE($1::uuid, gen_random_uuid()), $2, $3, $4, 'pending', $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, + CASE WHEN $6::char(64) IS NULL THEN 'present' ELSE 'none' END) + RETURNING *`, + [ + input.versionId ?? null, + input.sourceId, + versionNumber, baseActiveVersionId, input.originalManifestHash, input.sourceContentHash, @@ -436,34 +622,51 @@ export class CatalogRepository { input.embeddingProvider, input.embeddingModel, input.embeddingDimensions, - input.qdrantCollection, - input.expectedDocumentCount, - input.expectedPointCount + input.qdrantCollection, + input.expectedDocumentCount, + input.expectedPointCount ] ); const version = toVersion(versionResult.rows[0]); for (const document of input.documents) { - await client.query( - `INSERT INTO rag_version_documents( - version_id, document_id, document_key, original_hash, original_hash_kind, - content_hash, mime_type, title, index_state, chunk_count - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending', $9)`, - [ + await client.query( + `INSERT INTO rag_version_documents( + version_id, document_id, document_key, original_hash, original_hash_kind, + content_hash, mime_type, title, extraction_method, index_state, chunk_count, artifact_manifest_path + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'pending', $10, $11)`, + [ version.versionId, document.documentId, document.documentKey, document.originalHash, document.originalHashKind, document.contentHash, - document.mimeType, - document.title, - document.chunkCount - ] + document.mimeType, + document.title, + document.extractionMethod ?? "native", + document.chunkCount, + document.artifactManifestPath ?? null + ] + ); + } + + for (const page of input.ocrPages ?? []) { + await client.query( + `INSERT INTO rag_document_pages(version_id, document_id, page_number, extraction_method, native_text_hash) + VALUES ($1, $2, $3, $4, $5)`, + [version.versionId, page.documentId, page.page, page.extractionMethod, page.nativeTextHash] + ); + } + for (const job of input.ocrJobs ?? []) { + await client.query( + `INSERT INTO rag_ocr_jobs(version_id, document_id, remote_idempotency_key, state, requested_pages, config_version) + VALUES ($1, $2, $3, 'queued', $4, $5)`, + [version.versionId, job.documentId, job.remoteIdempotencyKey, job.requestedPages, job.configVersion] ); } - return version; + return version; }); } @@ -501,7 +704,7 @@ export class CatalogRepository { } const result = await this.pool.query( `UPDATE rag_source_versions - SET state = 'failed', error_code = $2, error_detail = $3 + SET state = 'failed', error_code = $2, error_detail = $3, retention_due_at = now() + interval '7 days' WHERE version_id = $1 AND state IN ('pending', 'indexing')`, [versionId, code, detail.slice(0, 2000)] ); @@ -538,6 +741,9 @@ export class CatalogRepository { if (!["ready", "superseded", "active"].includes(version.rows[0].state)) { throw new CatalogError("Only ready or superseded versions can be activated", 409, "VERSION_NOT_ACTIVATABLE"); } + if (["retention_deleting", "retention_deleted"].includes(version.rows[0].artifact_state ?? "none")) { + throw new CatalogError("Version artifacts are unavailable", 409, "VERSION_ARTIFACTS_UNAVAILABLE"); + } if (version.rows[0].state === "active") { if (currentActive !== versionId) { throw new CatalogError("Active version invariant is inconsistent", 503, "ACTIVE_VERSION_INVARIANT_FAILED"); @@ -547,7 +753,7 @@ export class CatalogRepository { if (currentActive) { await client.query( - "UPDATE rag_source_versions SET state = 'superseded', superseded_at = now() WHERE source_id = $1 AND version_id = $2", + "UPDATE rag_source_versions SET state = 'superseded', superseded_at = now(), retention_due_at = now() + interval '30 days' WHERE source_id = $1 AND version_id = $2", [sourceId, currentActive] ); } @@ -663,6 +869,50 @@ export class CatalogRepository { await this.pool.query("UPDATE rag_sources SET needs_reingest = true WHERE source_id = $1", [sourceId]); } + async listOcrRetentionCandidates(now: Date): Promise { + const result = await this.pool.query<{ version_id: string; source_id: string; state: SourceVersionState; artifact_state: OcrRetentionCandidate["artifactState"] }>( + `SELECT v.version_id, v.source_id, v.state, v.artifact_state FROM rag_source_versions v + JOIN rag_sources s ON s.source_id = v.source_id + WHERE v.artifact_state IN ('present', 'retention_deleting') + AND s.active_version_id IS DISTINCT FROM v.version_id + AND (v.artifact_state = 'retention_deleting' + OR (v.state = 'review_required' AND v.created_at + interval '30 days' <= $1) + OR (v.state IN ('failed', 'rejected') AND COALESCE(v.retention_due_at, v.reviewed_at + interval '7 days', v.created_at + interval '7 days') <= $1) + OR (v.state = 'superseded' AND COALESCE(v.retention_due_at, v.superseded_at + interval '30 days') <= $1)) + ORDER BY v.created_at, v.version_id`, [now] + ); + return result.rows.map((row) => ({ versionId: row.version_id, sourceId: row.source_id, state: row.state, artifactState: row.artifact_state })); + } + + async expireOcrReview(sourceId: string, versionId: string, now: Date): Promise { + const result = await this.pool.query( + `UPDATE rag_source_versions v SET state = 'rejected', reviewed_at = $3, error_code = 'REVIEW_EXPIRED', + error_detail = 'OCR review expired after 30 days', retention_due_at = $3 + interval '7 days' + FROM rag_sources s WHERE v.source_id = $1 AND v.version_id = $2 AND v.source_id = s.source_id + AND v.state = 'review_required' AND v.created_at + interval '30 days' <= $3 + AND s.active_version_id IS DISTINCT FROM v.version_id`, [sourceId, versionId, now] + ); + return result.rowCount === 1; + } + + async claimOcrRetentionDeletion(sourceId: string, versionId: string, state: SourceVersionState, artifactState: OcrRetentionCandidate["artifactState"]): Promise { + const result = await this.pool.query( + `UPDATE rag_source_versions v SET artifact_state = 'retention_deleting' FROM rag_sources s + WHERE v.source_id = $1 AND v.version_id = $2 AND v.state = $3 AND v.artifact_state = $4 + AND v.source_id = s.source_id AND s.active_version_id IS DISTINCT FROM v.version_id`, [sourceId, versionId, state, artifactState] + ); + return result.rowCount === 1; + } + + async completeOcrRetentionDeletion(sourceId: string, versionId: string, state: SourceVersionState): Promise { + const result = await this.pool.query( + `UPDATE rag_source_versions v SET artifact_state = 'retention_deleted' FROM rag_sources s + WHERE v.source_id = $1 AND v.version_id = $2 AND v.state = $3 AND v.artifact_state = 'retention_deleting' + AND v.source_id = s.source_id AND s.active_version_id IS DISTINCT FROM v.version_id`, [sourceId, versionId, state] + ); + return result.rowCount === 1; + } + async rollback(sourceId: string, targetVersionId: string, expectedActiveVersionId: string): Promise { return this.withSourceLock(sourceId, async () => this.withVersionExclusiveLock(targetVersionId, async () => { const version = await this.activateVersion(sourceId, targetVersionId, expectedActiveVersionId); diff --git a/src/modules/ingest/service.ts b/src/modules/ingest/service.ts index cd0ce6d..9ff965a 100644 --- a/src/modules/ingest/service.ts +++ b/src/modules/ingest/service.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { readFile } from "node:fs/promises"; -import { parseDocument, isSupportedDocument } from "../parsers/parser-registry.js"; +import { randomUUID } from "node:crypto"; +import { parseDocument, isSupportedDocument, parsePdfPages, type ParsedPdfPage } from "../parsers/parser-registry.js"; import { chunkDocument, codeChunkingPolicy, documentalChunkingPolicy } from "../process/chunking.js"; import type { EmbeddingProvider } from "../embeddings/provider.js"; import type { VectorStoreClient } from "../vectorstore/client.js"; @@ -21,6 +22,8 @@ import { import { listFilesRecursively } from "../../shared/utils/files.js"; import { env } from "../../config/env.js"; import { CatalogError, type CatalogDocumentInput, type CatalogRepository } from "../catalog/repository.js"; +import { selectPdfPagesForOcr } from "../ocr/detection.js"; +import { stageOcrArtifacts } from "../ocr/artifacts.js"; type PreparedDocument = CatalogDocumentInput & { chunks: ReturnType; @@ -36,11 +39,38 @@ type OriginalDocument = { originalHash: string; }; +interface OcrPlan { + original: OriginalDocument; + pages: ParsedPdfPage[]; + requestedPages: number[]; + contentHash: string; + mimeType: string; + title: string; +} + +export interface OcrAccepted { + accepted: true; + sourceId: string; + versionId: string; + versionNumber: number; + state: "indexing"; + phase: "ocr_queued"; + statusUrl: string; + reviewUrl: null; + activated: false; +} + +interface OcrIngestOptions { + enabled: boolean; + artifactRoot: string; +} + export class IngestService { constructor( private readonly embeddingProvider: EmbeddingProvider, private readonly vectorStore: VectorStoreClient, - private readonly catalog?: CatalogRepository + private readonly catalog?: CatalogRepository, + private readonly ocr?: OcrIngestOptions ) {} async cleanup(scope: RetrieveScope): Promise<{ deleted: number }> { @@ -51,7 +81,7 @@ export class IngestService { return { deleted: count }; } - async ingest(source: IngestSourceInput): Promise { + async ingest(source: IngestSourceInput): Promise { if (!env.ingestWritesEnabled) { throw new CatalogError("Ingest writes are disabled during knowledge catalog maintenance", 503, "INGEST_WRITES_DISABLED"); } @@ -135,7 +165,7 @@ export class IngestService { }; } - private async ingestWithLifecycle(source: IngestSourceInput, catalog: CatalogRepository): Promise { + private async ingestWithLifecycle(source: IngestSourceInput, catalog: CatalogRepository): Promise { const activate = source.activate ?? true; if (activate && source.expectedActiveVersionId === undefined) { throw new CatalogError("expectedActiveVersionId is required when activate is true", 400, "EXPECTED_ACTIVE_VERSION_REQUIRED"); @@ -158,16 +188,24 @@ export class IngestService { if (!originalManifestHash) { throw new CatalogError("Original manifest hash could not be calculated", 422, "ORIGINAL_MANIFEST_HASH_UNAVAILABLE"); } + const tags = source.tags ?? []; + const embeddingDimensions = this.embeddingProvider.dimensions ?? 4096; + const metadataHash = buildMetadataHash(tags, { sourceType: source.sourceType, sourceRef: source.sourceRef }); + if (this.ocr?.enabled) { + const plans = await this.planOcr(originalDocuments); + if (plans.some(({ requestedPages }) => requestedPages.length > 0)) { + return await this.acceptOcr(source, sourceId, attemptId, originalManifestHash, plans, tags, metadataHash, embeddingDimensions, catalog); + } + } + const preparedDocuments = await this.prepareDocuments(originalDocuments); if (preparedDocuments.length === 0) { throw new CatalogError("The source has no supported documents with useful content", 422, "EMPTY_SOURCE"); } - const tags = source.tags ?? []; const sourceContentHash = hashOrderedPairs(preparedDocuments.map((document) => [document.documentKey, document.contentHash])); - const embeddingDimensions = this.embeddingProvider.dimensions ?? 4096; const processingFingerprint = buildProcessingFingerprint({ parserVersion: "native-v1", normalizationPolicy: "bom-crlf-trim-final-lf-v1", @@ -179,8 +217,6 @@ export class IngestService { embeddingModel: this.embeddingProvider.modelName, embeddingDimensions }); - const metadataHash = buildMetadataHash(tags, { sourceType: source.sourceType, sourceRef: source.sourceRef }); - return await catalog.withSourceLock(sourceId, async () => { const reusable = await catalog.findReusableVersion({ sourceId, sourceContentHash, processingFingerprint, metadataHash }); if (reusable?.state === "active") { @@ -311,6 +347,146 @@ export class IngestService { } } + private async planOcr(originals: OriginalDocument[]): Promise { + const plans: OcrPlan[] = []; + for (const original of originals) { + if (path.extname(original.filePath).toLowerCase() === ".pdf") { + const pages = await parsePdfPages(original.filePath); + plans.push({ + original, + pages, + requestedPages: selectPdfPagesForOcr(original.filePath, pages), + contentHash: sha256Hex(normalizeContentForHash(pages.map(({ text }) => text).join("\n"))), + mimeType: "application/pdf", + title: path.basename(original.filePath) + }); + continue; + } + const parsed = await parseDocument(original.filePath); + plans.push({ + original, + pages: [], + requestedPages: [], + contentHash: sha256Hex(normalizeContentForHash(parsed.content)), + mimeType: parsed.mimeType, + title: parsed.title + }); + } + return plans; + } + + private async acceptOcr( + source: IngestSourceInput, + sourceId: string, + attemptId: string, + originalManifestHash: string, + plans: OcrPlan[], + tags: string[], + metadataHash: string, + embeddingDimensions: number, + catalog: CatalogRepository + ): Promise { + const processingFingerprint = buildProcessingFingerprint({ + parserVersion: "native-pages-v1+ocr-v1", + detectionPolicyVersion: "pdf-detection-v1", + normalizationPolicy: "bom-crlf-trim-final-lf-v1", + chunking: { code: codeChunkingPolicy, documental: documentalChunkingPolicy }, + embeddingProvider: this.embeddingProvider.providerName, + embeddingModel: this.embeddingProvider.modelName, + embeddingDimensions + }); + return catalog.withSourceLock(sourceId, async () => { + const pending = await catalog.findPendingOcrVersion({ sourceId, originalManifestHash, processingFingerprint, metadataHash }); + if (pending) { + await catalog.updateAttempt(attemptId, { state: "completed", versionId: pending.version.versionId, inputHash: originalManifestHash }); + return this.ocrAccepted(sourceId, pending.version.versionId, pending.version.versionNumber); + } + + const versionId = randomUUID(); + const staged = await stageOcrArtifacts({ + rootDirectory: this.ocr!.artifactRoot, + versionId, + createdAt: new Date().toISOString(), + documents: plans.map(({ original }) => ({ + documentId: original.documentId, + documentKey: original.documentKey, + bytes: original.bytes + })) + }); + if (staged.originalManifestHash !== originalManifestHash) { + throw new CatalogError("Durable OCR manifest does not match the source", 500, "OCR_ARTIFACT_INTEGRITY_FAILED"); + } + let created; + try { + created = await catalog.createPendingVersion({ + versionId, + sourceId, + sourceType: source.sourceType, + sourceRef: source.sourceRef, + originalManifestHash, + sourceContentHash: null, + processingFingerprint, + metadataHash, + tags, + activateRequested: source.activate ?? true, + embeddingProvider: this.embeddingProvider.providerName, + embeddingModel: this.embeddingProvider.modelName, + embeddingDimensions, + qdrantCollection: env.qdrantCollection, + expectedDocumentCount: plans.length, + expectedPointCount: 0, + documents: plans.map(({ original, requestedPages, contentHash, mimeType, title }) => ({ + documentId: original.documentId, + documentKey: original.documentKey, + originalHash: original.originalHash, + originalHashKind: "bytes", + contentHash: requestedPages.length > 0 ? null : contentHash, + mimeType, + title, + chunkCount: 0, + extractionMethod: requestedPages.length > 0 ? "ocr" : "native", + artifactManifestPath: staged.manifestPath + })), + ocrPages: plans.flatMap(({ original, pages, requestedPages }) => pages.map((page) => ({ + documentId: original.documentId, + page: page.page, + extractionMethod: requestedPages.includes(page.page) ? "ocr" as const : "native" as const, + nativeTextHash: page.textSha256 + }))), + ocrJobs: plans.filter(({ requestedPages }) => requestedPages.length > 0).map(({ original, requestedPages }) => ({ + documentId: original.documentId, + remoteIdempotencyKey: `${original.originalHash}:ocr-v1:${sha256Hex(JSON.stringify(requestedPages))}`, + requestedPages, + configVersion: "ocr-v1" + })) + }); + } catch (error) { + if ((error as { code?: unknown }).code !== "23505") throw error; + const winner = await catalog.findPendingOcrVersion({ sourceId, originalManifestHash, processingFingerprint, metadataHash }); + if (!winner) throw error; + await catalog.updateAttempt(attemptId, { state: "completed", versionId: winner.version.versionId, inputHash: originalManifestHash }); + return this.ocrAccepted(sourceId, winner.version.versionId, winner.version.versionNumber); + } + await catalog.markIndexing(created.versionId); + await catalog.updateAttempt(attemptId, { state: "completed", versionId: created.versionId, inputHash: originalManifestHash }); + return this.ocrAccepted(sourceId, created.versionId, created.versionNumber); + }); + } + + private ocrAccepted(sourceId: string, versionId: string, versionNumber: number): OcrAccepted { + return { + accepted: true, + sourceId, + versionId, + versionNumber, + state: "indexing", + phase: "ocr_queued", + statusUrl: `/ingestions/${versionId}`, + reviewUrl: null, + activated: false + }; + } + private async readOriginalDocuments(source: IngestSourceInput, sourceId: string, files: string[]): Promise { const originals: OriginalDocument[] = []; for (const filePath of files) { diff --git a/src/modules/ocr/client.ts b/src/modules/ocr/client.ts index dc4ec86..b63c93d 100644 --- a/src/modules/ocr/client.ts +++ b/src/modules/ocr/client.ts @@ -69,11 +69,12 @@ export class OcrClient { this.sleep = options.sleep ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds))); } - async submit(file: Buffer, expected: { documentSha256: string; pages: number[] }): Promise { + async submit(file: Buffer, expected: { documentSha256: string; pages: number[] }, persistedIdempotencyKey?: string): Promise { assertExpected(expected); if (sha256Hex(file) !== expected.documentSha256) throw integrityError(); const payload = { documentSha256: expected.documentSha256, pages: expected.pages, ...OCR_CONFIG }; - const idempotencyKey = `${expected.documentSha256}:ocr-v1:${sha256Hex(JSON.stringify(expected.pages))}`; + const idempotencyKey = persistedIdempotencyKey ?? `${expected.documentSha256}:ocr-v1:${sha256Hex(JSON.stringify(expected.pages))}`; + if (!idempotencyKey.trim() || /[\r\n]/u.test(idempotencyKey)) throw new TypeError("OCR idempotency key is invalid"); const value = await this.requestJson("/v1/jobs", () => { const form = new FormData(); form.set("file", new Blob([new Uint8Array(file)], { type: "application/pdf" }), "original.pdf"); diff --git a/src/modules/ocr/dispatcher.ts b/src/modules/ocr/dispatcher.ts new file mode 100644 index 0000000..01fe743 --- /dev/null +++ b/src/modules/ocr/dispatcher.ts @@ -0,0 +1,108 @@ +import type { OcrJobRow } from "../catalog/repository.js"; +import { OcrClientError, type OcrClient, type OcrResult } from "./client.js"; + +export interface OcrDispatchStore { + claimNextOcrJob(leaseMs: number): Promise; + claimOcrJob(jobId: string, leaseMs: number): Promise; + recoverExpiredOcrLeases(): Promise; + setOcrRemoteJob(jobId: string, remoteJobId: string, leaseMs: number): Promise; + requeueOcrJob(jobId: string, code: string, detail: string, delayMs: number): Promise; + completeOcrJob(jobId: string, result: OcrResult): Promise; + failOcrJob(jobId: string, code: string, detail: string): Promise; + markReviewRequired(versionId: string): Promise; + markFailed(versionId: string, code: string, detail: string): Promise; +} + +export type OcrDispatchInput = { bytes: Buffer; documentSha256: string }; +export type OcrDispatchResult = "idle" | "pending" | "succeeded" | "failed"; + +export class OcrDispatcher { + private activeDrain: Promise | undefined; + + constructor( + private readonly store: OcrDispatchStore, + private readonly client: OcrClient, + private readonly loadInput: (job: OcrJobRow) => Promise, + private readonly leaseMs = 30_000 + ) {} + + async runOnce(): Promise { + const job = await this.store.claimNextOcrJob(this.leaseMs); + if (!job) return "idle"; + + return this.dispatch(job); + } + + private async dispatch(job: OcrJobRow): Promise { + try { + const input = await this.loadInput(job); + let remoteJobId = job.remoteJobId; + if (!remoteJobId) { + const acknowledgement = await this.client.submit(input.bytes, { + documentSha256: input.documentSha256, + pages: job.requestedPages + }, job.remoteIdempotencyKey); + remoteJobId = acknowledgement.jobId; + await this.store.setOcrRemoteJob(job.jobId, remoteJobId, this.leaseMs); + } + + const status = await this.client.getStatus(remoteJobId); + if (status.status === "queued" || status.status === "running") { + const delayMs = Math.min(2_000 * 2 ** Math.max(0, job.attemptCount - 1), 15_000); + await this.store.requeueOcrJob(job.jobId, "OCR_PENDING", status.status, delayMs); + return "pending"; + } + if (status.status === "failed") { + const code = status.error?.code ?? "OCR_REMOTE_FAILED"; + await this.fail(job, code, status.error?.message ?? "OCR processing failed"); + return "failed"; + } + + const result = await this.client.getResult(remoteJobId, { + documentSha256: input.documentSha256, + pages: job.requestedPages + }); + const versionComplete = await this.store.completeOcrJob(job.jobId, result); + if (versionComplete) await this.store.markReviewRequired(job.versionId); + return "succeeded"; + } catch (error) { + const detail = error instanceof Error ? error.message : "Unknown OCR dispatch failure"; + if (error instanceof OcrClientError && error.retryable) { + await this.store.requeueOcrJob(job.jobId, error.code, detail, 15_000); + return "pending"; + } + const code = error instanceof OcrClientError ? error.code : "OCR_DISPATCH_FAILED"; + await this.fail(job, code, detail); + return "failed"; + } + } + + async recoverExpiredLeases(): Promise { + const recovered = await this.store.recoverExpiredOcrLeases(); + for (const job of recovered) { + const claimed = await this.store.claimOcrJob(job.jobId, this.leaseMs); + if (claimed) await this.dispatch(claimed); + } + return recovered.length; + } + + dispatchAvailable(): Promise { + if (this.activeDrain) return this.activeDrain; + const drain = this.drainAvailable(); + this.activeDrain = drain; + const clear = () => { if (this.activeDrain === drain) this.activeDrain = undefined; }; + void drain.then(clear, clear); + return drain; + } + + private async drainAvailable(): Promise { + let processed = 0; + while (await this.runOnce() !== "idle") processed += 1; + return processed; + } + + private async fail(job: OcrJobRow, code: string, detail: string): Promise { + await this.store.failOcrJob(job.jobId, code, detail); + await this.store.markFailed(job.versionId, code, detail); + } +} diff --git a/src/modules/ocr/indexing.ts b/src/modules/ocr/indexing.ts new file mode 100644 index 0000000..234c36c --- /dev/null +++ b/src/modules/ocr/indexing.ts @@ -0,0 +1,67 @@ +import { CatalogError } from "../catalog/errors.js"; + +export interface ApprovedOcrCandidate { + versionId: string; + sourceId: string; + state: "indexing" | "rejected"; + activateRequested: boolean; + expectedActiveVersionId: string | null; + reviewedText: string; + reviewedTextSha256: string; + processingFingerprint: string; + metadataHash: string; +} + +export interface OcrIndexingStore { + findReusableVersion(candidate: ApprovedOcrCandidate): Promise<{ versionId: string } | undefined>; + indexReviewed(candidate: ApprovedOcrCandidate): Promise; + markReady(versionId: string, verifiedPointCount: number): Promise; + settleReusable(candidate: ApprovedOcrCandidate, reusableVersionId: string, activate: boolean): Promise; + activateVersion(sourceId: string, versionId: string, expectedActiveVersionId: string | null): Promise; +} + +export type OcrIndexingResult = { + versionId: string; + state: "ready" | "active" | "rejected"; + activated: boolean; + activatedVersionId?: string; + errorCode?: "DUPLICATE_REUSABLE_VERSION"; +}; + +function activeRace(error: unknown): never { + if (error instanceof CatalogError && error.statusCode === 409) { + throw new CatalogError("Active version changed during OCR approval", 409, "ACTIVE_VERSION_CHANGED"); + } + throw error; +} + +export class OcrIndexingService { + constructor(private readonly store: OcrIndexingStore) {} + + async index(candidate: ApprovedOcrCandidate): Promise { + if (candidate.state !== "indexing") throw new CatalogError("Only approved OCR candidates can be indexed", 409, "INVALID_VERSION_STATE"); + const reusable = await this.store.findReusableVersion(candidate); + if (reusable) { + let activated: boolean; + try { + activated = await this.store.settleReusable(candidate, reusable.versionId, candidate.activateRequested); + } catch (error) { + return activeRace(error); + } + return { + versionId: candidate.versionId, state: "rejected", activated, + ...(activated ? { activatedVersionId: reusable.versionId } : {}), errorCode: "DUPLICATE_REUSABLE_VERSION" + }; + } + + const verifiedPointCount = await this.store.indexReviewed(candidate); + await this.store.markReady(candidate.versionId, verifiedPointCount); + if (!candidate.activateRequested) return { versionId: candidate.versionId, state: "ready", activated: false }; + try { + const activatedVersionId = await this.store.activateVersion(candidate.sourceId, candidate.versionId, candidate.expectedActiveVersionId); + return { versionId: candidate.versionId, state: "active", activated: true, activatedVersionId }; + } catch (error) { + return activeRace(error); + } + } +} diff --git a/src/modules/ocr/retention.ts b/src/modules/ocr/retention.ts new file mode 100644 index 0000000..d97527c --- /dev/null +++ b/src/modules/ocr/retention.ts @@ -0,0 +1,45 @@ +import { rm } from "node:fs/promises"; +import type { SourceVersionState } from "../../shared/types/rag.js"; +import { resolveArtifactPath } from "./artifacts.js"; + +export interface OcrRetentionCandidate { + versionId: string; + sourceId: string; + state: SourceVersionState; + artifactState: "present" | "retention_deleting"; +} + +export interface OcrRetentionStore { + listOcrRetentionCandidates(now: Date): Promise; + withVersionTryLock(versionId: string, handler: () => Promise): Promise; + expireOcrReview(sourceId: string, versionId: string, now: Date): Promise; + claimOcrRetentionDeletion(sourceId: string, versionId: string, state: SourceVersionState, artifactState: OcrRetentionCandidate["artifactState"]): Promise; + completeOcrRetentionDeletion(sourceId: string, versionId: string, state: SourceVersionState): Promise; +} + +export class OcrRetentionService { + constructor(private readonly store: OcrRetentionStore, private readonly artifactRoot: string) {} + + async runOnce(now = new Date()): Promise<{ expired: number; deleted: number; resumed: number; skipped: number }> { + const result = { expired: 0, deleted: 0, resumed: 0, skipped: 0 }; + for (const candidate of await this.store.listOcrRetentionCandidates(now)) { + if (candidate.state === "active") { + result.skipped += 1; + continue; + } + const outcome = await this.store.withVersionTryLock(candidate.versionId, async () => { + if (candidate.state === "review_required") { + return await this.store.expireOcrReview(candidate.sourceId, candidate.versionId, now) ? "expired" : "skipped"; + } + const claimed = await this.store.claimOcrRetentionDeletion(candidate.sourceId, candidate.versionId, candidate.state, candidate.artifactState); + if (!claimed) return "skipped"; + await rm(resolveArtifactPath(this.artifactRoot, candidate.versionId), { recursive: true, force: true }); + return await this.store.completeOcrRetentionDeletion(candidate.sourceId, candidate.versionId, candidate.state) + ? candidate.artifactState === "retention_deleting" ? "resumed" : "deleted" + : "skipped"; + }); + result[outcome ?? "skipped"] += 1; + } + return result; + } +} diff --git a/src/modules/ocr/review.ts b/src/modules/ocr/review.ts new file mode 100644 index 0000000..05a5c9f --- /dev/null +++ b/src/modules/ocr/review.ts @@ -0,0 +1,143 @@ +import { CatalogError } from "../catalog/errors.js"; +import { sha256Hex } from "../../shared/utils/ids.js"; + +export interface OcrReviewLine { + lineId: string; + text: string; + confidence: number; + bbox: [number, number, number, number]; + lineSha256: string; +} + +export interface OcrReviewCandidate { + versionId: string; + sourceId: string; + state: "review_required" | "indexing" | "rejected"; + candidateSha256: string; + baseActiveVersionId: string | null; + currentActiveVersionId: string | null; + activateRequested: boolean; + processingFingerprint: string; + metadataHash: string; + documents: Array<{ documentId: string; pages: Array<{ + page: number; + imageUrl: string; + nativeText: string; + ocr: { text: string; lines: OcrReviewLine[] }; + candidateText: string; + differences: string[]; + risks: string[]; + }> }>; +} + +export interface OcrCorrection { + documentId: string; + page: number; + lineId: string; + expectedLineSha256: string; + replacementText: string; +} + +export interface OcrReviewStore { + loadCandidate(versionId: string): Promise; + commitApproval(input: { + candidate: OcrReviewCandidate; + candidateSha256: string; + expectedActiveVersionId: string | null; + reviewedBy: string; + corrections: OcrCorrection[]; + reviewedText: string; + reviewedTextSha256: string; + }): Promise; + commitRejection(input: { + candidate: OcrReviewCandidate; + candidateSha256: string; + reviewedBy: string; + reason: string; + }): Promise; +} + +export interface ApprovedOcrReview { + versionId: string; + sourceId: string; + state: "indexing"; + activateRequested: boolean; + expectedActiveVersionId: string | null; + reviewedText: string; + reviewedTextSha256: string; + processingFingerprint: string; + metadataHash: string; +} + +function conflict(message: string, code = "REVIEW_CONFLICT"): CatalogError { + return new CatalogError(message, 409, code); +} + +export class OcrReviewService { + constructor(private readonly store: OcrReviewStore) {} + + async view(versionId: string): Promise { + const candidate = await this.store.loadCandidate(versionId); + if (!candidate) throw new CatalogError("OCR review candidate not found", 404, "REVIEW_NOT_FOUND"); + if (candidate.state !== "review_required") throw conflict("Version is not awaiting OCR review", "INVALID_VERSION_STATE"); + return candidate; + } + + async approve(versionId: string, input: { + candidateSha256: string; + expectedActiveVersionId: string | null; + reviewedBy: string; + corrections: OcrCorrection[]; + }): Promise { + const candidate = await this.view(versionId); + if (input.candidateSha256 !== candidate.candidateSha256) throw conflict("Candidate hash is stale"); + if (input.expectedActiveVersionId !== candidate.baseActiveVersionId || input.expectedActiveVersionId !== candidate.currentActiveVersionId) { + throw conflict("Active version changed during review", "ACTIVE_VERSION_CHANGED"); + } + + const reviewedCandidate = structuredClone(candidate); + const lines = new Map(); + for (const document of reviewedCandidate.documents) for (const page of document.pages) for (const line of page.ocr.lines) { + const key = `${document.documentId}:${page.page}:${line.lineId}`; + if (lines.has(key)) throw conflict("Candidate line identities are duplicated", "CORRECTION_CONFLICT"); + lines.set(key, line); + } + const targets = new Set(); + for (const correction of input.corrections) { + const key = `${correction.documentId}:${correction.page}:${correction.lineId}`; + const line = lines.get(key); + if (targets.has(key) || !line || line.lineSha256 !== correction.expectedLineSha256) { + throw conflict("Correction target is stale or duplicated", "CORRECTION_CONFLICT"); + } + targets.add(key); + line.text = correction.replacementText; + line.lineSha256 = sha256Hex(line.text); + } + for (const document of reviewedCandidate.documents) for (const page of document.pages) { + if (page.ocr.lines.length > 0) page.ocr.text = page.candidateText = page.ocr.lines.map(({ text }) => text).join("\n"); + } + const reviewedText = reviewedCandidate.documents.flatMap(({ pages }) => pages.filter(({ candidateText }) => candidateText).map(({ candidateText }) => candidateText)).join("\n\n"); + const reviewedTextSha256 = sha256Hex(reviewedText); + await this.store.commitApproval({ candidate: reviewedCandidate, ...input, reviewedText, reviewedTextSha256 }); + return { + versionId, sourceId: candidate.sourceId, state: "indexing", activateRequested: candidate.activateRequested, + expectedActiveVersionId: input.expectedActiveVersionId, reviewedText, reviewedTextSha256, + processingFingerprint: candidate.processingFingerprint, metadataHash: candidate.metadataHash + }; + } + + async reject(versionId: string, input: { candidateSha256: string; reviewedBy: string; reason: string }): Promise<{ + versionId: string; state: "rejected"; activated: false; + }> { + if (!input || typeof input.candidateSha256 !== "string" || typeof input.reviewedBy !== "string" || typeof input.reason !== "string") { + throw new CatalogError("Candidate hash, reviewer, and rejection reason are required", 400, "INVALID_REJECTION"); + } + const reviewedBy = input.reviewedBy?.trim(); + const reason = input.reason?.trim(); + if (!input.candidateSha256 || !reviewedBy || !reason) throw new CatalogError("Candidate hash, reviewer, and rejection reason are required", 400, "INVALID_REJECTION"); + const candidate = await this.view(versionId); + if (input.candidateSha256 !== candidate.candidateSha256) throw conflict("Candidate hash is stale"); + await this.store.commitRejection({ candidate, candidateSha256: input.candidateSha256, reviewedBy, reason }); + return { versionId, state: "rejected", activated: false }; + } +} diff --git a/tests/ocr/contracts-deploy.test.ts b/tests/ocr/contracts-deploy.test.ts new file mode 100644 index 0000000..e735184 --- /dev/null +++ b/tests/ocr/contracts-deploy.test.ts @@ -0,0 +1,71 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { buildApiHelp, openApiDocument } from "../../src/api/openapi.js"; +import { env } from "../../src/config/env.js"; + +const api = openApiDocument as unknown as { + paths: Record>>; + components: { schemas: Record> }; +}; + +test("OpenAPI describes authenticated OCR ingestion, status, review, approval, and rejection", () => { + const ingestResponses = api.paths["/ingest"]!.post!.responses as Record }>; + const uploadResponses = api.paths["/ingest/upload"]!.post!.responses as typeof ingestResponses; + assert.deepEqual(ingestResponses["202"]!.content!["application/json"]!.schema, { oneOf: [{ $ref: "#/components/schemas/OcrAccepted" }, { $ref: "#/components/schemas/IngestResponse" }] }); + assert.deepEqual(uploadResponses["202"]!.content!["application/json"]!.schema, { oneOf: [{ $ref: "#/components/schemas/OcrUploadAccepted" }, { $ref: "#/components/schemas/UploadIngestResponse" }] }); + + for (const [path, method] of [ + ["/ingestions/{versionId}", "get"], + ["/ingestions/{versionId}/review", "get"], + ["/ingestions/{versionId}/approve", "post"], + ["/ingestions/{versionId}/reject", "post"] + ]) { + const operation = api.paths[path]![method]!; + assert.deepEqual(operation.security, [{ bearerAuth: [] }], `${method.toUpperCase()} ${path} must require bearer auth`); + const responses = operation.responses as Record; + assert.ok("200" in responses && "401" in responses && "404" in responses && "503" in responses); + } + + const approve = api.paths["/ingestions/{versionId}/approve"]!.post!; + const reject = api.paths["/ingestions/{versionId}/reject"]!.post!; + assert.ok("409" in (approve.responses as Record)); + assert.ok("400" in (reject.responses as Record)); + assert.ok("409" in (reject.responses as Record)); + assert.deepEqual((approve.requestBody as { content: Record }).content["application/json"]!.schema, + { $ref: "#/components/schemas/OcrApprovalRequest" }); + assert.deepEqual((reject.requestBody as { content: Record }).content["application/json"]!.schema, + { $ref: "#/components/schemas/OcrRejectionRequest" }); + assert.match(buildApiHelp().authentication, /Bearer.*OCR/u); +}); + +test("OpenAPI OCR schemas preserve progress, review evidence, corrections, and decision states", () => { + const schemas = api.components.schemas; + assert.deepEqual(schemas.OcrPhase!.enum, ["native_extracting", "ocr_queued", "ocr_running", "review_required", "indexing", "ready", "active", "failed", "rejected"]); + assert.deepEqual(schemas.OcrAccepted!.required, ["accepted", "sourceId", "versionId", "versionNumber", "state", "phase", "statusUrl", "reviewUrl", "activated"]); + assert.deepEqual(schemas.IngestionStatus!.required, ["sourceId", "versionId", "state", "phase", "activated", "documents", "error", "statusUrl", "reviewUrl"]); + assert.deepEqual(schemas.OcrReview!.required, ["versionId", "sourceId", "state", "candidateSha256", "baseActiveVersionId", "currentActiveVersionId", "activateRequested", "processingFingerprint", "metadataHash", "documents"]); + const approvalProperties = schemas.OcrApprovalRequest!.properties as Record; + assert.deepEqual(approvalProperties.corrections!.items, { $ref: "#/components/schemas/OcrCorrection" }); + assert.deepEqual(schemas.OcrDecisionResponse!.properties, { + versionId: { type: "string", format: "uuid" }, state: { type: "string", enum: ["ready", "active", "rejected"] }, activated: { type: "boolean" }, + activatedVersionId: { type: "string", format: "uuid" }, errorCode: { type: "string", enum: ["DUPLICATE_REUSABLE_VERSION"] } + }); +}); + +test("OCR deployment defaults and container wiring match the private durable contract", async () => { + assert.deepEqual({ + root: env.ocrArtifactRoot, + uploadBytes: (env as unknown as Record).ocrMaxUploadBytes, + pages: (env as unknown as Record).ocrMaxPages, + pageTimeoutMs: (env as unknown as Record).ocrPageTimeoutMs, + totalTimeoutMs: (env as unknown as Record).ocrTotalTimeoutMs + }, { root: "/data/ingestions", uploadBytes: 50 * 1024 * 1024, pages: 100, pageTimeoutMs: 60_000, totalTimeoutMs: 15 * 60_000 }); + + const dockerfile = await readFile(new URL("../../Dockerfile", import.meta.url), "utf8"); + assert.match(dockerfile, /VOLUME \["\/data\/ingestions"\]/u); + assert.match(dockerfile, /ENV NODE_ENV=production OCR_ARTIFACT_ROOT=\/data\/ingestions/u); + assert.match(dockerfile, /USER node/u); + assert.match(dockerfile, /dist\/modules\/catalog\/migrations\.js.*dist\/server\.js/u); + assert.doesNotMatch(dockerfile, /(?:COPY|ADD)\s+\.env|ENV\s+(?:OCR_INTERNAL_TOKEN|LIFECYCLE_ADMIN_TOKEN)=/u); +}); diff --git a/tests/ocr/dispatcher.test.ts b/tests/ocr/dispatcher.test.ts new file mode 100644 index 0000000..2ce6a54 --- /dev/null +++ b/tests/ocr/dispatcher.test.ts @@ -0,0 +1,397 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { createApp } from "../../src/app.js"; +import { env } from "../../src/config/env.js"; +import { CatalogError } from "../../src/modules/catalog/errors.js"; +import { CatalogRepository } from "../../src/modules/catalog/repository.js"; +import { KnowledgeLifecycleReconciler } from "../../src/modules/catalog/reconciler.js"; +import { OcrDispatcher } from "../../src/modules/ocr/dispatcher.js"; +import { IngestService } from "../../src/modules/ingest/service.js"; +import type { EmbeddingProvider } from "../../src/modules/embeddings/provider.js"; +import type { VectorStoreClient } from "../../src/modules/vectorstore/client.js"; +import { sha256Hex } from "../../src/shared/utils/ids.js"; + +function setEnvFlag(name: keyof typeof env, value: unknown): void { + (env as unknown as Record)[name] = value; +} + +function buildPdf(text: string): Buffer { + const stream = text ? `BT /F1 10 Tf 40 760 Td (${text.replace(/([\\()])/g, "\\$1")}) Tj ET` : ""; + const objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>", + `<< /Length ${Buffer.byteLength(stream)} >>\nstream\n${stream}\nendstream`, + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>" + ]; + let pdf = "%PDF-1.4\n"; + const offsets = [0]; + objects.forEach((object, index) => { + offsets.push(Buffer.byteLength(pdf)); + pdf += `${index + 1} 0 obj\n${object}\nendobj\n`; + }); + const xref = Buffer.byteLength(pdf); + pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; + pdf += offsets.slice(1).map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`).join(""); + pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return Buffer.from(pdf); +} + +function embeddingProvider(): EmbeddingProvider { + return { + providerName: "test-provider", + modelName: "test-model", + dimensions: 3, + async embed(input) { return input.map(() => [0.1, 0.2, 0.3]); } + }; +} + +function vectorStore(): VectorStoreClient { + return { + kind: "fake", + async upsert() {}, + async countVersionPoints() { return 0; } + } as VectorStoreClient; +} + +test("OCR-disabled textual PDFs retain the native synchronous path", async (context) => { + const previousLifecycle = env.knowledgeLifecycleEnforced; + setEnvFlag("knowledgeLifecycleEnforced", true); + context.after(() => setEnvFlag("knowledgeLifecycleEnforced", previousLifecycle)); + const directory = await mkdtemp(path.join(os.tmpdir(), "rag-ocr-disabled-")); + context.after(() => rm(directory, { recursive: true, force: true })); + const filePath = path.join(directory, "native.pdf"); + const sufficient = Array.from({ length: 24 }, (_, index) => `Alpha${index} beta${index}`).join(" "); + await writeFile(filePath, buildPdf(sufficient)); + const calls: string[] = []; + const catalog = { + async beginAttempt() { return "attempt-1"; }, + async updateAttempt() {}, + async withSourceLock(_sourceId: string, handler: () => Promise) { return handler(); }, + async findReusableVersion() { return { versionId: "native-1", versionNumber: 1, state: "active", previousVersionId: null, expectedDocumentCount: 1 }; }, + async assertActiveVersionPrecondition() { calls.push("native"); } + }; + const service = new IngestService(embeddingProvider(), vectorStore(), catalog as never, { + enabled: false, + artifactRoot: directory + }); + + const result = await service.ingest({ sourceType: "file", sourceRef: "native.pdf", readPath: filePath, activate: true, expectedActiveVersionId: null }); + + assert.equal(result.state, "active"); + assert.equal("phase" in result, false); + assert.deepEqual(calls, ["native"]); +}); + +test("OCR-required PDFs are durably accepted once and duplicate pending ingestion reuses status identity", async (context) => { + const previousLifecycle = env.knowledgeLifecycleEnforced; + setEnvFlag("knowledgeLifecycleEnforced", true); + context.after(() => setEnvFlag("knowledgeLifecycleEnforced", previousLifecycle)); + const directory = await mkdtemp(path.join(os.tmpdir(), "rag-ocr-accept-")); + context.after(() => rm(directory, { recursive: true, force: true })); + const filePath = path.join(directory, "scanned.pdf"); + await writeFile(filePath, buildPdf("")); + let createdInput: Record | undefined; + let pending: Record | undefined; + const catalog = { + async beginAttempt() { return "attempt-1"; }, + async updateAttempt() {}, + async withSourceLock(_sourceId: string, handler: () => Promise) { return handler(); }, + async findPendingOcrVersion() { return pending; }, + async createPendingVersion(input: Record) { + createdInput = input; + pending = { version: { versionId: input.versionId, versionNumber: 4, state: "indexing" }, jobs: [{ jobId: "job-1" }] }; + return (pending as { version: unknown }).version; + }, + async markIndexing() {} + }; + const service = new IngestService(embeddingProvider(), vectorStore(), catalog as never, { + enabled: true, + artifactRoot: path.join(directory, "artifacts") + }); + const source = { sourceId: "src:scan", sourceType: "file" as const, sourceRef: "scanned.pdf", readPath: filePath, activate: true, expectedActiveVersionId: null }; + + const accepted = await service.ingest(source); + const duplicate = await service.ingest(source); + + assert.deepEqual(accepted, { + accepted: true, + sourceId: "src:scan", + versionId: accepted.versionId, + versionNumber: 4, + state: "indexing", + phase: "ocr_queued", + statusUrl: `/ingestions/${accepted.versionId}`, + reviewUrl: null, + activated: false + }); + assert.equal(duplicate.versionId, accepted.versionId); + assert.equal((createdInput?.sourceContentHash), null); + assert.deepEqual((createdInput?.ocrJobs as Array<{ requestedPages: number[] }>)[0]?.requestedPages, [1]); + const originalPath = path.join(directory, "artifacts", String(accepted.versionId), "documents"); + assert.equal((await readFile(path.join(directory, "artifacts", String(accepted.versionId), "manifest.json"), "utf8")).includes("scanned.pdf"), true); + assert.equal(path.isAbsolute(originalPath), true); +}); + +test("OCR acceptance retains every supported native document and reloads a concurrent winner", async (context) => { + const previousLifecycle = env.knowledgeLifecycleEnforced; + setEnvFlag("knowledgeLifecycleEnforced", true); + context.after(() => setEnvFlag("knowledgeLifecycleEnforced", previousLifecycle)); + const directory = await mkdtemp(path.join(os.tmpdir(), "rag-ocr-multi-")); + context.after(() => rm(directory, { recursive: true, force: true })); + await writeFile(path.join(directory, "scanned.pdf"), buildPdf("")); + await writeFile(path.join(directory, "notes.txt"), "Native companion document\n"); + const winner = { version: { versionId: "22222222-2222-4222-8222-222222222222", versionNumber: 7, state: "indexing" }, jobs: [{ jobId: "winner-job" }] }; + let lookupCount = 0; + let candidateInput: Record | undefined; + const catalog = { + async beginAttempt() { return "attempt-multi"; }, + async updateAttempt() {}, + async withSourceLock(_sourceId: string, handler: () => Promise) { return handler(); }, + async findPendingOcrVersion() { lookupCount += 1; return lookupCount === 1 ? undefined : winner; }, + async createPendingVersion(input: Record) { + candidateInput = input; + throw Object.assign(new Error("duplicate pending identity"), { code: "23505" }); + } + }; + const service = new IngestService(embeddingProvider(), vectorStore(), catalog as never, { enabled: true, artifactRoot: path.join(directory, "artifacts") }); + + const accepted = await service.ingest({ sourceId: "src:multi", sourceType: "folder", sourceRef: "bundle", readPath: directory, activate: true, expectedActiveVersionId: null }); + + assert.equal(accepted.versionId, winner.version.versionId); + assert.equal(candidateInput?.expectedDocumentCount, 2); + assert.deepEqual((candidateInput?.documents as Array<{ documentKey: string }>).map(({ documentKey }) => documentKey), ["notes.txt", "scanned.pdf"]); + assert.equal((candidateInput?.ocrJobs as unknown[]).length, 1); +}); + +test("application wiring dispatches a newly accepted OCR job from durable artifacts", async (context) => { + const previous = { + lifecycle: env.knowledgeLifecycleEnforced, + enabled: (env as unknown as Record).ocrIngestEnabled, + root: (env as unknown as Record).ocrArtifactRoot + }; + const directory = await mkdtemp(path.join(os.tmpdir(), "rag-ocr-runtime-")); + context.after(() => rm(directory, { recursive: true, force: true })); + setEnvFlag("knowledgeLifecycleEnforced", true); + setEnvFlag("ocrIngestEnabled" as keyof typeof env, true); + setEnvFlag("ocrArtifactRoot" as keyof typeof env, path.join(directory, "artifacts")); + context.after(() => { + setEnvFlag("knowledgeLifecycleEnforced", previous.lifecycle); + setEnvFlag("ocrIngestEnabled" as keyof typeof env, previous.enabled); + setEnvFlag("ocrArtifactRoot" as keyof typeof env, previous.root); + }); + const pdf = buildPdf(""); + let queuedJob: Record | undefined; + let persistedKey: unknown; + const dispatchCalls: string[] = []; + const catalog = { + async beginAttempt() { return "runtime-attempt"; }, async updateAttempt() {}, + async withSourceLock(_sourceId: string, handler: () => Promise) { return handler(); }, + async findPendingOcrVersion() { return undefined; }, + async createPendingVersion(input: Record) { + const job = (input.ocrJobs as Array>)[0]!; + persistedKey = job.remoteIdempotencyKey; + queuedJob = { ...job, jobId: "runtime-job", versionId: input.versionId, remoteJobId: null, state: "queued", completedPages: 0, attemptCount: 0, heartbeatAt: null, leaseExpiresAt: null, nextAttemptAt: null, errorCode: null, errorDetail: null }; + return { versionId: input.versionId, versionNumber: 3 }; + }, + async markIndexing() {}, + async claimNextOcrJob() { const job = queuedJob; queuedJob = undefined; return job ? { ...job, state: "running", attemptCount: 1 } : undefined; }, + async setOcrRemoteJob() {}, + async requeueOcrJob() { dispatchCalls.push("requeued"); }, + async completeOcrJob() { return false; }, async failOcrJob() {}, async markReviewRequired() {}, async markFailed() {} + }; + const client = { + async submit(bytes: Buffer, _expected: unknown, key: string) { assert.equal(key, persistedKey); dispatchCalls.push(`${bytes.length}:${key}`); return { jobId: "runtime-remote" }; }, + async getStatus() { return { jobId: "runtime-remote", status: "queued", completedPages: 0, totalPages: 1, error: null }; } + }; + const server = createApp({ catalog: catalog as never, ocrClient: client as never, startReconciler: false }).listen(0); + context.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address === "object"); + + const form = new FormData(); + form.set("file", new Blob([new Uint8Array(pdf)], { type: "application/pdf" }), "runtime.pdf"); + form.set("sourceId", "src:runtime"); + form.set("activate", "true"); + form.set("expectedActiveVersionId", "null"); + const response = await fetch(`http://127.0.0.1:${address.port}/ingest/upload`, { method: "POST", body: form }); + assert.equal(response.status, 202); + assert.equal((await response.json() as { uploadedResource: string }).uploadedResource, "runtime.pdf"); + for (let attempt = 0; attempt < 20 && !dispatchCalls.includes("requeued"); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(dispatchCalls.length, 2); + assert.match(dispatchCalls[0]!, /^\d+:.+:ocr-v1:.+$/u); + assert.equal(dispatchCalls[1], "requeued"); +}); + +test("dispatcher recovers only expired work, reuses its remote job, and completes it once", async () => { + const calls: string[] = []; + const job = { + jobId: "job-expired", + versionId: "version-1", + documentId: "document-1", + remoteJobId: "remote-1", + remoteIdempotencyKey: "persisted-key", + state: "queued" as const, + requestedPages: [1], + completedPages: 0, + configVersion: "ocr-v1", + attemptCount: 1, + heartbeatAt: null, + leaseExpiresAt: null, + nextAttemptAt: null, + errorCode: null, + errorDetail: null + }; + let available = true; + const repository = { + async recoverExpiredOcrLeases() { calls.push("recover"); return [job]; }, + async claimNextOcrJob() { throw new Error("recovery must not claim unrelated queued work"); }, + async claimOcrJob(jobId: string) { assert.equal(jobId, job.jobId); if (!available) return undefined; available = false; calls.push("claim-exact"); return { ...job, state: "running" as const }; }, + async setOcrRemoteJob() { calls.push("submit-persist"); }, + async requeueOcrJob() { calls.push("requeue"); }, + async completeOcrJob() { calls.push("complete"); return true; }, + async failOcrJob() { calls.push("job-failed"); }, + async markReviewRequired() { calls.push("review-required"); }, + async markFailed() { calls.push("version-failed"); } + }; + const client = { + async submit() { calls.push("submit"); throw new Error("existing remote work must not be duplicated"); }, + async getStatus() { calls.push("status"); return { jobId: "remote-1", status: "succeeded", completedPages: 1, totalPages: 1, error: null }; }, + async getResult() { calls.push("result"); return { pages: [{ page: 1 }] }; } + }; + const dispatcher = new OcrDispatcher(repository, client as never, async () => ({ bytes: Buffer.from("pdf"), documentSha256: sha256Hex("pdf") })); + + assert.equal(await dispatcher.recoverExpiredLeases(), 1); + assert.deepEqual(calls, ["recover", "claim-exact", "status", "result", "complete", "review-required"]); +}); + +test("OCR exhaustion fails the leased candidate without activation or engine substitution", async () => { + const calls: string[] = []; + const repository = { + async claimNextOcrJob() { + return { jobId: "job-1", versionId: "version-1", documentId: "document-1", remoteJobId: null, remoteIdempotencyKey: "same-key", state: "running", requestedPages: [1], completedPages: 0, configVersion: "ocr-v1", attemptCount: 1, heartbeatAt: null, leaseExpiresAt: null, nextAttemptAt: null, errorCode: null, errorDetail: null }; + }, + async setOcrRemoteJob() { calls.push("remote"); }, + async requeueOcrJob() { calls.push("requeue"); }, + async completeOcrJob() { calls.push("complete"); return false; }, + async failOcrJob(_jobId: string, code: string) { calls.push(`job:${code}`); }, + async markReviewRequired() { calls.push("review"); }, + async markFailed(_versionId: string, code: string) { calls.push(`version:${code}`); } + }; + const client = { async submit() { throw new Error("OCR unavailable after retries"); } }; + const dispatcher = new OcrDispatcher(repository as never, client as never, async () => ({ bytes: Buffer.from("pdf"), documentSha256: sha256Hex("pdf") })); + + assert.equal(await dispatcher.runOnce(), "failed"); + assert.deepEqual(calls, ["job:OCR_DISPATCH_FAILED", "version:OCR_DISPATCH_FAILED"]); +}); + +test("reconciler makes expired OCR leases dispatchable while leaving live leases to PostgreSQL", async () => { + const calls: string[] = []; + const catalog = { + async withGlobalTryLock(_name: string, handler: () => Promise) { return handler(); }, + async resolveActiveVersions() { return []; }, + async validateActiveInvariant() { return []; }, + async listOrphanedIndexingCandidates() { return []; } + }; + const dispatcher = { + async recoverExpiredLeases() { calls.push("ocr-recovery"); return 1; }, + async dispatchAvailable() { calls.push("ocr-dispatch"); return 2; } + }; + const reconciler = new KnowledgeLifecycleReconciler(catalog as never, vectorStore(), dispatcher as never); + + const status = await reconciler.runOnce(); + + assert.equal(status.ocrLeasesRecovered, 1); + assert.deepEqual(calls, ["ocr-recovery", "ocr-dispatch"]); +}); + +test("runtime HTTP routing returns native 201, OCR 202/status, and catalog-down 503", async () => { + const previous = { knowledgeLifecycleEnforced: env.knowledgeLifecycleEnforced, lifecycleAdminToken: env.lifecycleAdminToken, ocrIngestEnabled: env.ocrIngestEnabled }; + setEnvFlag("knowledgeLifecycleEnforced", true); + setEnvFlag("lifecycleAdminToken", "unit-8-token"); + setEnvFlag("ocrIngestEnabled", true); + const versionId = "11111111-1111-4111-8111-111111111111"; + const ingestService = { + async ingest(input: Record) { + return input.sourceRef === "native.pdf" + ? { accepted: true, sourceId: "src:native", versionId: "native-1", versionNumber: 1, state: "active", filesDiscovered: 1, documentsProcessed: 1, chunksStored: 1, activated: true, noOp: false, collectionName: "rag" } + : { accepted: true, sourceId: "src:scan", versionId, versionNumber: 2, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false }; + }, + async cleanup() { return { deleted: 0 }; } + }; + const catalog = { + async getIngestionStatus(id: string) { + assert.equal(id, versionId); + return { sourceId: "src:scan", versionId, state: "indexing", phase: "ocr_running", activated: false, documents: [{ documentId: "doc:scan", state: "ocr_running", completedPages: 0, totalPages: 1, pages: [{ page: 1, method: "ocr", state: "ocr_running" }] }], error: null, statusUrl: `/ingestions/${versionId}`, reviewUrl: null }; + } + }; + const app = createApp({ ingestService: ingestService as never, catalog: catalog as never, startReconciler: false }); + const server = app.listen(0); + try { + const address = server.address(); + assert.ok(address && typeof address === "object"); + const baseUrl = `http://127.0.0.1:${address.port}`; + const request = (sourceRef: string) => fetch(`${baseUrl}/ingest`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ sourceType: "file", sourceRef }) }); + + assert.equal((await request("native.pdf")).status, 201); + const accepted = await request("scan.pdf"); + assert.equal(accepted.status, 202); + assert.equal((await accepted.json() as { statusUrl: string }).statusUrl, `/ingestions/${versionId}`); + assert.equal((await fetch(`${baseUrl}/ingestions/${versionId}`)).status, 401); + const status = await fetch(`${baseUrl}/ingestions/${versionId}`, { headers: { authorization: "Bearer unit-8-token" } }); + assert.equal(status.status, 200); + assert.equal((await status.json() as { phase: string }).phase, "ocr_running"); + + const unavailable = createApp({ ingestService: ingestService as never, catalog: undefined, startReconciler: false }).listen(0); + try { + const unavailableAddress = unavailable.address(); + assert.ok(unavailableAddress && typeof unavailableAddress === "object"); + const response = await fetch(`http://127.0.0.1:${unavailableAddress.port}/ingestions/${versionId}`, { headers: { authorization: "Bearer unit-8-token" } }); + assert.equal(response.status, 503); + assert.equal((await response.json() as { code: string }).code, "CATALOG_UNAVAILABLE"); + const retrieval = await fetch(`http://127.0.0.1:${unavailableAddress.port}/retrieve`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ query: "previous active content", mode: "documental", intent: "specific" }) + }); + assert.equal(retrieval.status, 503); + assert.equal((await retrieval.json() as { code: string }).code, "CATALOG_UNAVAILABLE"); + } finally { + unavailable.close(); + } + } finally { + server.close(); + setEnvFlag("knowledgeLifecycleEnforced", previous.knowledgeLifecycleEnforced); + setEnvFlag("lifecycleAdminToken", previous.lifecycleAdminToken); + setEnvFlag("ocrIngestEnabled", previous.ocrIngestEnabled); + } +}); + +test("status reports all native and OCR documents and fails the version when one document fails", async () => { + let query = 0; + const pool = { async query() { + query += 1; + if (query === 1) return { rowCount: 1, rows: [{ source_id: "src:mixed", state: "failed", error_code: "OCR_QUALITY_BLOCKED", error_detail: "Visible ink failed OCR quality" }] }; + if (query === 2) return { rowCount: 2, rows: [ + { document_id: "doc:native", index_state: "indexing", job_state: null, completed_pages: null, requested_pages: null }, + { document_id: "doc:ocr", index_state: "failed", job_state: "failed", completed_pages: 0, requested_pages: [2] } + ] }; + return { rowCount: 3, rows: [ + { document_id: "doc:native", page_number: 1, extraction_method: "native", blocked_reason: null }, + { document_id: "doc:ocr", page_number: 1, extraction_method: "native", blocked_reason: null }, + { document_id: "doc:ocr", page_number: 2, extraction_method: "ocr", blocked_reason: "OCR_QUALITY_BLOCKED" } + ] }; + } }; + const status = await new CatalogRepository(pool as never).getIngestionStatus("version-mixed") as { + phase: string; documents: Array<{ documentId: string; completedPages: number; totalPages: number }>; error: { code: string } + }; + + assert.equal(status.phase, "failed"); + assert.deepEqual(status.documents.map(({ documentId, completedPages, totalPages }) => [documentId, completedPages, totalPages]), [ + ["doc:native", 1, 1], ["doc:ocr", 1, 2] + ]); + assert.equal(status.error.code, "OCR_QUALITY_BLOCKED"); +}); diff --git a/tests/ocr/e2e.test.ts b/tests/ocr/e2e.test.ts new file mode 100644 index 0000000..0a3914a --- /dev/null +++ b/tests/ocr/e2e.test.ts @@ -0,0 +1,183 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createApp } from "../../src/app.js"; +import { env } from "../../src/config/env.js"; +import { OcrDispatcher } from "../../src/modules/ocr/dispatcher.js"; +import { OcrIndexingService } from "../../src/modules/ocr/indexing.js"; +import { OcrReviewService, type OcrReviewCandidate } from "../../src/modules/ocr/review.js"; +import { sha256Hex } from "../../src/shared/utils/ids.js"; + +const token = "local-e2e-token"; +const scanVersion = "11111111-1111-4111-8111-111111111111"; +const mixedVersion = "22222222-2222-4222-8222-222222222222"; + +function headers(json = false): Record { + return { authorization: `Bearer ${token}`, ...(json ? { "content-type": "application/json" } : {}) }; +} + +function candidate(): OcrReviewCandidate { + const text = "CBGO4a"; + return { + versionId: scanVersion, + sourceId: "src:scan", + state: "review_required", + candidateSha256: "candidate-hash", + baseActiveVersionId: "previous-active", + currentActiveVersionId: "previous-active", + activateRequested: true, + processingFingerprint: "fingerprint", + metadataHash: "metadata", + documents: [{ + documentId: "doc:scan", + pages: [{ + page: 1, + imageUrl: `/ingestions/${scanVersion}/documents/doc:scan/pages/1/image`, + nativeText: "", + ocr: { text, lines: [{ lineId: "line-1", text, confidence: 0.72, bbox: [1, 2, 30, 12], lineSha256: sha256Hex(text) }] }, + candidateText: text, + differences: ["native text is empty"], + risks: [text] + }] + }] + }; +} + +test("local HTTP flow keeps native ingestion synchronous and activates only reviewed OCR content", async (context) => { + const previous = { lifecycle: env.knowledgeLifecycleEnforced, enabled: env.ocrIngestEnabled, token: env.lifecycleAdminToken }; + Object.assign(env, { knowledgeLifecycleEnforced: true, ocrIngestEnabled: true, lifecycleAdminToken: token }); + context.after(() => Object.assign(env, { + knowledgeLifecycleEnforced: previous.lifecycle, + ocrIngestEnabled: previous.enabled, + lifecycleAdminToken: previous.token + })); + let reviewCandidate = candidate(); + let reviewedText = ""; + const states = new Map([[scanVersion, "review_required"], [mixedVersion, "review_required"]]); + const ingestService = { + async ingest(input: { sourceRef: string }) { + if (input.sourceRef === "native.pdf") { + return { accepted: true, sourceId: "src:native", versionId: "native-active", versionNumber: 1, state: "active", filesDiscovered: 1, documentsProcessed: 1, chunksStored: 1, activated: true, noOp: false, collectionName: "rag" }; + } + const versionId = input.sourceRef === "mixed.pdf" ? mixedVersion : scanVersion; + return { accepted: true, sourceId: `src:${input.sourceRef}`, versionId, versionNumber: 2, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false }; + }, + async cleanup() { return { deleted: 0 }; } + }; + const catalog = { + async claimNextOcrJob() { return undefined; }, + async getIngestionStatus(versionId: string) { + const state = states.get(versionId); + if (!state) return undefined; + const mixed = versionId === mixedVersion; + return { + sourceId: mixed ? "src:mixed.pdf" : "src:scan.pdf", versionId, state, phase: state, + activated: state === "active", + documents: mixed + ? [{ documentId: "doc:mixed", state: "ocr_complete", completedPages: 2, totalPages: 2, pages: [ + { page: 1, method: "native", state: "native_complete" }, { page: 2, method: "ocr", state: "ocr_complete" } + ] }] + : [{ documentId: "doc:scan", state: "ocr_complete", completedPages: 1, totalPages: 1, pages: [{ page: 1, method: "ocr", state: "ocr_complete" }] }], + error: null, statusUrl: `/ingestions/${versionId}`, reviewUrl: state === "review_required" ? `/ingestions/${versionId}/review` : null + }; + } + }; + const review = new OcrReviewService({ + async loadCandidate(versionId) { return versionId === scanVersion ? reviewCandidate : undefined; }, + async commitApproval(input) { + reviewedText = input.reviewedText; + reviewCandidate = { ...reviewCandidate, state: "indexing" }; + states.set(scanVersion, "indexing"); + }, + async commitRejection() { throw new Error("rejection is outside this flow"); } + }); + const indexing = new OcrIndexingService({ + async findReusableVersion() { return undefined; }, + async indexReviewed(value) { assert.equal(value.reviewedText, "CBG04a"); return 1; }, + async markReady(versionId) { states.set(versionId, "ready"); }, + async settleReusable() { throw new Error("candidate is not reusable"); }, + async activateVersion(_sourceId, versionId, expected) { + assert.equal(expected, "previous-active"); + states.set(versionId, "active"); + return versionId; + } + }); + const server = createApp({ ingestService: ingestService as never, catalog: catalog as never, ocrClient: {} as never, reviewService: review, indexingService: indexing, startReconciler: false }).listen(0); + context.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const base = `http://127.0.0.1:${address.port}`; + const ingest = (sourceRef: string) => fetch(`${base}/ingest`, { method: "POST", headers: headers(true), body: JSON.stringify({ sourceType: "file", sourceRef }) }); + + assert.equal((await ingest("native.pdf")).status, 201); + const accepted = await ingest("scanned.pdf"); + assert.equal(accepted.status, 202); + assert.deepEqual(await accepted.json(), { + accepted: true, sourceId: "src:scanned.pdf", versionId: scanVersion, versionNumber: 2, + state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${scanVersion}`, reviewUrl: null, activated: false + }); + const reviewResponse = await fetch(`${base}/ingestions/${scanVersion}/review`, { headers: headers() }); + assert.equal(reviewResponse.status, 200); + assert.deepEqual((await reviewResponse.json() as OcrReviewCandidate).documents[0]?.pages[0]?.risks, ["CBGO4a"]); + const approval = await fetch(`${base}/ingestions/${scanVersion}/approve`, { + method: "POST", headers: headers(true), body: JSON.stringify({ + candidateSha256: "candidate-hash", expectedActiveVersionId: "previous-active", reviewedBy: "local-reviewer", + corrections: [{ documentId: "doc:scan", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("CBGO4a"), replacementText: "CBG04a" }] + }) + }); + assert.equal(approval.status, 200); + assert.deepEqual(await approval.json(), { versionId: scanVersion, state: "active", activated: true, activatedVersionId: scanVersion }); + assert.equal(reviewedText, "CBG04a"); + assert.equal((await fetch(`${base}/ingestions/${scanVersion}`, { headers: headers() }).then((response) => response.json()) as { state: string }).state, "active"); + + assert.equal((await ingest("mixed.pdf")).status, 202); + const mixed = await fetch(`${base}/ingestions/${mixedVersion}`, { headers: headers() }).then((response) => response.json()) as { documents: Array<{ pages: Array<{ method: string }> }> }; + assert.deepEqual(mixed.documents[0]?.pages.map(({ method }) => method), ["native", "ocr"]); +}); + +test("resends reuse identity, OCR exhaustion fails closed, and catalog absence returns 503", async (context) => { + const previous = { lifecycle: env.knowledgeLifecycleEnforced, enabled: env.ocrIngestEnabled, token: env.lifecycleAdminToken }; + Object.assign(env, { knowledgeLifecycleEnforced: true, ocrIngestEnabled: true, lifecycleAdminToken: token }); + context.after(() => Object.assign(env, { + knowledgeLifecycleEnforced: previous.lifecycle, + ocrIngestEnabled: previous.enabled, + lifecycleAdminToken: previous.token + })); + const identities = new Map(); + let creations = 0; + const ingestService = { + async ingest(input: { sourceRef: string }) { + let versionId = identities.get(input.sourceRef); + if (!versionId) { versionId = input.sourceRef === "scan.pdf" ? scanVersion : mixedVersion; identities.set(input.sourceRef, versionId); creations += 1; } + return { accepted: true, sourceId: `src:${input.sourceRef}`, versionId, versionNumber: 1, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false }; + }, + async cleanup() { return { deleted: 0 }; } + }; + const server = createApp({ ingestService: ingestService as never, catalog: undefined, startReconciler: false }).listen(0); + context.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const base = `http://127.0.0.1:${address.port}`; + const resend = () => fetch(`${base}/ingest`, { method: "POST", headers: headers(true), body: JSON.stringify({ sourceType: "file", sourceRef: "scan.pdf" }) }).then((response) => response.json()) as Promise<{ versionId: string }>; + assert.deepEqual([(await resend()).versionId, (await resend()).versionId], [scanVersion, scanVersion]); + assert.equal(creations, 1); + + const calls: string[] = []; + let candidateState = "indexing"; + const dispatcher = new OcrDispatcher({ + async claimNextOcrJob() { return { jobId: "job", versionId: scanVersion, documentId: "doc", remoteJobId: null, remoteIdempotencyKey: "stable-key", state: "running", requestedPages: [1], completedPages: 0, configVersion: "ocr-v1", attemptCount: 1, heartbeatAt: null, leaseExpiresAt: null, nextAttemptAt: null, errorCode: null, errorDetail: null }; }, + async claimOcrJob() { return undefined; }, async recoverExpiredOcrLeases() { return []; }, async setOcrRemoteJob() { calls.push("remote"); }, + async requeueOcrJob() { calls.push("requeue"); }, async completeOcrJob() { calls.push("complete"); return false; }, + async failOcrJob() { calls.push("job-failed"); }, async markReviewRequired() { calls.push("review"); }, + async markFailed() { candidateState = "failed"; calls.push("version-failed"); } + }, { async submit() { throw new Error("OCR connection unavailable after bounded retries"); } } as never, async () => ({ bytes: Buffer.from("pdf"), documentSha256: sha256Hex("pdf") })); + assert.equal(await dispatcher.runOnce(), "failed"); + assert.deepEqual(calls, ["job-failed", "version-failed"]); + assert.equal(candidateState, "failed"); + + const unavailableStatus = await fetch(`${base}/ingestions/${scanVersion}`, { headers: headers() }); + assert.equal(unavailableStatus.status, 503); + assert.equal((await unavailableStatus.json() as { code: string }).code, "CATALOG_UNAVAILABLE"); + const unavailableRetrieval = await fetch(`${base}/retrieve`, { method: "POST", headers: headers(true), body: JSON.stringify({ query: "prior active", mode: "documental", intent: "specific" }) }); + assert.equal(unavailableRetrieval.status, 503); + assert.equal((await unavailableRetrieval.json() as { code: string }).code, "CATALOG_UNAVAILABLE"); +}); diff --git a/tests/ocr/retention.test.ts b/tests/ocr/retention.test.ts new file mode 100644 index 0000000..61b573c --- /dev/null +++ b/tests/ocr/retention.test.ts @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { access, mkdir, mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { createApp } from "../../src/app.js"; +import { env } from "../../src/config/env.js"; +import { KnowledgeLifecycleReconciler } from "../../src/modules/catalog/reconciler.js"; +import { CatalogRepository } from "../../src/modules/catalog/repository.js"; +import { OcrRetentionService, type OcrRetentionCandidate, type OcrRetentionStore } from "../../src/modules/ocr/retention.js"; + +const activeId = "11111111-1111-4111-8111-111111111111"; +const failedId = "22222222-2222-4222-8222-222222222222"; +const deletingId = "33333333-3333-4333-8333-333333333333"; + +function candidate(versionId: string, state: OcrRetentionCandidate["state"], artifactState: OcrRetentionCandidate["artifactState"] = "present"): OcrRetentionCandidate { + return { versionId, sourceId: `source:${versionId}`, state, artifactState }; +} + +test("repository retention selection applies exact TTLs and CAS-protects active versions", async () => { + const queries: Array<{ sql: string; params?: unknown[] }> = []; + const pool = { async query(sql: string, params?: unknown[]) { + queries.push({ sql, params }); + if (sql.includes("SELECT v.version_id")) return { rowCount: 1, rows: [{ version_id: failedId, source_id: "source:failed", state: "failed", artifact_state: "present" }] }; + return { rowCount: 1, rows: [] }; + } }; + const repository = new CatalogRepository(pool as never); + const now = new Date("2026-09-15T12:00:00.000Z"); + + assert.deepEqual(await repository.listOcrRetentionCandidates(now), [{ ...candidate(failedId, "failed"), sourceId: "source:failed" }]); + assert.equal(await repository.expireOcrReview("source:review", activeId, now), true); + assert.equal(await repository.claimOcrRetentionDeletion("source:failed", failedId, "failed", "present"), true); + assert.equal(await repository.completeOcrRetentionDeletion("source:failed", failedId, "failed"), true); + + assert.match(queries[0]!.sql, /review_required.*30 days.*failed.*rejected.*7 days.*superseded.*30 days/s); + assert.match(queries[0]!.sql, /active_version_id IS DISTINCT FROM v\.version_id/); + assert.deepEqual(queries[0]!.params, [now]); + assert.match(queries[1]!.sql, /state = 'rejected'.*REVIEW_EXPIRED.*retention_due_at/s); + assert.match(queries[2]!.sql, /SET artifact_state = 'retention_deleting'.*state = \$3.*artifact_state = \$4/s); + assert.match(queries[2]!.sql, /active_version_id IS DISTINCT FROM v\.version_id/); + assert.match(queries[3]!.sql, /artifact_state = 'retention_deleted'.*artifact_state = 'retention_deleting'/s); +}); + +test("activation cannot race an artifact deletion already claimed by retention", async () => { + const pool = { + async query(sql: string) { + if (sql.includes("maintenance")) return { rowCount: 1, rows: [{ maintenance: false }] }; + if (sql.includes("FROM rag_sources")) return { rowCount: 1, rows: [{ active_version_id: null }] }; + return { rowCount: 1, rows: [{ version_id: failedId, source_id: "source:failed", version_number: 2, previous_version_id: null, state: "ready", tags: [], source_content_hash: "hash", processing_fingerprint: "fingerprint", metadata_hash: "metadata", embedding_provider: "test", embedding_model: "test", embedding_dimensions: 3, expected_document_count: 1, expected_point_count: 1, verified_point_count: 1, qdrant_collection: "rag", artifact_state: "retention_deleting" }] }; + }, + async connect() { return { query: this.query, release() {} }; } + }; + const repository = new CatalogRepository(pool as never); + + await assert.rejects(repository.activateVersion("source:failed", failedId, null), (error) => error instanceof Error && (error as { code?: string }).code === "VERSION_ARTIFACTS_UNAVAILABLE"); +}); + +test("retention expires review before removal and preserves active artifacts", async (context) => { + const root = await mkdtemp(path.join(os.tmpdir(), "rag-retention-")); + context.after(() => rm(root, { recursive: true, force: true })); + await Promise.all([activeId, failedId].map((id) => mkdir(path.join(root, id)))); + const calls: string[] = []; + const store: OcrRetentionStore = { + async listOcrRetentionCandidates() { return [candidate(activeId, "active"), candidate(failedId, "failed"), candidate(deletingId, "review_required")]; }, + async withVersionTryLock(versionId, handler) { calls.push(`lock:${versionId}`); return handler(); }, + async expireOcrReview(_sourceId, versionId) { calls.push(`expire:${versionId}`); return true; }, + async claimOcrRetentionDeletion(_sourceId, versionId) { calls.push(`claim:${versionId}`); return true; }, + async completeOcrRetentionDeletion(_sourceId, versionId) { calls.push(`complete:${versionId}`); return true; } + }; + + assert.deepEqual(await new OcrRetentionService(store, root).runOnce(), { expired: 1, deleted: 1, resumed: 0, skipped: 1 }); + await access(path.join(root, activeId)); + await assert.rejects(access(path.join(root, failedId))); + assert.equal(calls.some((call) => call.includes(activeId)), false); + assert.equal(calls.includes(`expire:${deletingId}`), true); +}); + +test("interrupted deletion resumes idempotently without affecting another version", async (context) => { + const root = await mkdtemp(path.join(os.tmpdir(), "rag-retention-resume-")); + context.after(() => rm(root, { recursive: true, force: true })); + await Promise.all([deletingId, activeId].map((id) => mkdir(path.join(root, id)))); + let completed = false; + let failCompletion = true; + const store: OcrRetentionStore = { + async listOcrRetentionCandidates() { return completed ? [] : [candidate(deletingId, "rejected", "retention_deleting")]; }, + async withVersionTryLock(_versionId, handler) { return handler(); }, async expireOcrReview() { return false; }, + async claimOcrRetentionDeletion() { return true; }, + async completeOcrRetentionDeletion() { if (failCompletion) { failCompletion = false; throw new Error("simulated restart"); } completed = true; return true; } + }; + const retention = new OcrRetentionService(store, root); + + await assert.rejects(retention.runOnce(), /simulated restart/); + await assert.rejects(access(path.join(root, deletingId))); + assert.deepEqual(await retention.runOnce(), { expired: 0, deleted: 0, resumed: 1, skipped: 0 }); + assert.deepEqual(await retention.runOnce(), { expired: 0, deleted: 0, resumed: 0, skipped: 0 }); + await access(path.join(root, activeId)); +}); + +test("OCR flag off keeps native HTTP synchronous and hides candidate surfaces", async (context) => { + const previous = { enabled: env.ocrIngestEnabled, lifecycle: env.knowledgeLifecycleEnforced, token: env.lifecycleAdminToken }; + Object.assign(env, { ocrIngestEnabled: false, knowledgeLifecycleEnforced: true, lifecycleAdminToken: "retention-token" }); + context.after(() => Object.assign(env, previous)); + let candidateReads = 0; + const native = { accepted: true, sourceId: "source:native", versionId: "native-1", state: "active" }; + const app = createApp({ + ingestService: { async ingest() { return native; }, async cleanup() { return { deleted: 0 }; } } as never, + catalog: { async getIngestionStatus() { candidateReads += 1; return {}; } } as never, + reviewService: { async view() { candidateReads += 1; return {}; }, async approve() { candidateReads += 1; return {} as never; }, async reject() { candidateReads += 1; return {} as never; } }, + indexingService: { async index() { candidateReads += 1; return {} as never; } }, startReconciler: false + }); + const server = app.listen(0); + context.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const base = `http://127.0.0.1:${address.port}`; + + assert.equal((await fetch(`${base}/ingest`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status, 201); + for (const [suffix, method] of [["", "GET"], ["/review", "GET"], ["/approve", "POST"], ["/reject", "POST"]] as const) { + assert.equal((await fetch(`${base}/ingestions/${failedId}${suffix}`, { method, headers: { authorization: "Bearer retention-token", "content-type": "application/json" }, body: method === "POST" ? "{}" : undefined })).status, 404); + } + assert.equal(candidateReads, 0); +}); + +test("reconciler runs retention inside its exclusive global lock", async () => { + let locked = false; + const catalog = { + async withGlobalTryLock(_name: string, handler: () => Promise) { locked = true; try { return await handler(); } finally { locked = false; } }, + async resolveActiveVersions() { return []; }, async validateActiveInvariant() { return []; }, async listOrphanedIndexingCandidates() { return []; } + }; + const retention = { async runOnce() { assert.equal(locked, true); return { expired: 0, deleted: 0, resumed: 0, skipped: 0 }; } }; + const reconciler = new KnowledgeLifecycleReconciler(catalog as never, {} as never, undefined, retention); + + assert.equal((await reconciler.runOnce()).ocrRetentionDeleted, 0); +}); diff --git a/tests/ocr/review.test.ts b/tests/ocr/review.test.ts new file mode 100644 index 0000000..d180e7f --- /dev/null +++ b/tests/ocr/review.test.ts @@ -0,0 +1,185 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createApp } from "../../src/app.js"; +import { env } from "../../src/config/env.js"; +import { CatalogError } from "../../src/modules/catalog/errors.js"; +import { OcrIndexingService, type ApprovedOcrCandidate, type OcrIndexingStore } from "../../src/modules/ocr/indexing.js"; +import { OcrReviewService, type OcrReviewCandidate } from "../../src/modules/ocr/review.js"; +import { sha256Hex } from "../../src/shared/utils/ids.js"; + +function candidate(state: OcrReviewCandidate["state"] = "review_required"): OcrReviewCandidate { + const lines = ["CBGO4a", "FATo7"].map((text, index) => ({ + lineId: `line-${index + 1}`, text, confidence: 0.7, bbox: [0, index * 10, 50, index * 10 + 8] as [number, number, number, number], lineSha256: sha256Hex(text) + })); + return { + versionId: "version-2", sourceId: "source-1", state, candidateSha256: "candidate-hash", + baseActiveVersionId: "version-1", currentActiveVersionId: "version-1", activateRequested: true, + processingFingerprint: "fingerprint", metadataHash: "metadata", documents: [{ documentId: "document-1", pages: [{ + page: 1, imageUrl: "/private/page-1.png", nativeText: "", ocr: { text: "CBGO4a\nFATo7", lines }, + candidateText: "CBGO4a\nFATo7", differences: ["native text is empty"], risks: ["CBGO4a", "FATo7"] + }] }] + }; +} + +test("review HTTP rejects unauthorized and premature access without exposing artifacts", async (context) => { + const previous = { token: env.lifecycleAdminToken, enabled: env.ocrIngestEnabled }; + Object.assign(env, { lifecycleAdminToken: "review-token", ocrIngestEnabled: true }); + context.after(() => Object.assign(env, { lifecycleAdminToken: previous.token, ocrIngestEnabled: previous.enabled })); + let value = candidate(); + let reads = 0; + const review = new OcrReviewService({ async loadCandidate() { reads += 1; return value; }, async commitApproval() { throw new Error("not called"); } }); + const server = createApp({ reviewService: review, startReconciler: false }).listen(0); + context.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const url = `http://127.0.0.1:${address.port}/ingestions/version-2/review`; + + const unauthorized = await fetch(url); + assert.equal(unauthorized.status, 401); + assert.equal((await fetch(url.replace("/review", "/approve"), { method: "POST" })).status, 401); + assert.equal(reads, 0); + value = candidate("indexing"); + const premature = await fetch(url, { headers: { authorization: "Bearer review-token" } }); + assert.equal(premature.status, 409); + assert.deepEqual(await premature.json(), { ok: false, error: "Version is not awaiting OCR review", code: "INVALID_VERSION_STATE" }); +}); + +test("review exposes audit detail and commits current corrections as one immutable set", async () => { + const commits: unknown[] = []; + const service = new OcrReviewService({ async loadCandidate() { return candidate(); }, async commitApproval(value) { commits.push(value); } }); + const review = await service.view("version-2"); + assert.deepEqual(review.documents[0]?.pages[0]?.ocr.lines.map(({ text, confidence, bbox }) => [text, confidence, bbox]), [ + ["CBGO4a", 0.7, [0, 0, 50, 8]], ["FATo7", 0.7, [0, 10, 50, 18]] + ]); + assert.deepEqual(review.documents[0]?.pages[0]?.risks, ["CBGO4a", "FATo7"]); + + const approved = await service.approve("version-2", { + candidateSha256: "candidate-hash", expectedActiveVersionId: "version-1", reviewedBy: "admin", + corrections: [ + { documentId: "document-1", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("CBGO4a"), replacementText: "CBG04a" }, + { documentId: "document-1", page: 1, lineId: "line-2", expectedLineSha256: sha256Hex("FATo7"), replacementText: "FAT07" } + ] + }); + assert.equal(commits.length, 1); + assert.deepEqual((commits[0] as { corrections: Array<{ replacementText: string }> }).corrections.map(({ replacementText }) => replacementText), ["CBG04a", "FAT07"]); + assert.equal(approved.reviewedText, "CBG04a\nFAT07"); + assert.equal(approved.reviewedTextSha256, sha256Hex("CBG04a\nFAT07")); +}); + +test("stale or duplicate corrections conflict before any correction or transition", async () => { + let commits = 0; + const service = new OcrReviewService({ async loadCandidate() { return candidate(); }, async commitApproval() { commits += 1; } }); + const base = { candidateSha256: "stale", expectedActiveVersionId: "version-1", reviewedBy: "admin", corrections: [] }; + await assert.rejects(service.approve("version-2", base), (error) => error instanceof CatalogError && error.statusCode === 409); + const duplicate = { ...base, candidateSha256: "candidate-hash", corrections: Array(2).fill({ + documentId: "document-1", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("CBGO4a"), replacementText: "CBG04a" + }) }; + await assert.rejects(service.approve("version-2", duplicate), (error) => error instanceof CatalogError && error.code === "CORRECTION_CONFLICT"); + await assert.rejects(service.approve("version-2", { ...base, candidateSha256: "candidate-hash", expectedActiveVersionId: "version-old" }), + (error) => error instanceof CatalogError && error.code === "ACTIVE_VERSION_CHANGED"); + await assert.rejects(service.approve("version-2", { ...base, candidateSha256: "candidate-hash", corrections: [{ + documentId: "document-1", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("stale"), replacementText: "CBG04a" + }] }), (error) => error instanceof CatalogError && error.code === "CORRECTION_CONFLICT"); + assert.equal(commits, 0); +}); + +function approved(activateRequested: boolean, state: ApprovedOcrCandidate["state"] = "indexing"): ApprovedOcrCandidate { + return { versionId: "version-2", sourceId: "source-1", state, activateRequested, expectedActiveVersionId: "version-1", reviewedText: "reviewed", reviewedTextSha256: sha256Hex("reviewed"), processingFingerprint: "fingerprint", metadataHash: "metadata" }; +} + +test("indexing activates only when requested and leaves a new candidate ready on an activation race", async () => { + const calls: string[] = []; + const store: OcrIndexingStore = { + async findReusableVersion() { return undefined; }, async indexReviewed() { calls.push("embed"); return 1; }, + async markReady() { calls.push("ready"); }, async settleReusable() { throw new Error("not reusable"); }, + async activateVersion() { calls.push("activate"); return "version-2"; } + }; + const service = new OcrIndexingService(store); + assert.deepEqual(await service.index(approved(true)), { versionId: "version-2", state: "active", activated: true, activatedVersionId: "version-2" }); + assert.deepEqual(calls.splice(0), ["embed", "ready", "activate"]); + assert.deepEqual(await service.index(approved(false)), { versionId: "version-2", state: "ready", activated: false }); + assert.deepEqual(calls.splice(0), ["embed", "ready"]); + store.activateVersion = async () => { throw new CatalogError("race", 409, "ACTIVE_VERSION_PRECONDITION_FAILED"); }; + await assert.rejects(service.index(approved(true)), (error) => error instanceof CatalogError && error.code === "ACTIVE_VERSION_CHANGED"); + assert.deepEqual(calls, ["embed", "ready"]); +}); + +test("reusable and rejected candidates create no embeddings and reusable activation obeys CAS intent", async () => { + const calls: string[] = []; + const store: OcrIndexingStore = { + async findReusableVersion() { return { versionId: "version-existing" }; }, async indexReviewed() { calls.push("embed"); return 1; }, + async markReady() { calls.push("ready"); }, async settleReusable(_candidate, _reusable, activate) { calls.push(`settle:${activate}`); return activate; }, + async activateVersion() { throw new Error("new activation must not run"); } + }; + const service = new OcrIndexingService(store); + assert.deepEqual(await service.index(approved(true)), { versionId: "version-2", state: "rejected", activated: true, activatedVersionId: "version-existing", errorCode: "DUPLICATE_REUSABLE_VERSION" }); + assert.deepEqual(await service.index(approved(false)), { versionId: "version-2", state: "rejected", activated: false, errorCode: "DUPLICATE_REUSABLE_VERSION" }); + store.settleReusable = async () => { throw new CatalogError("race", 409, "ACTIVE_VERSION_PRECONDITION_FAILED"); }; + await assert.rejects(service.index(approved(true)), (error) => error instanceof CatalogError && error.code === "ACTIVE_VERSION_CHANGED"); + await assert.rejects(service.index(approved(true, "rejected")), (error) => error instanceof CatalogError && error.code === "INVALID_VERSION_STATE"); + assert.deepEqual(calls, ["settle:true", "settle:false"]); +}); + +test("authenticated rejection is durable, conflict-safe, and never indexes or activates", async (context) => { + const previous = { token: env.lifecycleAdminToken, enabled: env.ocrIngestEnabled }; + Object.assign(env, { lifecycleAdminToken: "review-token", ocrIngestEnabled: true }); + context.after(() => Object.assign(env, { lifecycleAdminToken: previous.token, ocrIngestEnabled: previous.enabled })); + let value = candidate(); + let reads = 0; + const audits: Array<{ reviewedBy: string; reason: string }> = []; + const review = new OcrReviewService({ + async loadCandidate() { reads += 1; return value; }, + async commitApproval() { throw new Error("approval must not run"); }, + async commitRejection({ reviewedBy, reason }) { + audits.push({ reviewedBy, reason }); + value = { ...value, state: "rejected" }; + } + }); + let indexingCalls = 0; + const catalog = { async getIngestionStatus() { + return { versionId: value.versionId, state: value.state, phase: value.state, activated: false, error: value.state === "rejected" ? { code: "OCR_REJECTED", message: audits[0]?.reason, retryable: false } : null }; + } }; + const server = createApp({ catalog: catalog as never, reviewService: review, indexingService: { async index() { indexingCalls += 1; throw new Error("indexing must not run"); } }, startReconciler: false }).listen(0); + context.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const url = `http://127.0.0.1:${address.port}/ingestions/version-2`; + + const unauthorized = await fetch(`${url}/reject`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ candidateSha256: "candidate-hash", reviewedBy: "admin", reason: "Unreadable code" }) }); + assert.equal(unauthorized.status, 401); + assert.equal(reads, 0); + const invalid = await fetch(`${url}/reject`, { method: "POST", headers: { authorization: "Bearer review-token", "content-type": "application/json" }, body: "{}" }); + assert.equal(invalid.status, 400); + assert.deepEqual(await invalid.json(), { ok: false, error: "Candidate hash, reviewer, and rejection reason are required", code: "INVALID_REJECTION" }); + assert.equal(reads, 0); + const stale = await fetch(`${url}/reject`, { method: "POST", headers: { authorization: "Bearer review-token", "content-type": "application/json" }, body: JSON.stringify({ candidateSha256: "stale", reviewedBy: "admin", reason: "Unreadable code" }) }); + assert.equal(stale.status, 409); + assert.equal(audits.length, 0); + const rejected = await fetch(`${url}/reject`, { method: "POST", headers: { authorization: "Bearer review-token", "content-type": "application/json" }, body: JSON.stringify({ candidateSha256: "candidate-hash", reviewedBy: "admin", reason: "Unreadable code" }) }); + assert.equal(rejected.status, 200); + assert.deepEqual(await rejected.json(), { versionId: "version-2", state: "rejected", activated: false }); + assert.deepEqual(audits, [{ reviewedBy: "admin", reason: "Unreadable code" }]); + const status = await fetch(url, { headers: { authorization: "Bearer review-token" } }); + assert.equal(status.status, 200); + assert.deepEqual(await status.json(), { versionId: "version-2", state: "rejected", phase: "rejected", activated: false, error: { code: "OCR_REJECTED", message: "Unreadable code", retryable: false } }); + const repeated = await fetch(`${url}/reject`, { method: "POST", headers: { authorization: "Bearer review-token", "content-type": "application/json" }, body: JSON.stringify({ candidateSha256: "candidate-hash", reviewedBy: "admin", reason: "Again" }) }); + assert.equal(repeated.status, 409); + assert.equal(indexingCalls, 0); + assert.equal(audits.length, 1); +}); + +test("playground serves the authenticated OCR review controls and audit fields", async (context) => { + const server = createApp({ startReconciler: false }).listen(0); + context.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const base = `http://127.0.0.1:${address.port}`; + const [html, script, styles] = await Promise.all([ + fetch(`${base}/playground`).then((response) => response.text()), + fetch(`${base}/playground/app.js`).then((response) => response.text()), + fetch(`${base}/playground/styles.css`).then((response) => response.text()) + ]); + for (const marker of ["data-tab=\"review\"", "reviewVersionId", "reviewToken", "loadReviewButton", "approveReviewButton", "rejectReviewButton", "reviewCandidate"]) assert.match(html, new RegExp(marker)); + for (const marker of ["Authorization", "/review", "/approve", "/reject", "candidateSha256", "expectedLineSha256", "imageUrl", "nativeText", "confidence", "bbox", "differences", "risks"]) assert.match(script, new RegExp(marker)); + assert.match(styles, /\.review-page/); +});