feat(ocr): complete reviewed ingestion workflow
This commit is contained in:
parent
ac046e37c1
commit
53c4e0a62b
23 changed files with 2484 additions and 79 deletions
|
|
@ -9,11 +9,14 @@ RUN npm run build
|
|||
|
||||
FROM node:22-bookworm-slim AS runtime
|
||||
WORKDIR /app
|
||||
ENV NODE_ENV=production
|
||||
ENV NODE_ENV=production OCR_ARTIFACT_ROOT=/data/ingestions
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci --omit=dev
|
||||
COPY --from=build /app/dist ./dist
|
||||
COPY --from=build /app/migrations ./migrations
|
||||
COPY public ./public
|
||||
RUN install -d -o node -g node -m 700 /data/ingestions
|
||||
VOLUME ["/data/ingestions"]
|
||||
USER node
|
||||
EXPOSE 3000
|
||||
CMD ["node", "dist/server.js"]
|
||||
CMD ["sh", "-c", "node dist/modules/catalog/migrations.js && exec node dist/server.js"]
|
||||
|
|
|
|||
|
|
@ -2,14 +2,64 @@
|
|||
|
||||
**Proyecto:** Workspace de tools IA para empresas
|
||||
**Modulo:** RAG
|
||||
**Ultima actualizacion:** 2026-09-14
|
||||
**Ultima modificacion por:** Subagente OCR Unit 7 Client
|
||||
**Ultima actualizacion:** 2026-09-15
|
||||
**Ultima modificacion por:** Subagente Implement Unit 13 Local E2E
|
||||
**Estado:** Activo
|
||||
|
||||
---
|
||||
|
||||
## Registro de sesion
|
||||
|
||||
### 2026-09-15 - Subagente Implement Unit 13 Local E2E
|
||||
**Agent:** Subagente Implement Unit 13 Local E2E · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5af65977ffehYI4eK2l177YAL`
|
||||
**Responsibility:** Implement only OCR Unit 13 tasks 7.1–7.3 under strict TDD, preserving Units 8–12 and excluding production acceptance task 7.4 and Git delivery work.
|
||||
**Work:** Added deterministic localhost E2E coverage for native, scanned review/approval/activation, mixed documents, resend identity, OCR-down fail-closed behavior, and catalog-down `503` responses. No production code changed.
|
||||
**Validation:** The new requirement tests passed immediately as 2/2 baseline/characterization evidence; canonical Node passed 78/78, offline Python passed 13/13, and check/build/whitespace/cleanup/process gates passed.
|
||||
**Accounting:** 183 functional plus 52 metadata changed lines, 235 total, within the 400-line Unit 13 budget. Tasks 7.1–7.3 are complete; production task 7.4 remains unchecked and untouched.
|
||||
**Files:** `tests/ocr/e2e.test.ts`, OpenSpec tasks/progress, and this history. No external or production service was contacted; `ocr-service/.venv` was preserved.
|
||||
|
||||
---
|
||||
|
||||
### 2026-09-15 - Subagente Implement Unit 12 Contracts Deploy
|
||||
**Agent:** Subagente Implement Unit 12 Contracts Deploy · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b09a3a3ffetN0oeikOd4f33P`
|
||||
**Responsibility:** Implement only OCR Unit 12 tasks 6.3–6.4 under strict TDD, preserving Units 8–11 and excluding all 7.x E2E, production, and Git delivery work.
|
||||
**Work:** Added accurate authenticated OCR ingestion/status/review/correction/decision OpenAPI contracts, durable private OCR deployment defaults, and a non-root migration-aware RAG container with `/data/ingestions` volume wiring.
|
||||
**Validation:** Genuine initial RED was 0/3 and triangulation produced a second RED; focused GREEN passed 3/3, relevant regressions 45/45, canonical Node 76/76, check/build/whitespace gates, sanitized Docker build/inspection, cleanup, and process checks passed.
|
||||
**Accounting:** 219 functional plus 50 metadata changed lines, 269 total, within the 400-line Unit 12 budget. Tasks 6.3–6.4 are complete; all 7.x tasks remain pending.
|
||||
**Files:** `src/api/openapi.ts`, `src/config/env.ts`, `Dockerfile`, `tests/ocr/contracts-deploy.test.ts`, OpenSpec tasks/progress, and this history. The authoritative lifecycle/OCR contract remained read-only.
|
||||
|
||||
---
|
||||
|
||||
### 2026-09-15 - Subagente Implement Unit 11 Retention
|
||||
**Agent:** Subagente Implement Unit 11 Retention · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b1c9293ffeDc3y3lJLgJ8UHF`
|
||||
**Responsibility:** Implement only OCR Unit 11 tasks 6.1–6.2 under strict TDD, preserving Units 8–10 and excluding OpenAPI, deployment, E2E, production, and Git delivery work.
|
||||
**Work:** Added state-aware OCR retention with exact TTL selection, review expiry, active-safe CAS deletion, restart resumption, reconciler execution, and flag-off candidate API isolation while preserving native synchronous ingestion.
|
||||
**Validation:** Genuine missing-module and activation-race RED evidence reached focused GREEN 6/6; relevant regressions passed 28/28; canonical Node passed 73/73; check, build, tracked/untracked whitespace, temporary-directory cleanup, and process checks passed.
|
||||
**Accounting:** 277 functional plus 50 metadata changed lines, 327 total, within the 400-line Unit 11 budget. Tasks 6.1–6.2 are complete; tasks 6.3–6.4 and 7.x remain pending.
|
||||
**Files:** `src/app.ts`, `src/modules/catalog/{repository,reconciler}.ts`, `src/modules/ocr/retention.ts`, `tests/ocr/{dispatcher,review,retention}.test.ts`, OpenSpec tasks/progress, and this history.
|
||||
|
||||
---
|
||||
|
||||
### 2026-09-15 - Subagente Implement Unit 10 Review UI
|
||||
**Agent:** Subagente Implement Unit 10 Review UI · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b3af43bffewaqPFGRUOsi6Qt`
|
||||
**Responsibility:** Implement only OCR Unit 10 task 5.4 under strict TDD, preserving Units 8–9 and excluding retention, deploy, and E2E work.
|
||||
**Work:** Added authenticated, state-safe rejection with durable audit completion and zero indexing/activation side effects. Extended the existing static playground with authenticated candidate inspection, protected images, corrections, approval, and rejection controls.
|
||||
**Validation:** Genuine RED was 5/7; focused GREEN/refactor passed 7/7, the localhost rejection harness passed 1/1, canonical Node passed 67/67, and check/build/whitespace gates passed. No live external dependency or persistent process was used.
|
||||
**Accounting:** 271 functional plus 42 metadata changed lines, 313 total, within the 400-line Unit 10 budget. Task 5.4 is complete; tasks 6.x/7.x remain pending.
|
||||
**Files:** `src/app.ts`, `src/modules/ocr/review.ts`, `tests/ocr/review.test.ts`, `public/playground/{index.html,app.js,styles.css}`, OpenSpec tasks/progress, and this history.
|
||||
|
||||
---
|
||||
|
||||
### 2026-09-15 - Subagente Implement Unit 9 Core
|
||||
**Agent:** Subagente Implement Unit 9 Core · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b4d20b9ffewoOAG5oiIu25fk`
|
||||
**Responsibility:** Implement only OCR Unit 9 tasks 5.1–5.3 under strict TDD, preserving Unit 8 and excluding rejection/UI work.
|
||||
**Work:** Added authenticated review/approval routes, auditable review and atomic correction boundaries, stale-write conflicts, and new/reusable indexing activation CAS behavior. Rejected candidates are refused by indexing without embeddings.
|
||||
**Validation:** RED failed on the missing indexing module; focused GREEN passed 5/5, canonical Node passed 65/65, and check/build/tracked plus untracked whitespace gates passed. No external service or persistent process was used.
|
||||
**Accounting:** 341 functional plus 38 metadata changed lines, 379 total. Unit 10 task 5.4 remains pending.
|
||||
**Files:** `src/app.ts`, `src/modules/ocr/{review,indexing}.ts`, `tests/ocr/review.test.ts`, OpenSpec tasks/progress, and this history.
|
||||
|
||||
---
|
||||
|
||||
### 2026-09-14 - Agente RAG 2 - Diagnostico de resultados vacios de subagentes
|
||||
|
||||
**Modelo:** openai/gpt-5.6-sol
|
||||
|
|
@ -32,6 +82,16 @@
|
|||
|
||||
**Archivos modificados:**
|
||||
- `docs/HISTORIAL_SESIONES.md`
|
||||
|
||||
---
|
||||
### 2026-09-15 - Subagente Recover OCR Unit 8
|
||||
**Agent:** Subagente Recover OCR Unit 8 · **Model:** openai/gpt-5.6-sol · **Session:** `ses_f5b7b4242ffekQxMZTYXOvPWGA`
|
||||
**Responsibility:** Recover only Unit 8 tasks 4.1, 4.4, and 4.5 under strict TDD; no Unit 9 or Git delivery work.
|
||||
**Work:** Completed OCR runtime wiring, exact lease recovery, deterministic pending reuse, mixed-document progress, upload/status HTTP behavior, and fail-closed retrieval evidence.
|
||||
**Validation:** Persisted evidence records focused dispatcher 9/9, focused repository OCR 6/6, canonical Node 60/60, and clean check/build/whitespace gates; curl returned 202 then authenticated 200. The initially leaked harness child was detected and terminated; temporary files and port were rechecked clean.
|
||||
**Evidence chain:** Native failed/interrupted revision `sha256:182cc73954a8518d03bd6c3ef5f4141b352aa9b69fca4177447d76deaddad223` preserved an 815-line candidate. The maintainer-selected `auto-chain` / `feature-branch-chain` preflight reset at revision `sha256:c0db82cf3db8ae790d34e4c310b998f97bb36864c97c3aa84e4c87ab77d07c2a` preserved that candidate and bounded only the additional recovery work. Fresh passed evidence revision: `sha256:e424afe7b2f8efd547dcfd0baf64ec1b46ac344f39afd87259560a094d59163f`.
|
||||
**Accounting:** Relative to committed HEAD `ac046e3`, the complete Unit 8 review slice is 1,045 functional changed lines plus 41 required metadata lines, for 1,086 total; it is not within 400 lines and has no `size:exception`. Only the post-reset recovery delta—350 functional plus 41 metadata lines, 391 total—fits the separate 400-line recovery objective.
|
||||
**Files:** `src/app.ts`, `src/config/env.ts`, `src/modules/{catalog,ingest,ocr}/`, `tests/ocr/dispatcher.test.ts`, OpenSpec tasks/progress, and this history.
|
||||
- `/home/pancho/Documentos/Empresa/IA/herramientas/docs/gentle-ai/SEGUIMIENTO_DESCUBRIMIENTOS_MEJORAS_GENTLE_AI.md`
|
||||
|
||||
---
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@
|
|||
## Current State
|
||||
|
||||
- **Mode:** Strict TDD
|
||||
- **Delivery:** Feature-branch-chain; Units 1–7 implemented; the maintainer approved `size:exception` for Unit 7's cohesive 506 functional authored lines and 585 total native-accounting lines
|
||||
- **Completed tasks:** 1.1, 1.2, 1.3, 1.4, 2.1, 2.2, 2.3, 2.4, 2.5, 3.1, 3.2, 3.3, 3.4, 4.2, 4.3
|
||||
- **Overall task progress:** 15/30 complete
|
||||
- **Delivery:** Feature-branch-chain; Units 1–13 implemented as bounded slices; production acceptance remains pending
|
||||
- **Completed tasks:** 1.1–1.4, 2.1–2.5, 3.1–3.4, 4.1–4.5, 5.1–5.4, 6.1–6.4, 7.1–7.3
|
||||
- **Overall task progress:** 29/30 complete
|
||||
|
||||
## Unit 1: Migration
|
||||
|
||||
|
|
@ -271,3 +271,158 @@ None — the migration follows the proposal, specifications, design, and closed
|
|||
- Corrective work unit `unit-7-size-exception-validation` uses attempt token `sha256:0109cfd8d3b37a5672c65a37700269b89cb80e4e357a39f68a920779a1715c39` and must settle against the failed revision through the parent.
|
||||
- Production and test files remain unchanged for the corrective rerun; only approval/reset evidence and fresh validation metadata may change.
|
||||
- Task 4.1 remains pending, while tasks 4.2 and 4.3 remain complete. Unit 8 was not started.
|
||||
|
||||
## Unit 8: Dispatcher and Runtime Orchestration
|
||||
- Completed multi-document progress, uniqueness-race reuse, exact lease recovery, persisted-key dispatch, default runtime wiring, authenticated status, and fail-closed retrieval; no activation or embeddings occur on OCR failure.
|
||||
|
||||
### Evidence Chain and Review Accounting
|
||||
- **Complete review slice:** Relative to committed HEAD `ac046e3`, Unit 8 contains 1,045 functional changed lines plus 41 required metadata lines, for 1,086 total changed lines. The complete slice is not within 400 lines, and no `size:exception` was approved or recorded for it.
|
||||
- **Interrupted preserved candidate:** Native failed/interrupted revision `sha256:182cc73954a8518d03bd6c3ef5f4141b352aa9b69fca4177447d76deaddad223` contained 815 changed lines.
|
||||
- **Auto-chain reset:** The maintainer-selected preflight strategy was `auto-chain` with `feature-branch-chain`. Native reset revision `sha256:c0db82cf3db8ae790d34e4c310b998f97bb36864c97c3aa84e4c87ab77d07c2a` preserved the interrupted candidate and scoped the next bounded recovery objective only to additional changes; it did not approve the complete Unit 8 slice as a size exception.
|
||||
- **Recovery delta:** After that reset, the bounded recovery added 350 functional changed lines plus 41 required metadata lines, for 391 total changed lines within the separate 400-line recovery objective.
|
||||
- **Passed recovery:** Fresh passed evidence revision `sha256:e424afe7b2f8efd547dcfd0baf64ec1b46ac344f39afd87259560a094d59163f` closes the interrupted → auto-chain reset → passed recovery chain.
|
||||
|
||||
### TDD Cycle Evidence
|
||||
| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 4.1 | `tests/ocr/{dispatcher,client}.test.ts` | HTTP/integration | Recovered baseline 7/7 passed; not labeled RED | New scenarios produced genuine 5/9 RED; catalog `/retrieve` already failed closed | Unit 8 9/9; client 6/6 | 201/202/status, OCR failure, 503 retrieval, retries/integrity | Consolidated persisted-key assertion; 9/9 |
|
||||
| 4.4 | `tests/ocr/dispatcher.test.ts` | Unit/integration | Recovered baseline 7/7 | Exact recovery failed by claiming unrelated work | 9/9 | Live/new queue separation plus periodic dispatch | Exact-job dispatch helper; 9/9 |
|
||||
| 4.5 | `tests/ocr/dispatcher.test.ts` | HTTP/filesystem | Recovered baseline 7/7 | Runtime wiring RED was 8/10 | 9/9 | Multipart upload, durable reload, duplicate race, mixed documents | Shared queue drain; 9/9 |
|
||||
|
||||
### Work Unit Evidence
|
||||
| Evidence | Result |
|
||||
|---|---|
|
||||
| Focused test | `NODE_ENV=test npx --no-install tsx --test tests/ocr/dispatcher.test.ts` — 9 passed, 0 failed. |
|
||||
| Focused repository OCR test | `NODE_ENV=test npx --no-install tsx --test tests/catalog/repository-ocr.test.ts` — 6 passed, 0 failed. |
|
||||
| Canonical Node test | `npm test` — 60 passed, 0 failed. |
|
||||
| Runtime harness | Bounded localhost curl — ingest `202/ocr_queued`; authenticated status `200/ocr_queued`; leaked child from initial shell cleanup was detected, terminated, and port/temp cleanup reverified. |
|
||||
| Rollback boundary | Revert Unit 8 deltas in `src/{app,config/env}.ts`, catalog/ingest/client modules, `src/modules/ocr/dispatcher.ts`, and its test; preserve Units 1–7. |
|
||||
|
||||
## Unit 9: Review and Indexing Core
|
||||
- Added authenticated review/approval routes with injectable service boundaries, immutable correction preparation, stale/duplicate conflict guards, and new/reusable activation CAS behavior. Rejection transition and playground UI remain Unit 10.
|
||||
|
||||
### TDD Cycle Evidence
|
||||
| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 5.1 | `tests/ocr/review.test.ts` | HTTP/unit | App suites 9/9 and 20/20 | Missing `indexing.js`; exit 1 | 5/5 | Unauthorized/premature, candidate/line/active staleness, duplicate targets, new/reusable races, rejected no-index | Assertions remained 5/5 |
|
||||
| 5.2 | `tests/ocr/review.test.ts` | HTTP/unit | 29/29 | Tests preceded `review.ts` | 5/5 | Audit view plus two corrections and four conflict paths | Clone prevents failed commits mutating loaded candidates; 5/5 |
|
||||
| 5.3 | `tests/ocr/review.test.ts` | Unit | N/A (new file) | Tests preceded `indexing.ts` | 5/5 | New/reusable true/false, both CAS races, rejected no-index | Named CAS error mapping; 5/5 |
|
||||
|
||||
### Work Unit Evidence
|
||||
| Evidence | Result |
|
||||
|---|---|
|
||||
| Focused test | `NODE_ENV=test npx --no-install tsx --test tests/ocr/review.test.ts` — exit 0; 5 passed, 0 failed. |
|
||||
| Runtime harness | Same runner with `--test-name-pattern "review HTTP rejects"` — exit 0; 1 passed; real localhost Express requests returned authenticated boundary outcomes without external services. |
|
||||
| Rollback boundary | Remove `src/modules/ocr/{review,indexing}.ts` and `tests/ocr/review.test.ts`; revert only the Unit 9 route additions in `src/app.ts` and Unit 9 metadata. Preserve Units 1–8. |
|
||||
|
||||
### Validation and Boundary
|
||||
- Canonical `NODE_ENV=test npm test` passed 65/65; `npm run check`, `npm run build`, tracked whitespace, and no-index whitespace checks passed.
|
||||
- Unit 9 adds 341 functional changed lines and 38 metadata changed lines, 379 total, within the 400-line budget. No live PostgreSQL, OpenAI, Qdrant, or OCR process was used; no test process remained.
|
||||
- Start: Unit 8 dispatch/status is complete. End: review/correction and indexing/CAS ports plus authenticated routes are green. Unit 10 task 5.4 remains unchecked.
|
||||
|
||||
## Unit 10: Rejection and Review UI
|
||||
- Added authenticated rejection with current candidate-hash and state guards, required audit fields, durable store completion before success, and no indexing or activation call.
|
||||
- Extended the existing static playground with authenticated candidate loading, protected page images, native/raw/candidate comparisons, line confidence/boxes, corrections, risks, approval, and rejection controls.
|
||||
|
||||
### TDD Cycle Evidence
|
||||
| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 5.4 | `tests/ocr/review.test.ts` | HTTP integration/static HTTP | Existing review suite passed 5/5 before edits. | Genuine RED passed only 5/7: rejection returned 404 instead of authenticated 401 and the playground lacked the review contract. A malformed-body edge then returned 409 instead of 400. | Focused suite passed 7/7 after minimal rejection and UI implementation. | Unauthorized, malformed, stale-hash, successful, repeated-state, durable-status, and static audit-field paths exercise distinct outcomes. | Shared UI decision-state helper extracted; focused suite remained 7/7. |
|
||||
|
||||
### Work Unit Evidence
|
||||
| Evidence | Result |
|
||||
|---|---|
|
||||
| Focused test command and exact result | `NODE_ENV=test npx --no-install tsx --test tests/ocr/review.test.ts` — exit 0; 7 passed, 0 failed. |
|
||||
| Runtime harness command/scenario and exact result | `NODE_ENV=test npx --no-install tsx --test --test-name-pattern "authenticated rejection" tests/ocr/review.test.ts` — exit 0; 1 passed, 0 failed. Real localhost Express requests returned 401/400/409/200, persisted the rejected audit state through the injected store, retrieved rejected status, and made zero indexing calls. |
|
||||
| Rollback boundary | Revert Unit 10 deltas in `src/app.ts`, `src/modules/ocr/review.ts`, `tests/ocr/review.test.ts`, and `public/playground/{index.html,app.js,styles.css}` plus Unit 10 metadata; preserve Units 1–9. |
|
||||
|
||||
### Validation and Boundary
|
||||
- Canonical `NODE_ENV=test npm test` passed 67/67; `npm run check` and `npm run build` exited 0.
|
||||
- Previously observed tracked `git diff --check` exited 0; no-index whitespace checks for Unit 10's untracked `src/modules/ocr/review.ts` and `tests/ocr/review.test.ts` exited 0.
|
||||
- Unit 10 adds 271 functional and 42 metadata changed lines, 313 total, below the 400-line budget. No live database, embedding, vector, OCR, or external service was used; no persistent process was started.
|
||||
- Start: Unit 9 review/indexing ports and approval routes are green. End: rejection isolation and the static review UI are green. Retention, deploy, OpenAPI, and E2E tasks remain untouched.
|
||||
- No specification or design deviation.
|
||||
|
||||
## Unit 11: Retention and Flag-Off Isolation
|
||||
|
||||
### Implementation Summary
|
||||
- Added a retention boundary that expires 30-day reviews before deletion, retains failed/rejected artifacts for 7 days, and retains superseded OCR artifacts for 30 days.
|
||||
- Added exact state/artifact CAS claims, active-pointer checks, activation exclusion after a deletion claim, and restart-safe `retention_deleting→retention_deleted` completion.
|
||||
- Wired retention into the existing exclusive reconciler cycle and made status/review/decision APIs return `404` without store access while OCR is disabled; native synchronous ingestion remains unchanged.
|
||||
|
||||
### TDD Cycle Evidence
|
||||
| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 6.1 | `tests/ocr/retention.test.ts` | Repository/HTTP/filesystem integration | Existing repository, dispatcher, and review suites passed 22/22. | Genuine missing-module RED failed before `retention.ts`; activation-race RED then failed 0/1 with “Missing expected rejection.” | Focused suite passed 6/6. | Exact TTL SQL, active pointer/state guards, flag-off `201/404`, and deletion-claim activation conflict exercise independent paths. | No production refactor was needed; two test-side expectations were corrected during GREEN without weakening behavior. |
|
||||
| 6.2 | `tests/ocr/retention.test.ts` | Filesystem/reconciler integration | Same 22/22 safety net. | The missing-module RED preceded all production retention code. | Focused suite passed 6/6; relevant regression set passed 28/28. | Review expiry, terminal deletion, active preservation, simulated restart, repeat no-op, and exclusive reconciler execution cover distinct paths. | Named store boundary keeps filesystem behavior isolated; final focused suite remained 6/6. |
|
||||
|
||||
### Work Unit Evidence
|
||||
| Evidence | Result |
|
||||
|---|---|
|
||||
| Focused test command and exact result | `NODE_ENV=test npx --no-install tsx --test tests/ocr/retention.test.ts` — exit 0; 6 passed, 0 failed. |
|
||||
| Runtime harness command/scenario and exact result | The focused suite used real temporary directories and localhost Express requests: interrupted deletion removed only the claimed directory, resumed to `retention_deleted`, repeated as a no-op, preserved the active directory, returned native `201`, and returned `404` for all disabled OCR candidate APIs without store access. |
|
||||
| Rollback boundary | Remove `src/modules/ocr/retention.ts` and `tests/ocr/retention.test.ts`; revert only Unit 11 seams in `src/app.ts`, catalog repository/reconciler, and explicit enabled-state setup in dispatcher/review tests. Preserve Units 1–10. |
|
||||
|
||||
### Validation and Boundary
|
||||
- Relevant regression suites passed 28/28 after explicitly enabling OCR in pre-existing enabled-path tests; the initial 19/22 run exposed only that test-fixture assumption.
|
||||
- Canonical `NODE_ENV=test npm test` passed 73/73; `npm run check`, `npm run build`, tracked whitespace, and no-index whitespace checks for all seven intended untracked files passed.
|
||||
- Temporary `rag-retention-*` directories were removed and no `tsx --test` or Node test process remained. No live database, OCR, embedding, vector, or external service was used; `ocr-service/.venv` was preserved.
|
||||
- Unit 11 adds 277 functional changed lines and 50 metadata changed lines, 327 total, below the 400-line budget.
|
||||
- Start: Unit 10 rejection/UI is complete. End: tasks 6.1–6.2 are complete. OpenAPI, deployment/configuration, E2E, production, commit, push, and PR work remain untouched.
|
||||
- No specification or design deviation.
|
||||
|
||||
## Unit 12: Contracts and Deployment Wiring
|
||||
|
||||
### Implementation Summary
|
||||
- Documented OCR `202` ingestion variants and authenticated status, review, correction/approval, and rejection routes with behavior-matched success and error responses.
|
||||
- Added progress, review evidence, optimistic correction, decision, and structured error schemas while preserving the legacy-disabled `202` variants.
|
||||
- Added private durable artifact defaults and OCR limits/timeouts; the root image now declares `/data/ingestions`, runs as `node`, and applies catalog migrations before startup without embedding secrets.
|
||||
|
||||
### TDD Cycle Evidence
|
||||
| Task | Test File | Layer | Safety Net | RED | GREEN | TRIANGULATE | REFACTOR |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 6.3 | `tests/ocr/contracts-deploy.test.ts` | OpenAPI contract | Existing lifecycle/dispatcher/review/retention suites passed 42/42. | Genuine initial RED was 0/3: OCR `202` referenced the native schema and OCR routes/schemas were absent. | Focused suite passed 3/3. | A second RED required legacy-disabled `202` variants and accurate help authentication; final suite remained 3/3. | Shared schema references kept nested contracts reviewable; no behavior changed. |
|
||||
| 6.4 | `tests/ocr/contracts-deploy.test.ts` | Config/static container | Same 42/42 safety net. | Initial RED found the old relative artifact root, absent limits/timeouts, volume, non-root user, and migration-aware command. | Focused suite passed 3/3; sanitized Docker build and image inspection passed. | A second RED required explicit runtime artifact-root wiring; final suite remained 3/3. | No further refactor needed. |
|
||||
|
||||
### Work Unit Evidence
|
||||
| Evidence | Result |
|
||||
|---|---|
|
||||
| Focused test command and exact result | `NODE_ENV=test npx --no-install tsx --test tests/ocr/contracts-deploy.test.ts` — exit 0; 3 passed, 0 failed. |
|
||||
| Relevant regression command and exact result | Lifecycle, dispatcher, review, retention, and contract suites — exit 0; 45 passed, 0 failed. |
|
||||
| Runtime harness | N/A — Unit 12 is static-only by task forecast. A sanitized-context Docker build succeeded and inspection proved user `node`, durable volume, production artifact root, and migration-before-server command. |
|
||||
| Rollback boundary | Remove `tests/ocr/contracts-deploy.test.ts` and revert only Unit 12 deltas in `src/api/openapi.ts`, `src/config/env.ts`, and `Dockerfile`; preserve Units 1–11. |
|
||||
|
||||
### Validation and Boundary
|
||||
- Canonical `NODE_ENV=test npm test` passed 76/76; `npm run check`, `npm run build`, tracked whitespace, and all intended-untracked whitespace checks passed.
|
||||
- Docker build `rag-service:unit12-contract` succeeded from a sanitized 430.31 kB context; inspection matched the deployment contract. The image and temporary context were removed, no test process remained, and `ocr-service/.venv` was preserved.
|
||||
- Unit 12 adds 219 functional lines and 50 metadata lines, 269 total, below the 400-line budget.
|
||||
- Start: Unit 11 retention is complete. End: tasks 6.3–6.4 are complete. Unit 13 / all 7.x E2E and production work remains untouched.
|
||||
- `docs/CONTRATO_CICLO_VIDA_Y_OCR.md` was read as authoritative and remained unchanged. No specification or design deviation.
|
||||
|
||||
## Unit 13: Local Deterministic E2E and Complete Gate
|
||||
|
||||
### Implementation Summary
|
||||
- Added a bounded localhost HTTP suite covering native `201`, scanned `202 → review → approve → active`, and exact mixed native/OCR page reporting.
|
||||
- Added resend identity reuse, fail-closed OCR exhaustion, and catalog-unavailable `503` coverage without external services.
|
||||
- Made no production-code changes; the new scenarios passed immediately as baseline/characterization evidence, so no RED was fabricated.
|
||||
|
||||
### TDD Cycle Evidence
|
||||
| Task | Test File | Layer | Safety Net | RED / Baseline | GREEN | TRIANGULATE | REFACTOR |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 7.1 | `tests/ocr/e2e.test.ts` | Local HTTP E2E | Existing OCR suites passed 28/28. | Baseline/characterization: the test was written first and passed 2/2 immediately; no implementation gap existed at the local fake boundary. | Focused suite passed 2/2. | Native, scanned, corrected approval, post-activation status, and mixed page methods exercise distinct paths. | Test-only state naming improved; focused suite remained 2/2. |
|
||||
| 7.2 | `tests/ocr/e2e.test.ts` | Local HTTP/dispatcher E2E | Same 28/28 safety net. | Baseline/characterization: resend, OCR-down, and catalog-down assertions passed immediately; no production code was written. | Focused suite passed 2/2. | Duplicate resend, terminal dispatch failure, status `503`, and retrieval `503` cover independent outcomes. | None needed beyond the shared test harness. |
|
||||
| 7.3 | Gate commands below | Complete gate | Focused E2E passed 2/2. | N/A — verification-only task with no production behavior to implement. | All required gates passed. | Node type/build/test and offline Python tests provide independent stack evidence. | N/A — no production change. |
|
||||
|
||||
### Work Unit Evidence
|
||||
| Evidence | Result |
|
||||
|---|---|
|
||||
| Focused test | `NODE_ENV=test npx --no-install tsx --test tests/ocr/e2e.test.ts` — exit 0; 2 passed, 0 failed. |
|
||||
| Runtime harness | The focused suite started bounded ephemeral localhost Express servers and exercised real HTTP requests through ingest, status, review, approval, and retrieval routes; all servers closed through test cleanup. |
|
||||
| Complete gate | `npm run check`, `npm run build`, and `NODE_ENV=test npm test` exited 0; canonical Node passed 78/78. `PYTHONDONTWRITEBYTECODE=1 ocr-service/.venv/bin/python -m pytest ocr-service/tests -q` passed 13/13 offline with one dependency deprecation warning. |
|
||||
| Rollback boundary | Remove `tests/ocr/e2e.test.ts` and revert only tasks 7.1–7.3 plus this Unit 13 progress/history metadata; preserve Units 1–12. |
|
||||
|
||||
### Validation and Boundary
|
||||
- Unit 13 adds 183 functional lines. Required tasks/progress/history metadata adds 52 changed lines, for 235 total, below the 400-line budget.
|
||||
- Tracked and complete intended-untracked whitespace checks passed. No matching `tsx --test`, E2E Node, or OCR Uvicorn process remained after the corrected self-excluding process check.
|
||||
- Intended untracked inventory: `src/modules/ocr/{dispatcher,indexing,retention,review}.ts`; `tests/ocr/{contracts-deploy,dispatcher,e2e,retention,review}.test.ts`.
|
||||
- Tasks 7.1–7.3 are complete. Task 7.4 remains unchecked and untouched; no production flag, service, database, vector store, embedding provider, OCR endpoint, commit, push, or PR was used.
|
||||
|
|
|
|||
|
|
@ -51,29 +51,29 @@ Tracker feature/ocr-ingest-integration is draft/no-merge and sole main target. P
|
|||
|
||||
## 4 Orchestration (Units 7–8; O1–O4)
|
||||
|
||||
- [ ] 4.1 RED HTTP: flag-off 201; OCR 202/status; OCR-down fail-closed; catalog-down 503; retries/integrity
|
||||
- [x] 4.1 RED HTTP: flag-off 201; OCR 202/status; OCR-down fail-closed; catalog-down 503; retries/integrity
|
||||
- [x] 4.2 GREEN: `src/modules/ocr/client.ts` retries and integrity
|
||||
- [x] 4.3 GREEN: `src/modules/ocr/artifacts.ts` originals, manifest, sweep
|
||||
- [ ] 4.4 GREEN: `src/modules/ocr/dispatcher.ts` leases; `src/modules/catalog/reconciler.ts`
|
||||
- [ ] 4.5 GREEN: `src/modules/ingest/service.ts` branch; `src/app.ts` upload/status
|
||||
- [x] 4.4 GREEN: `src/modules/ocr/dispatcher.ts` leases; `src/modules/catalog/reconciler.ts`
|
||||
- [x] 4.5 GREEN: `src/modules/ingest/service.ts` branch; `src/app.ts` upload/status
|
||||
|
||||
## 5 Review (Units 9–10; V1–V4)
|
||||
|
||||
- [ ] 5.1 RED: auth/premature access; atomic corrections/stale 409; new/reusable activation race; reject/no embeddings
|
||||
- [ ] 5.2 GREEN: `src/modules/ocr/review.ts` view, corrections, conflicts
|
||||
- [ ] 5.3 GREEN: `src/modules/ocr/indexing.ts` CAS; new/reusable activate_requested=true activates, false stays ready
|
||||
- [ ] 5.4 GREEN: rejection and `public/playground/` review UI
|
||||
- [x] 5.1 RED: auth/premature access; atomic corrections/stale 409; new/reusable activation race; reject/no embeddings
|
||||
- [x] 5.2 GREEN: `src/modules/ocr/review.ts` view, corrections, conflicts
|
||||
- [x] 5.3 GREEN: `src/modules/ocr/indexing.ts` CAS; new/reusable activate_requested=true activates, false stays ready
|
||||
- [x] 5.4 GREEN: rejection and `public/playground/` review UI
|
||||
|
||||
## 6 Retention/Deploy (Units 11–12; O5/V5)
|
||||
|
||||
- [ ] 6.1 RED: TTL/CAS/resume/active protection; flag-off native-only and candidate invisibility
|
||||
- [ ] 6.2 GREEN: `src/modules/ocr/retention.ts`
|
||||
- [ ] 6.3 GREEN: `src/api/openapi.ts` OCR/status/review contracts
|
||||
- [ ] 6.4 GREEN: `src/config/env.ts`, `Dockerfile`; follow `docs/CONTRATO_CICLO_VIDA_Y_OCR.md` (read-only)
|
||||
- [x] 6.1 RED: TTL/CAS/resume/active protection; flag-off native-only and candidate invisibility
|
||||
- [x] 6.2 GREEN: `src/modules/ocr/retention.ts`
|
||||
- [x] 6.3 GREEN: `src/api/openapi.ts` OCR/status/review contracts
|
||||
- [x] 6.4 GREEN: `src/config/env.ts`, `Dockerfile`; follow `docs/CONTRATO_CICLO_VIDA_Y_OCR.md` (read-only)
|
||||
|
||||
## 7 E2E (Unit 13; V6)
|
||||
|
||||
- [ ] 7.1 E2E: native 201; scanned 202→review→approve→active; mixed
|
||||
- [ ] 7.2 E2E: resend, OCR-down fail-closed, catalog-down 503
|
||||
- [ ] 7.3 Gate: check, build, test, offline pytest
|
||||
- [x] 7.1 E2E: native 201; scanned 202→review→approve→active; mixed
|
||||
- [x] 7.2 E2E: resend, OCR-down fail-closed, catalog-down 503
|
||||
- [x] 7.3 Gate: check, build, test, offline pytest
|
||||
- [ ] 7.4 Production: flag-off verify→enable→FacturaTech 34 entries; CBG04a/FAT07/DSAU08/NSAV06
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ const manualLogButton = document.getElementById("manualLogButton");
|
|||
const presetDocs = document.getElementById("presetDocs");
|
||||
const presetRagDocs = document.getElementById("presetRagDocs");
|
||||
const presetCode = document.getElementById("presetCode");
|
||||
const loadReviewButton = document.getElementById("loadReviewButton");
|
||||
const approveReviewButton = document.getElementById("approveReviewButton");
|
||||
const rejectReviewButton = document.getElementById("rejectReviewButton");
|
||||
|
||||
const healthResult = document.getElementById("healthResult");
|
||||
const ingestResult = document.getElementById("ingestResult");
|
||||
|
|
@ -24,6 +27,12 @@ const contextStatusText = document.getElementById("contextStatusText");
|
|||
const contextScopeText = document.getElementById("contextScopeText");
|
||||
const logsResult = document.getElementById("logsResult");
|
||||
const logCounterValue = document.getElementById("logCounterValue");
|
||||
const reviewCandidate = document.getElementById("reviewCandidate");
|
||||
const reviewResult = document.getElementById("reviewResult");
|
||||
const reviewVersionId = document.getElementById("reviewVersionId");
|
||||
const reviewToken = document.getElementById("reviewToken");
|
||||
const reviewedBy = document.getElementById("reviewedBy");
|
||||
const rejectionReason = document.getElementById("rejectionReason");
|
||||
|
||||
const ingestSourceType = document.getElementById("ingestSourceType");
|
||||
const ingestScopeMode = document.getElementById("ingestScopeMode");
|
||||
|
|
@ -69,6 +78,8 @@ let lastBootstrapMeta = null;
|
|||
let chatHistory = [];
|
||||
let availableScopes = [];
|
||||
let lastInteraction = null;
|
||||
let loadedReview = null;
|
||||
let reviewImageUrls = [];
|
||||
|
||||
let currentUploadType = null; // 'file' o 'folder'
|
||||
|
||||
|
|
@ -179,6 +190,72 @@ function request(url, payload, method = "POST") {
|
|||
});
|
||||
}
|
||||
|
||||
async function authorizedReviewRequest(path, payload) {
|
||||
const response = await fetch(`/ingestions/${encodeURIComponent(reviewVersionId.value.trim())}${path}`, {
|
||||
method: payload ? "POST" : "GET",
|
||||
headers: { Authorization: `Bearer ${reviewToken.value}`, ...(payload ? { "Content-Type": "application/json" } : {}) },
|
||||
body: payload ? JSON.stringify(payload) : undefined
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(`${data.code || response.status}: ${data.error || "Review request failed"}`);
|
||||
return data;
|
||||
}
|
||||
|
||||
function reviewText(name, value) {
|
||||
const element = document.createElement("pre");
|
||||
element.textContent = `${name}\n${Array.isArray(value) ? value.join("\n") : value || "(empty)"}`;
|
||||
return element;
|
||||
}
|
||||
|
||||
async function renderReview(candidate) {
|
||||
for (const url of reviewImageUrls) URL.revokeObjectURL(url);
|
||||
reviewImageUrls = [];
|
||||
reviewCandidate.replaceChildren();
|
||||
const identity = document.createElement("p");
|
||||
identity.textContent = `Candidate SHA-256: ${candidate.candidateSha256} · Base active version: ${candidate.baseActiveVersionId || "none"}`;
|
||||
reviewCandidate.append(identity);
|
||||
for (const document of candidate.documents) for (const page of document.pages) {
|
||||
const card = document.createElement("article");
|
||||
card.className = "review-page";
|
||||
const title = document.createElement("h3");
|
||||
title.textContent = `${document.documentId} · Page ${page.page}`;
|
||||
const image = document.createElement("img");
|
||||
image.className = "review-image";
|
||||
image.alt = `Source page ${page.page}`;
|
||||
card.append(title, image, reviewText("Native text", page.nativeText), reviewText("Raw OCR", page.ocr.text), reviewText("Candidate text", page.candidateText), reviewText("Differences", page.differences), reviewText("Risks", page.risks));
|
||||
for (const line of page.ocr.lines) {
|
||||
const label = document.createElement("label");
|
||||
label.className = "review-line";
|
||||
const detail = document.createElement("span");
|
||||
detail.textContent = `${line.lineId} · confidence ${line.confidence} · bbox ${line.bbox.join(", ")}`;
|
||||
const input = document.createElement("textarea");
|
||||
input.value = line.text;
|
||||
Object.assign(input.dataset, { original: line.text, documentId: document.documentId, page: String(page.page), lineId: line.lineId, expectedLineSha256: line.lineSha256 });
|
||||
label.append(detail, input);
|
||||
card.append(label);
|
||||
}
|
||||
reviewCandidate.append(card);
|
||||
void fetch(page.imageUrl, { headers: { Authorization: `Bearer ${reviewToken.value}` } }).then(async (response) => {
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
const url = URL.createObjectURL(await response.blob());
|
||||
reviewImageUrls.push(url);
|
||||
image.src = url;
|
||||
}).catch((error) => { image.alt = `Protected image unavailable: ${error}`; });
|
||||
}
|
||||
}
|
||||
|
||||
function reviewDecisionBase() {
|
||||
if (!loadedReview) throw new Error("Load a current candidate first");
|
||||
const reviewer = reviewedBy.value.trim();
|
||||
if (!reviewer) throw new Error("Reviewer identity is required");
|
||||
return { candidateSha256: loadedReview.candidateSha256, reviewedBy: reviewer };
|
||||
}
|
||||
|
||||
function setReviewDecisionEnabled(enabled) {
|
||||
approveReviewButton.disabled = !enabled;
|
||||
rejectReviewButton.disabled = !enabled;
|
||||
}
|
||||
|
||||
function renderBootstrapContext() {
|
||||
if (!lastBootstrapContext) {
|
||||
bootstrapContextResult.textContent = "Aun no hay bootstrap cargado.";
|
||||
|
|
@ -468,6 +545,54 @@ ingestButton.addEventListener("click", async () => {
|
|||
}
|
||||
});
|
||||
|
||||
loadReviewButton.addEventListener("click", async () => {
|
||||
reviewResult.textContent = "Loading authenticated review...";
|
||||
try {
|
||||
loadedReview = await authorizedReviewRequest("/review");
|
||||
await renderReview(loadedReview);
|
||||
setReviewDecisionEnabled(true);
|
||||
reviewResult.textContent = `Loaded review_required version ${loadedReview.versionId}.`;
|
||||
} catch (error) {
|
||||
loadedReview = null;
|
||||
setReviewDecisionEnabled(false);
|
||||
reviewCandidate.textContent = "No candidate loaded.";
|
||||
reviewResult.textContent = String(error);
|
||||
}
|
||||
});
|
||||
|
||||
approveReviewButton.addEventListener("click", async () => {
|
||||
reviewResult.textContent = "Submitting reviewed text...";
|
||||
try {
|
||||
const corrections = [...reviewCandidate.querySelectorAll(".review-line textarea")]
|
||||
.filter((input) => input.value !== input.dataset.original)
|
||||
.map((input) => ({
|
||||
documentId: input.dataset.documentId,
|
||||
page: Number(input.dataset.page),
|
||||
lineId: input.dataset.lineId,
|
||||
expectedLineSha256: input.dataset.expectedLineSha256,
|
||||
replacementText: input.value
|
||||
}));
|
||||
const result = await authorizedReviewRequest("/approve", { ...reviewDecisionBase(), expectedActiveVersionId: loadedReview.baseActiveVersionId, corrections });
|
||||
reviewResult.textContent = format(result);
|
||||
setReviewDecisionEnabled(false);
|
||||
} catch (error) {
|
||||
reviewResult.textContent = String(error);
|
||||
}
|
||||
});
|
||||
|
||||
rejectReviewButton.addEventListener("click", async () => {
|
||||
reviewResult.textContent = "Rejecting candidate...";
|
||||
try {
|
||||
const reason = rejectionReason.value.trim();
|
||||
if (!reason) throw new Error("Rejection reason is required");
|
||||
const result = await authorizedReviewRequest("/reject", { ...reviewDecisionBase(), reason });
|
||||
reviewResult.textContent = format(result);
|
||||
setReviewDecisionEnabled(false);
|
||||
} catch (error) {
|
||||
reviewResult.textContent = String(error);
|
||||
}
|
||||
});
|
||||
|
||||
cleanupButton.addEventListener("click", async () => {
|
||||
if (!cleanupScopeSelect.value) {
|
||||
cleanupResult.textContent = "Error: Debes seleccionar un scope primero.";
|
||||
|
|
|
|||
|
|
@ -29,11 +29,40 @@
|
|||
|
||||
<section class="tabs">
|
||||
<button class="tab-button active" data-tab="ingest">Ingesta</button>
|
||||
<button class="tab-button" data-tab="review">OCR Review</button>
|
||||
<button class="tab-button" data-tab="cleanup">Limpieza</button>
|
||||
<button class="tab-button" data-tab="bootstrap">Bootstrap</button>
|
||||
<button class="tab-button" data-tab="chat">Chat</button>
|
||||
</section>
|
||||
|
||||
<section id="tab-review" class="tab-panel">
|
||||
<article class="panel">
|
||||
<h2>OCR Candidate Review</h2>
|
||||
<p class="helper">Inspect every page and submit an authenticated approval or rejection. OCR content remains unavailable to retrieval until approval completes.</p>
|
||||
<div class="grid single-grid">
|
||||
<label>Version ID
|
||||
<input id="reviewVersionId" autocomplete="off" placeholder="OCR version UUID" />
|
||||
</label>
|
||||
<label>Administrator token
|
||||
<input id="reviewToken" type="password" autocomplete="off" />
|
||||
</label>
|
||||
<label>Reviewed by
|
||||
<input id="reviewedBy" autocomplete="off" placeholder="Reviewer identity" />
|
||||
</label>
|
||||
<label>Rejection reason
|
||||
<textarea id="rejectionReason" rows="2" placeholder="Required only when rejecting"></textarea>
|
||||
</label>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<button id="loadReviewButton">Load candidate</button>
|
||||
<button id="approveReviewButton" disabled>Approve reviewed text</button>
|
||||
<button id="rejectReviewButton" class="danger" disabled>Reject candidate</button>
|
||||
</div>
|
||||
<div id="reviewCandidate" class="review-candidate" aria-live="polite">No candidate loaded.</div>
|
||||
<pre id="reviewResult">No review request submitted.</pre>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<section id="tab-ingest" class="tab-panel active">
|
||||
<article class="panel">
|
||||
<h2>Ingesta</h2>
|
||||
|
|
|
|||
|
|
@ -182,6 +182,23 @@ button.secondary {
|
|||
border: 1px solid var(--border);
|
||||
}
|
||||
|
||||
button.danger { background: var(--danger); color: #0b1020; }
|
||||
button:disabled { cursor: not-allowed; opacity: 0.45; }
|
||||
|
||||
.review-candidate { display: grid; gap: 16px; margin: 20px 0; }
|
||||
.review-page {
|
||||
display: grid;
|
||||
gap: 14px;
|
||||
padding: 18px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 16px;
|
||||
background: #0b1020;
|
||||
}
|
||||
.review-page pre { margin: 0; min-height: 0; }
|
||||
.review-image { max-width: 100%; max-height: 520px; object-fit: contain; justify-self: start; }
|
||||
.review-line { display: grid; gap: 6px; }
|
||||
.review-line span { color: var(--muted); font-size: 12px; }
|
||||
|
||||
.actions, .checkbox {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ export const openApiDocument = {
|
|||
{ name: "Discovery", description: "API contract and browser playground." },
|
||||
{ name: "Status", description: "Service capabilities and available resources." },
|
||||
{ name: "Ingestion", description: "Knowledge ingestion and cleanup." },
|
||||
{ name: "OCR Review", description: "Authenticated OCR progress, review, correction, and decisions." },
|
||||
{ name: "Lifecycle", description: "Knowledge source versions, activation, rollback, and purge." },
|
||||
{ name: "Retrieval", description: "Context retrieval and model-backed answers." },
|
||||
{ name: "Evaluation", description: "Evaluation log capture and review." }
|
||||
|
|
@ -222,7 +223,7 @@ export const openApiDocument = {
|
|||
},
|
||||
responses: {
|
||||
"201": jsonResponse("Lifecycle ingestion completed and committed.", ref("IngestResponse")),
|
||||
"202": jsonResponse("Legacy ingestion completed and accepted while lifecycle enforcement is disabled.", ref("IngestResponse")),
|
||||
"202": jsonResponse("OCR ingestion accepted, or legacy ingestion accepted while lifecycle enforcement is disabled.", { oneOf: [ref("OcrAccepted"), ref("IngestResponse")] }),
|
||||
"400": jsonResponse("Invalid lifecycle activation request.", ref("Error")),
|
||||
"409": jsonResponse("Concurrent active version change or reusable version in progress.", ref("Error")),
|
||||
"422": jsonResponse("Empty or unsupported source.", ref("Error")),
|
||||
|
|
@ -245,7 +246,7 @@ export const openApiDocument = {
|
|||
},
|
||||
responses: {
|
||||
"201": jsonResponse("Lifecycle upload ingested successfully.", ref("UploadIngestResponse")),
|
||||
"202": jsonResponse("Legacy upload ingested successfully while lifecycle enforcement is disabled.", ref("UploadIngestResponse")),
|
||||
"202": jsonResponse("OCR upload accepted, or legacy upload accepted while lifecycle enforcement is disabled.", { oneOf: [ref("OcrUploadAccepted"), ref("UploadIngestResponse")] }),
|
||||
"400": jsonResponse("The file field is missing.", ref("Error"), { ok: false, error: "Missing file upload" }),
|
||||
"409": jsonResponse("Concurrent active version change or reusable version in progress.", ref("Error")),
|
||||
"422": jsonResponse("Empty or unsupported source.", ref("Error")),
|
||||
|
|
@ -254,6 +255,60 @@ export const openApiDocument = {
|
|||
}
|
||||
}
|
||||
},
|
||||
"/ingestions/{versionId}": {
|
||||
get: {
|
||||
tags: ["OCR Review"], summary: "Get OCR ingestion progress", security: [{ bearerAuth: [] }],
|
||||
parameters: [{ name: "versionId", in: "path", required: true, schema: { type: "string", format: "uuid" } }],
|
||||
responses: {
|
||||
"200": jsonResponse("Current document and page progress.", ref("IngestionStatus")),
|
||||
"401": jsonResponse("Missing or invalid lifecycle admin token.", ref("Error")),
|
||||
"404": jsonResponse("OCR is disabled or the ingestion does not exist.", ref("Error")),
|
||||
"503": serverError, "500": serverError
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ingestions/{versionId}/review": {
|
||||
get: {
|
||||
tags: ["OCR Review"], summary: "Inspect an OCR candidate", security: [{ bearerAuth: [] }],
|
||||
parameters: [{ name: "versionId", in: "path", required: true, schema: { type: "string", format: "uuid" } }],
|
||||
responses: {
|
||||
"200": jsonResponse("Review candidate with page evidence.", ref("OcrReview")),
|
||||
"401": jsonResponse("Missing or invalid lifecycle admin token.", ref("Error")),
|
||||
"404": jsonResponse("OCR is disabled or the candidate does not exist.", ref("Error")),
|
||||
"409": jsonResponse("The version is not awaiting review.", ref("Error")),
|
||||
"503": serverError, "500": serverError
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ingestions/{versionId}/approve": {
|
||||
post: {
|
||||
tags: ["OCR Review"], summary: "Correct and approve an OCR candidate", security: [{ bearerAuth: [] }],
|
||||
parameters: [{ name: "versionId", in: "path", required: true, schema: { type: "string", format: "uuid" } }],
|
||||
requestBody: { required: true, content: jsonContent(ref("OcrApprovalRequest")) },
|
||||
responses: {
|
||||
"200": jsonResponse("Candidate indexed and settled according to activation intent.", ref("OcrDecisionResponse")),
|
||||
"401": jsonResponse("Missing or invalid lifecycle admin token.", ref("Error")),
|
||||
"404": jsonResponse("OCR is disabled or the candidate does not exist.", ref("Error")),
|
||||
"409": jsonResponse("Candidate, correction, state, or active-version precondition conflict.", ref("Error")),
|
||||
"503": serverError, "500": serverError
|
||||
}
|
||||
}
|
||||
},
|
||||
"/ingestions/{versionId}/reject": {
|
||||
post: {
|
||||
tags: ["OCR Review"], summary: "Reject an OCR candidate", security: [{ bearerAuth: [] }],
|
||||
parameters: [{ name: "versionId", in: "path", required: true, schema: { type: "string", format: "uuid" } }],
|
||||
requestBody: { required: true, content: jsonContent(ref("OcrRejectionRequest")) },
|
||||
responses: {
|
||||
"200": jsonResponse("Candidate rejected without indexing or activation.", ref("OcrDecisionResponse")),
|
||||
"400": jsonResponse("Required rejection fields are missing.", ref("Error")),
|
||||
"401": jsonResponse("Missing or invalid lifecycle admin token.", ref("Error")),
|
||||
"404": jsonResponse("OCR is disabled or the candidate does not exist.", ref("Error")),
|
||||
"409": jsonResponse("Candidate hash or review state conflict.", ref("Error")),
|
||||
"503": serverError, "500": serverError
|
||||
}
|
||||
}
|
||||
},
|
||||
"/cleanup": {
|
||||
post: {
|
||||
tags: ["Ingestion"],
|
||||
|
|
@ -402,7 +457,8 @@ export const openApiDocument = {
|
|||
required: ["ok", "error"],
|
||||
properties: {
|
||||
ok: { type: "boolean", const: false },
|
||||
error: { type: "string" }
|
||||
error: { type: "string" },
|
||||
code: { type: "string" }
|
||||
}
|
||||
},
|
||||
Scope: {
|
||||
|
|
@ -490,6 +546,77 @@ export const openApiDocument = {
|
|||
}
|
||||
]
|
||||
},
|
||||
OcrPhase: {
|
||||
type: "string",
|
||||
enum: ["native_extracting", "ocr_queued", "ocr_running", "review_required", "indexing", "ready", "active", "failed", "rejected"]
|
||||
},
|
||||
OcrAccepted: {
|
||||
type: "object",
|
||||
required: ["accepted", "sourceId", "versionId", "versionNumber", "state", "phase", "statusUrl", "reviewUrl", "activated"],
|
||||
properties: {
|
||||
accepted: { type: "boolean", const: true }, sourceId: { type: "string" }, versionId: { type: "string", format: "uuid" },
|
||||
versionNumber: { type: "integer" }, state: { type: "string", const: "indexing" }, phase: { type: "string", const: "ocr_queued" },
|
||||
statusUrl: { type: "string" }, reviewUrl: { type: "null" }, activated: { type: "boolean", const: false }
|
||||
}
|
||||
},
|
||||
OcrUploadAccepted: {
|
||||
allOf: [ref("OcrAccepted"), { type: "object", required: ["uploadedResource"], properties: { uploadedResource: { type: "string" } } }]
|
||||
},
|
||||
IngestionStatus: {
|
||||
type: "object",
|
||||
required: ["sourceId", "versionId", "state", "phase", "activated", "documents", "error", "statusUrl", "reviewUrl"],
|
||||
properties: {
|
||||
sourceId: { type: "string" }, versionId: { type: "string", format: "uuid" }, state: ref("SourceVersionState"), phase: ref("OcrPhase"), activated: { type: "boolean" },
|
||||
documents: { type: "array", items: ref("OcrProgressDocument") },
|
||||
error: { oneOf: [ref("OcrStatusError"), { type: "null" }] }, statusUrl: { type: "string" }, reviewUrl: { type: ["string", "null"] }
|
||||
}
|
||||
},
|
||||
OcrProgressDocument: {
|
||||
type: "object", required: ["documentId", "state", "completedPages", "totalPages", "pages"],
|
||||
properties: {
|
||||
documentId: { type: "string" }, state: { type: "string", enum: ["native_complete", "ocr_queued", "ocr_running", "ocr_complete", "failed"] },
|
||||
completedPages: { type: "integer" }, totalPages: { type: "integer" }, pages: { type: "array", items: ref("OcrProgressPage") }
|
||||
}
|
||||
},
|
||||
OcrProgressPage: {
|
||||
type: "object", required: ["page", "method", "state"],
|
||||
properties: { page: { type: "integer" }, method: { type: "string", enum: ["native", "ocr", "blank"] }, state: { type: "string", enum: ["pending", "native_complete", "ocr_queued", "ocr_running", "ocr_complete", "blank", "failed"] }, errorCode: { type: "string" } }
|
||||
},
|
||||
OcrStatusError: { type: "object", required: ["code", "message", "retryable"], properties: { code: { type: "string" }, message: { type: "string" }, retryable: { type: "boolean" } } },
|
||||
OcrLine: {
|
||||
type: "object", required: ["lineId", "text", "confidence", "bbox", "lineSha256"],
|
||||
properties: { lineId: { type: "string" }, text: { type: "string" }, confidence: { type: "number" }, bbox: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4 }, lineSha256: { type: "string", pattern: "^[a-f0-9]{64}$" } }
|
||||
},
|
||||
OcrReview: {
|
||||
type: "object",
|
||||
required: ["versionId", "sourceId", "state", "candidateSha256", "baseActiveVersionId", "currentActiveVersionId", "activateRequested", "processingFingerprint", "metadataHash", "documents"],
|
||||
properties: {
|
||||
versionId: { type: "string", format: "uuid" }, sourceId: { type: "string" }, state: { type: "string", const: "review_required" }, candidateSha256: { type: "string", pattern: "^[a-f0-9]{64}$" },
|
||||
baseActiveVersionId: { type: ["string", "null"], format: "uuid" }, currentActiveVersionId: { type: ["string", "null"], format: "uuid" }, activateRequested: { type: "boolean" }, processingFingerprint: { type: "string" }, metadataHash: { type: "string" },
|
||||
documents: { type: "array", items: ref("OcrReviewDocument") }
|
||||
}
|
||||
},
|
||||
OcrReviewDocument: { type: "object", required: ["documentId", "pages"], properties: { documentId: { type: "string" }, pages: { type: "array", items: ref("OcrReviewPage") } } },
|
||||
OcrReviewPage: {
|
||||
type: "object", required: ["page", "imageUrl", "nativeText", "ocr", "candidateText", "differences", "risks"],
|
||||
properties: { page: { type: "integer" }, imageUrl: { type: "string" }, nativeText: { type: "string" }, ocr: { type: "object", required: ["text", "lines"], properties: { text: { type: "string" }, lines: { type: "array", items: ref("OcrLine") } } }, candidateText: { type: "string" }, differences: { type: "array", items: { type: "string" } }, risks: { type: "array", items: { type: "string" } } }
|
||||
},
|
||||
OcrCorrection: {
|
||||
type: "object", required: ["documentId", "page", "lineId", "expectedLineSha256", "replacementText"],
|
||||
properties: { documentId: { type: "string" }, page: { type: "integer" }, lineId: { type: "string" }, expectedLineSha256: { type: "string", pattern: "^[a-f0-9]{64}$" }, replacementText: { type: "string" } }
|
||||
},
|
||||
OcrApprovalRequest: {
|
||||
type: "object", required: ["candidateSha256", "expectedActiveVersionId", "reviewedBy", "corrections"],
|
||||
properties: { candidateSha256: { type: "string", pattern: "^[a-f0-9]{64}$" }, expectedActiveVersionId: { type: ["string", "null"], format: "uuid" }, reviewedBy: { type: "string" }, corrections: { type: "array", items: ref("OcrCorrection") } }
|
||||
},
|
||||
OcrRejectionRequest: {
|
||||
type: "object", required: ["candidateSha256", "reviewedBy", "reason"],
|
||||
properties: { candidateSha256: { type: "string", pattern: "^[a-f0-9]{64}$" }, reviewedBy: { type: "string" }, reason: { type: "string" } }
|
||||
},
|
||||
OcrDecisionResponse: {
|
||||
type: "object", required: ["versionId", "state", "activated"],
|
||||
properties: { versionId: { type: "string", format: "uuid" }, state: { type: "string", enum: ["ready", "active", "rejected"] }, activated: { type: "boolean" }, activatedVersionId: { type: "string", format: "uuid" }, errorCode: { type: "string", enum: ["DUPLICATE_REUSABLE_VERSION"] } }
|
||||
},
|
||||
CleanupRequest: {
|
||||
type: "object",
|
||||
required: ["scope"],
|
||||
|
|
@ -877,7 +1004,7 @@ export function buildApiHelp() {
|
|||
openapiPurpose: "Consult this contract for complete parameters, request bodies, responses, errors, and examples for every endpoint.",
|
||||
playground: "/playground"
|
||||
},
|
||||
authentication: "None. The current API is publicly accessible; authentication is planned separately.",
|
||||
authentication: "Bearer authentication is required for lifecycle administration and all OCR candidate routes; other routes remain public.",
|
||||
endpoints
|
||||
};
|
||||
}
|
||||
|
|
|
|||
136
src/app.ts
136
src/app.ts
|
|
@ -1,10 +1,10 @@
|
|||
import express from "express";
|
||||
import multer from "multer";
|
||||
import type { Request } from "express";
|
||||
import { writeFile, unlink, mkdtemp, rm } from "node:fs/promises";
|
||||
import { readFile, unlink, mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { timingSafeEqual } from "node:crypto";
|
||||
import { randomUUID, timingSafeEqual } from "node:crypto";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import AdmZip from "adm-zip";
|
||||
import { buildApiHelp, openApiDocument } from "./api/openapi.js";
|
||||
|
|
@ -20,28 +20,66 @@ import { QdrantVectorStoreClient } from "./modules/vectorstore/client.js";
|
|||
import { getCatalogPool } from "./modules/catalog/client.js";
|
||||
import { KnowledgeLifecycleReconciler } from "./modules/catalog/reconciler.js";
|
||||
import { CatalogError, CatalogRepository } from "./modules/catalog/repository.js";
|
||||
import { OcrClient } from "./modules/ocr/client.js";
|
||||
import { OcrDispatcher } from "./modules/ocr/dispatcher.js";
|
||||
import { resolveArtifactPath } from "./modules/ocr/artifacts.js";
|
||||
import type { OcrReviewService } from "./modules/ocr/review.js";
|
||||
import type { OcrIndexingService } from "./modules/ocr/indexing.js";
|
||||
import { OcrRetentionService } from "./modules/ocr/retention.js";
|
||||
import type { ChatMessage, ChunkMode, RetrieveIntent, RetrieveScope } from "./shared/types/rag.js";
|
||||
import { sha256Hex } from "./shared/utils/ids.js";
|
||||
|
||||
type UploadRequest = Request & {
|
||||
file?: Express.Multer.File;
|
||||
};
|
||||
|
||||
export function createApp() {
|
||||
interface AppOptions {
|
||||
ingestService?: Pick<IngestService, "ingest" | "cleanup">;
|
||||
catalog?: CatalogRepository;
|
||||
ocrClient?: OcrClient;
|
||||
reviewService?: Pick<OcrReviewService, "view" | "approve" | "reject">;
|
||||
indexingService?: Pick<OcrIndexingService, "index">;
|
||||
startReconciler?: boolean;
|
||||
}
|
||||
|
||||
export function createApp(options: AppOptions = {}) {
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const publicDir = path.resolve(__dirname, "../public");
|
||||
const app = express();
|
||||
const upload = multer({ storage: multer.memoryStorage() });
|
||||
const upload = multer({
|
||||
storage: multer.diskStorage({
|
||||
destination: os.tmpdir(),
|
||||
filename: (_request, file, callback) => callback(null, `${randomUUID()}${path.extname(file.originalname).toLowerCase()}`)
|
||||
}),
|
||||
limits: { fileSize: 50 * 1024 * 1024 }
|
||||
});
|
||||
const embeddingProvider = new OpenRouterEmbeddingProvider();
|
||||
const vectorStore = new QdrantVectorStoreClient();
|
||||
const catalogPool = getCatalogPool();
|
||||
const catalog = catalogPool ? new CatalogRepository(catalogPool) : undefined;
|
||||
const reconciler = new KnowledgeLifecycleReconciler(catalog, vectorStore);
|
||||
const catalog = Object.prototype.hasOwnProperty.call(options, "catalog")
|
||||
? options.catalog
|
||||
: catalogPool ? new CatalogRepository(catalogPool) : undefined;
|
||||
const ocr = { enabled: env.ocrIngestEnabled, artifactRoot: path.resolve(env.ocrArtifactRoot) };
|
||||
const ocrClient = ocr.enabled ? options.ocrClient ?? new OcrClient({ baseUrl: env.ocrServiceUrl, token: env.ocrInternalToken }) : undefined;
|
||||
const ocrDispatcher = catalog && ocrClient ? new OcrDispatcher(catalog, ocrClient, async (job) => {
|
||||
const versionDirectory = path.join(ocr.artifactRoot, job.versionId);
|
||||
const manifest = JSON.parse(await readFile(path.join(versionDirectory, "manifest.json"), "utf8")) as {
|
||||
documents?: Array<{ documentId: string; originalPath: string; originalSha256: string }>;
|
||||
};
|
||||
const document = manifest.documents?.find(({ documentId }) => documentId === job.documentId);
|
||||
if (!document) throw new Error("OCR artifact manifest does not contain the queued document");
|
||||
const bytes = await readFile(resolveArtifactPath(versionDirectory, document.originalPath));
|
||||
if (sha256Hex(bytes) !== document.originalSha256) throw new Error("OCR artifact integrity validation failed");
|
||||
return { bytes, documentSha256: document.originalSha256 };
|
||||
}) : undefined;
|
||||
const retention = catalog ? new OcrRetentionService(catalog, ocr.artifactRoot) : undefined;
|
||||
const reconciler = new KnowledgeLifecycleReconciler(catalog, vectorStore, ocrDispatcher, retention);
|
||||
const evaluationLogs = new EvaluationLogService(embeddingProvider);
|
||||
const ingestService = new IngestService(embeddingProvider, vectorStore, catalog);
|
||||
const ingestService = options.ingestService ?? new IngestService(embeddingProvider, vectorStore, catalog, ocr);
|
||||
const retrieveService = new RetrieveService(embeddingProvider, vectorStore, catalog);
|
||||
const answerService = new AnswerService(retrieveService);
|
||||
reconciler.start();
|
||||
if (options.startReconciler !== false) reconciler.start();
|
||||
|
||||
function sendError(res: express.Response, error: unknown, fallback: string) {
|
||||
const upstreamUnavailable = error instanceof Error && /(ECONNREFUSED|ETIMEDOUT|timeout|connection|database|postgres|qdrant)/i.test(error.message);
|
||||
|
|
@ -54,6 +92,10 @@ export function createApp() {
|
|||
res.status(statusCode).json(body);
|
||||
}
|
||||
|
||||
function dispatchAcceptedOcr(result: Awaited<ReturnType<IngestService["ingest"]>>): void {
|
||||
if ("phase" in result) void ocrDispatcher?.dispatchAvailable();
|
||||
}
|
||||
|
||||
function requireLifecycleAdmin(req: express.Request, res: express.Response): boolean {
|
||||
const header = req.header("authorization") ?? "";
|
||||
const token = header.startsWith("Bearer ") ? header.slice("Bearer ".length) : "";
|
||||
|
|
@ -71,6 +113,12 @@ export function createApp() {
|
|||
return true;
|
||||
}
|
||||
|
||||
function requireOcrEnabled(res: express.Response): boolean {
|
||||
if (ocr.enabled) return true;
|
||||
res.status(404).json({ ok: false, error: "Not found" });
|
||||
return false;
|
||||
}
|
||||
|
||||
function requireCatalog(res: express.Response): CatalogRepository | undefined {
|
||||
if (!catalog) {
|
||||
res.status(503).json({ ok: false, error: "Knowledge catalog is not configured", code: "CATALOG_UNAVAILABLE" });
|
||||
|
|
@ -201,6 +249,66 @@ export function createApp() {
|
|||
}
|
||||
});
|
||||
|
||||
app.get("/ingestions/:versionId", async (req, res) => {
|
||||
if (!requireOcrEnabled(res)) return;
|
||||
if (!requireLifecycleAdmin(req, res)) return;
|
||||
const repository = requireCatalog(res);
|
||||
if (!repository) return;
|
||||
try {
|
||||
const status = await repository.getIngestionStatus(String(req.params.versionId));
|
||||
if (!status) {
|
||||
res.status(404).json({ ok: false, error: "Ingestion not found", code: "INGESTION_NOT_FOUND" });
|
||||
return;
|
||||
}
|
||||
res.json(status);
|
||||
} catch (error) {
|
||||
sendError(res, error, "Unknown ingestion status error");
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/ingestions/:versionId/review", async (req, res) => {
|
||||
if (!requireOcrEnabled(res)) return;
|
||||
if (!requireLifecycleAdmin(req, res)) return;
|
||||
if (!options.reviewService) {
|
||||
res.status(503).json({ ok: false, error: "OCR review service is not configured", code: "OCR_REVIEW_UNAVAILABLE" });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
res.json(await options.reviewService.view(String(req.params.versionId)));
|
||||
} catch (error) {
|
||||
sendError(res, error, "Unknown OCR review error");
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/ingestions/:versionId/approve", async (req, res) => {
|
||||
if (!requireOcrEnabled(res)) return;
|
||||
if (!requireLifecycleAdmin(req, res)) return;
|
||||
if (!options.reviewService || !options.indexingService) {
|
||||
res.status(503).json({ ok: false, error: "OCR approval service is not configured", code: "OCR_REVIEW_UNAVAILABLE" });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const approved = await options.reviewService.approve(String(req.params.versionId), req.body);
|
||||
res.json(await options.indexingService.index(approved));
|
||||
} catch (error) {
|
||||
sendError(res, error, "Unknown OCR approval error");
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/ingestions/:versionId/reject", async (req, res) => {
|
||||
if (!requireOcrEnabled(res)) return;
|
||||
if (!requireLifecycleAdmin(req, res)) return;
|
||||
if (!options.reviewService) {
|
||||
res.status(503).json({ ok: false, error: "OCR review service is not configured", code: "OCR_REVIEW_UNAVAILABLE" });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
res.json(await options.reviewService.reject(String(req.params.versionId), req.body));
|
||||
} catch (error) {
|
||||
sendError(res, error, "Unknown OCR rejection error");
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/models/answer", async (_req, res) => {
|
||||
try {
|
||||
const models = await answerService.listAvailableAnswerModels();
|
||||
|
|
@ -226,7 +334,8 @@ export function createApp() {
|
|||
app.post("/ingest", async (req, res) => {
|
||||
try {
|
||||
const result = await ingestService.ingest(req.body);
|
||||
res.status(env.knowledgeLifecycleEnforced ? 201 : 202).json(result);
|
||||
dispatchAcceptedOcr(result);
|
||||
res.status("phase" in result ? 202 : env.knowledgeLifecycleEnforced ? 201 : 202).json(result);
|
||||
} catch (error) {
|
||||
sendError(res, error, "Unknown ingest error");
|
||||
}
|
||||
|
|
@ -322,7 +431,8 @@ export function createApp() {
|
|||
activate: req.body.activate === undefined ? true : Boolean(req.body.activate),
|
||||
expectedActiveVersionId: req.body.expectedActiveVersionId === undefined ? null : req.body.expectedActiveVersionId
|
||||
});
|
||||
res.status(201).json(result);
|
||||
dispatchAcceptedOcr(result);
|
||||
res.status("phase" in result ? 202 : 201).json(result);
|
||||
return;
|
||||
}
|
||||
res.status(202).json({
|
||||
|
|
@ -371,8 +481,7 @@ export function createApp() {
|
|||
|
||||
const isZipFolder = req.body.isZipFolder === "true";
|
||||
const tempDirBase = await os.tmpdir();
|
||||
tempFilePath = path.join(tempDirBase, `${Date.now()}-${req.file.originalname}`);
|
||||
await writeFile(tempFilePath, req.file.buffer);
|
||||
tempFilePath = req.file.path;
|
||||
|
||||
const tags = typeof req.body.tags === "string"
|
||||
? req.body.tags.split(",").map((entry: string) => entry.trim()).filter(Boolean)
|
||||
|
|
@ -408,7 +517,8 @@ export function createApp() {
|
|||
});
|
||||
}
|
||||
|
||||
res.status(env.knowledgeLifecycleEnforced ? 201 : 202).json({
|
||||
dispatchAcceptedOcr(result);
|
||||
res.status("phase" in result ? 202 : env.knowledgeLifecycleEnforced ? 201 : 202).json({
|
||||
...result,
|
||||
uploadedResource: req.file.originalname
|
||||
});
|
||||
|
|
|
|||
|
|
@ -52,5 +52,13 @@ export const env = {
|
|||
knowledgeLifecycleEnforced: booleanEnv("KNOWLEDGE_LIFECYCLE_ENFORCED", false),
|
||||
ingestWritesEnabled: booleanEnv("INGEST_WRITES_ENABLED", true),
|
||||
lifecycleReconcileIntervalMs: Number(process.env.LIFECYCLE_RECONCILE_INTERVAL_MS ?? 300000),
|
||||
lifecycleIndexingStaleTimeoutMs: Number(process.env.LIFECYCLE_INDEXING_STALE_TIMEOUT_MS ?? 1800000)
|
||||
lifecycleIndexingStaleTimeoutMs: Number(process.env.LIFECYCLE_INDEXING_STALE_TIMEOUT_MS ?? 1800000),
|
||||
ocrIngestEnabled: booleanEnv("OCR_INGEST_ENABLED", false),
|
||||
ocrServiceUrl: process.env.OCR_SERVICE_URL ?? "http://ocr-service:8000",
|
||||
ocrInternalToken: process.env.OCR_INTERNAL_TOKEN ?? "",
|
||||
ocrArtifactRoot: process.env.OCR_ARTIFACT_ROOT ?? "/data/ingestions",
|
||||
ocrMaxUploadBytes: Number(process.env.OCR_MAX_UPLOAD_BYTES ?? 50 * 1024 * 1024),
|
||||
ocrMaxPages: Number(process.env.OCR_MAX_PAGES ?? 100),
|
||||
ocrPageTimeoutMs: Number(process.env.OCR_PAGE_TIMEOUT_MS ?? 60_000),
|
||||
ocrTotalTimeoutMs: Number(process.env.OCR_TOTAL_TIMEOUT_MS ?? 15 * 60_000)
|
||||
} as const;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import { env } from "../../config/env.js";
|
||||
import type { VectorStoreClient } from "../vectorstore/client.js";
|
||||
import type { CatalogRepository } from "./repository.js";
|
||||
import type { OcrDispatcher } from "../ocr/dispatcher.js";
|
||||
import type { OcrRetentionService } from "../ocr/retention.js";
|
||||
|
||||
export interface ReconcilerStatus {
|
||||
ok: boolean;
|
||||
|
|
@ -9,6 +11,8 @@ export interface ReconcilerStatus {
|
|||
inconsistentSources: string[];
|
||||
orphanedVersionsRecovered?: number;
|
||||
orphanedVersionsFailed?: number;
|
||||
ocrLeasesRecovered?: number;
|
||||
ocrRetentionDeleted?: number;
|
||||
invariantViolations?: string[];
|
||||
}
|
||||
|
||||
|
|
@ -18,7 +22,9 @@ export class KnowledgeLifecycleReconciler {
|
|||
|
||||
constructor(
|
||||
private readonly catalog: CatalogRepository | undefined,
|
||||
private readonly vectorStore: VectorStoreClient
|
||||
private readonly vectorStore: VectorStoreClient,
|
||||
private readonly ocrDispatcher?: Pick<OcrDispatcher, "recoverExpiredLeases" | "dispatchAvailable">,
|
||||
private readonly ocrRetention?: Pick<OcrRetentionService, "runOnce">
|
||||
) {}
|
||||
|
||||
getStatus(): ReconcilerStatus {
|
||||
|
|
@ -49,6 +55,9 @@ export class KnowledgeLifecycleReconciler {
|
|||
const activeVersions = await catalog.resolveActiveVersions();
|
||||
const inconsistentSources: string[] = [];
|
||||
const invariantViolations = await catalog.validateActiveInvariant();
|
||||
const ocrLeasesRecovered = await this.ocrDispatcher?.recoverExpiredLeases() ?? 0;
|
||||
await this.ocrDispatcher?.dispatchAvailable();
|
||||
const retention = await this.ocrRetention?.runOnce();
|
||||
const orphanRecovery = await this.recoverOrphanedVersions(catalog);
|
||||
|
||||
for (const version of activeVersions) {
|
||||
|
|
@ -71,6 +80,8 @@ export class KnowledgeLifecycleReconciler {
|
|||
inconsistentSources,
|
||||
orphanedVersionsRecovered: orphanRecovery.ready,
|
||||
orphanedVersionsFailed: orphanRecovery.failed,
|
||||
ocrLeasesRecovered,
|
||||
ocrRetentionDeleted: (retention?.deleted ?? 0) + (retention?.resumed ?? 0),
|
||||
invariantViolations
|
||||
};
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
import type { AvailableScope, ChunkMode, IngestSourceInput, SourceType, SourceVersionState, RetrieveScope } from "../../shared/types/rag.js";
|
||||
import { sha256Hex } from "../../shared/utils/ids.js";
|
||||
import { withCatalogAdvisoryLock, withCatalogAdvisorySharedLocks, withCatalogTryAdvisoryLock, withTransaction, type PgPool, type PgPoolClient } from "./client.js";
|
||||
import { CatalogError } from "./errors.js";
|
||||
import { normalizeExpectedActiveVersion } from "./lifecycle.js";
|
||||
import type { OcrResult } from "../ocr/client.js";
|
||||
import type { OcrRetentionCandidate } from "../ocr/retention.js";
|
||||
|
||||
export interface CatalogDocumentInput {
|
||||
documentId: string;
|
||||
|
|
@ -12,9 +15,26 @@ export interface CatalogDocumentInput {
|
|||
mimeType: string;
|
||||
title: string;
|
||||
chunkCount: number;
|
||||
extractionMethod?: "native" | "ocr";
|
||||
artifactManifestPath?: string;
|
||||
}
|
||||
|
||||
export interface OcrPageInput {
|
||||
documentId: string;
|
||||
page: number;
|
||||
extractionMethod: "native" | "ocr";
|
||||
nativeTextHash: string;
|
||||
}
|
||||
|
||||
export interface OcrJobInput {
|
||||
documentId: string;
|
||||
remoteIdempotencyKey: string;
|
||||
requestedPages: number[];
|
||||
configVersion: "ocr-v1";
|
||||
}
|
||||
|
||||
export interface CreateVersionInput {
|
||||
versionId?: string;
|
||||
sourceId: string;
|
||||
sourceType: SourceType;
|
||||
sourceRef: string;
|
||||
|
|
@ -31,6 +51,8 @@ export interface CreateVersionInput {
|
|||
expectedDocumentCount: number;
|
||||
expectedPointCount: number;
|
||||
documents: CatalogDocumentInput[];
|
||||
ocrPages?: OcrPageInput[];
|
||||
ocrJobs?: OcrJobInput[];
|
||||
}
|
||||
|
||||
export interface CatalogVersionRow {
|
||||
|
|
@ -79,6 +101,7 @@ export interface PendingOcrIdentity {
|
|||
|
||||
export interface OcrJobRow {
|
||||
jobId: string;
|
||||
versionId: string;
|
||||
documentId: string;
|
||||
remoteJobId: string | null;
|
||||
remoteIdempotencyKey: string;
|
||||
|
|
@ -116,10 +139,12 @@ type VersionDbRow = {
|
|||
expected_point_count: string | number;
|
||||
verified_point_count: string | number;
|
||||
qdrant_collection: string;
|
||||
artifact_state?: "none" | "present" | "retention_deleting" | "retention_deleted";
|
||||
};
|
||||
|
||||
type OcrJobDbRow = {
|
||||
ocr_job_id: string;
|
||||
ocr_version_id: string;
|
||||
ocr_document_id: string;
|
||||
ocr_remote_job_id: string | null;
|
||||
ocr_remote_idempotency_key: string;
|
||||
|
|
@ -136,7 +161,7 @@ type OcrJobDbRow = {
|
|||
};
|
||||
|
||||
const OCR_JOB_COLUMNS = [
|
||||
["job_id", "ocr_job_id"], ["document_id", "ocr_document_id"],
|
||||
["job_id", "ocr_job_id"], ["version_id", "ocr_version_id"], ["document_id", "ocr_document_id"],
|
||||
["remote_job_id", "ocr_remote_job_id"], ["remote_idempotency_key", "ocr_remote_idempotency_key"],
|
||||
["state", "ocr_state"], ["requested_pages", "ocr_requested_pages"],
|
||||
["completed_pages", "ocr_completed_pages"], ["config_version", "ocr_config_version"],
|
||||
|
|
@ -173,6 +198,7 @@ function toVersion(row: VersionDbRow): CatalogVersionRow {
|
|||
function toOcrJob(row: OcrJobDbRow): OcrJobRow {
|
||||
return {
|
||||
jobId: row.ocr_job_id,
|
||||
versionId: row.ocr_version_id,
|
||||
documentId: row.ocr_document_id,
|
||||
remoteJobId: row.ocr_remote_job_id,
|
||||
remoteIdempotencyKey: row.ocr_remote_idempotency_key,
|
||||
|
|
@ -189,6 +215,14 @@ function toOcrJob(row: OcrJobDbRow): OcrJobRow {
|
|||
};
|
||||
}
|
||||
|
||||
function ingestionPhase(state: SourceVersionState, jobs: Array<OcrJobRow["state"] | null>): string {
|
||||
if (state !== "indexing") return state;
|
||||
if (jobs.some((job) => job === "failed")) return "failed";
|
||||
if (jobs.some((job) => job === "running")) return "ocr_running";
|
||||
if (jobs.some((job) => job === "queued")) return "ocr_queued";
|
||||
return "indexing";
|
||||
}
|
||||
|
||||
export class CatalogRepository {
|
||||
constructor(private readonly pool: PgPool) {}
|
||||
|
||||
|
|
@ -338,6 +372,19 @@ export class CatalogRepository {
|
|||
});
|
||||
}
|
||||
|
||||
async claimOcrJob(jobId: string, leaseMs: number): Promise<OcrJobRow | undefined> {
|
||||
const result = await this.pool.query<OcrJobDbRow>(
|
||||
`UPDATE rag_ocr_jobs
|
||||
SET state = 'running', attempt_count = attempt_count + 1,
|
||||
started_at = COALESCE(started_at, now()), heartbeat_at = now(),
|
||||
lease_expires_at = now() + ($2::text || ' milliseconds')::interval
|
||||
WHERE job_id = $1 AND state = 'queued' AND (next_attempt_at IS NULL OR next_attempt_at <= now())
|
||||
RETURNING ${ocrJobColumns()}`,
|
||||
[jobId, String(leaseMs)]
|
||||
);
|
||||
return result.rows[0] ? toOcrJob(result.rows[0]) : undefined;
|
||||
}
|
||||
|
||||
async recoverExpiredOcrLeases(): Promise<OcrJobRow[]> {
|
||||
const result = await this.pool.query<OcrJobDbRow>(
|
||||
`UPDATE rag_ocr_jobs
|
||||
|
|
@ -348,6 +395,142 @@ export class CatalogRepository {
|
|||
return result.rows.map(toOcrJob);
|
||||
}
|
||||
|
||||
async setOcrRemoteJob(jobId: string, remoteJobId: string, leaseMs: number): Promise<void> {
|
||||
const result = await this.pool.query(
|
||||
`UPDATE rag_ocr_jobs SET remote_job_id = $2, heartbeat_at = now(),
|
||||
lease_expires_at = now() + ($3::text || ' milliseconds')::interval
|
||||
WHERE job_id = $1 AND state = 'running'`,
|
||||
[jobId, remoteJobId, String(leaseMs)]
|
||||
);
|
||||
if (result.rowCount !== 1) throw new CatalogError("OCR job lease was lost", 409, "OCR_LEASE_LOST");
|
||||
}
|
||||
|
||||
async requeueOcrJob(jobId: string, code: string, detail: string, delayMs: number): Promise<void> {
|
||||
await this.pool.query(
|
||||
`UPDATE rag_ocr_jobs SET state = 'queued', heartbeat_at = NULL, lease_expires_at = NULL,
|
||||
next_attempt_at = now() + ($4::text || ' milliseconds')::interval, error_code = $2, error_detail = $3
|
||||
WHERE job_id = $1 AND state = 'running'`,
|
||||
[jobId, code, detail.slice(0, 2000), String(delayMs)]
|
||||
);
|
||||
}
|
||||
|
||||
async completeOcrJob(jobId: string, result: OcrResult): Promise<boolean> {
|
||||
return withTransaction(this.pool, async (client) => {
|
||||
const identity = await client.query<{ version_id: string; document_id: string }>(
|
||||
"SELECT version_id, document_id FROM rag_ocr_jobs WHERE job_id = $1 AND state = 'running' FOR UPDATE",
|
||||
[jobId]
|
||||
);
|
||||
if (!identity.rowCount) throw new CatalogError("OCR job lease was lost", 409, "OCR_LEASE_LOST");
|
||||
const { version_id: versionId, document_id: documentId } = identity.rows[0];
|
||||
for (const page of result.pages) {
|
||||
await client.query(
|
||||
`UPDATE rag_document_pages SET ocr_text_hash = $4, candidate_text_hash = $4, metrics = $5
|
||||
WHERE version_id = $1 AND document_id = $2 AND page_number = $3 AND extraction_method = 'ocr'`,
|
||||
[versionId, documentId, page.page, sha256Hex(page.text), page.metrics]
|
||||
);
|
||||
}
|
||||
await client.query(
|
||||
`UPDATE rag_ocr_jobs SET state = 'succeeded', completed_pages = cardinality(requested_pages),
|
||||
heartbeat_at = now(), lease_expires_at = NULL, next_attempt_at = NULL,
|
||||
error_code = NULL, error_detail = NULL, completed_at = now()
|
||||
WHERE job_id = $1`,
|
||||
[jobId]
|
||||
);
|
||||
const pending = await client.query(
|
||||
"SELECT 1 FROM rag_ocr_jobs WHERE version_id = $1 AND state <> 'succeeded' LIMIT 1",
|
||||
[versionId]
|
||||
);
|
||||
return pending.rowCount === 0;
|
||||
});
|
||||
}
|
||||
|
||||
async failOcrJob(jobId: string, code: string, detail: string): Promise<void> {
|
||||
await this.pool.query(
|
||||
`UPDATE rag_ocr_jobs SET state = 'failed', lease_expires_at = NULL, next_attempt_at = NULL,
|
||||
error_code = $2, error_detail = $3, completed_at = now() WHERE job_id = $1 AND state = 'running'`,
|
||||
[jobId, code, detail.slice(0, 2000)]
|
||||
);
|
||||
}
|
||||
|
||||
async getIngestionStatus(versionId: string): Promise<Record<string, unknown> | undefined> {
|
||||
const version = await this.pool.query<{
|
||||
source_id: string;
|
||||
state: SourceVersionState;
|
||||
error_code: string | null;
|
||||
error_detail: string | null;
|
||||
}>("SELECT source_id, state, error_code, error_detail FROM rag_source_versions WHERE version_id = $1", [versionId]);
|
||||
if (!version.rowCount) return undefined;
|
||||
const documents = await this.pool.query<{
|
||||
document_id: string;
|
||||
index_state: "pending" | "indexing" | "ready" | "failed";
|
||||
job_state: OcrJobRow["state"] | null;
|
||||
completed_pages: string | number | null;
|
||||
requested_pages: number[] | null;
|
||||
}>(
|
||||
`SELECT d.document_id, d.index_state, j.state AS job_state, j.completed_pages, j.requested_pages
|
||||
FROM rag_version_documents d LEFT JOIN rag_ocr_jobs j
|
||||
ON j.version_id = d.version_id AND j.document_id = d.document_id
|
||||
WHERE d.version_id = $1 ORDER BY d.document_id`,
|
||||
[versionId]
|
||||
);
|
||||
const pages = await this.pool.query<{
|
||||
document_id: string;
|
||||
page_number: string | number;
|
||||
extraction_method: "native" | "ocr" | "blank";
|
||||
blocked_reason: string | null;
|
||||
}>(
|
||||
"SELECT document_id, page_number, extraction_method, blocked_reason FROM rag_document_pages WHERE version_id = $1 ORDER BY document_id, page_number",
|
||||
[versionId]
|
||||
);
|
||||
const row = version.rows[0];
|
||||
const phase = ingestionPhase(row.state, documents.rows.map(({ job_state }) => job_state));
|
||||
return {
|
||||
sourceId: row.source_id,
|
||||
versionId,
|
||||
state: row.state,
|
||||
phase,
|
||||
activated: row.state === "active",
|
||||
documents: documents.rows.map((document) => {
|
||||
const documentPages = pages.rows.filter((page) => page.document_id === document.document_id);
|
||||
const state = document.job_state === "failed" || document.index_state === "failed"
|
||||
? "failed"
|
||||
: document.job_state === "succeeded"
|
||||
? "ocr_complete"
|
||||
: document.job_state === "running"
|
||||
? "ocr_running"
|
||||
: document.job_state === "queued"
|
||||
? "ocr_queued"
|
||||
: "native_complete";
|
||||
return {
|
||||
documentId: document.document_id,
|
||||
state,
|
||||
completedPages: documentPages.filter((page) => page.extraction_method !== "ocr" && !page.blocked_reason).length
|
||||
+ Number(document.completed_pages ?? 0),
|
||||
totalPages: documentPages.length,
|
||||
pages: documentPages.map((page) => ({
|
||||
page: Number(page.page_number),
|
||||
method: page.extraction_method,
|
||||
state: page.blocked_reason
|
||||
? "failed"
|
||||
: page.extraction_method === "native"
|
||||
? "native_complete"
|
||||
: page.extraction_method === "blank"
|
||||
? "blank"
|
||||
: document.job_state === "succeeded"
|
||||
? "ocr_complete"
|
||||
: document.job_state === "running"
|
||||
? "ocr_running"
|
||||
: "ocr_queued",
|
||||
...(page.blocked_reason ? { errorCode: page.blocked_reason } : {})
|
||||
}))
|
||||
};
|
||||
}),
|
||||
error: row.error_code ? { code: row.error_code, message: row.error_detail ?? row.error_code, retryable: false } : null,
|
||||
statusUrl: `/ingestions/${versionId}`,
|
||||
reviewUrl: row.state === "review_required" ? `/ingestions/${versionId}/review` : null
|
||||
};
|
||||
}
|
||||
|
||||
async markReviewRequired(versionId: string): Promise<void> {
|
||||
const result = await this.pool.query(
|
||||
`UPDATE rag_source_versions SET state = 'review_required'
|
||||
|
|
@ -379,7 +562,8 @@ export class CatalogRepository {
|
|||
async rejectOcrVersion(versionId: string, reviewedBy: string, reason: string): Promise<void> {
|
||||
const result = await this.pool.query(
|
||||
`UPDATE rag_source_versions
|
||||
SET state = 'rejected', reviewed_at = now(), reviewed_by = $2, error_code = 'OCR_REJECTED', error_detail = $3
|
||||
SET state = 'rejected', reviewed_at = now(), reviewed_by = $2, error_code = 'OCR_REJECTED', error_detail = $3,
|
||||
retention_due_at = now() + interval '7 days'
|
||||
WHERE version_id = $1 AND state = 'review_required'`,
|
||||
[versionId, reviewedBy, reason.slice(0, 2000)]
|
||||
);
|
||||
|
|
@ -414,17 +598,19 @@ export class CatalogRepository {
|
|||
[input.sourceId]
|
||||
);
|
||||
const versionNumber = Number(versionNumberResult.rows[0].next_version_number);
|
||||
const versionResult = await client.query<VersionDbRow>(
|
||||
`INSERT INTO rag_source_versions(
|
||||
source_id, version_number, previous_version_id, state, original_manifest_hash,
|
||||
source_content_hash, processing_fingerprint, metadata_hash, tags, activate_requested,
|
||||
base_active_version_id, embedding_provider, embedding_model, embedding_dimensions,
|
||||
qdrant_collection, expected_document_count, expected_point_count
|
||||
) VALUES ($1, $2, $3, 'pending', $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
|
||||
RETURNING *`,
|
||||
[
|
||||
input.sourceId,
|
||||
versionNumber,
|
||||
const versionResult = await client.query<VersionDbRow>(
|
||||
`INSERT INTO rag_source_versions(
|
||||
version_id, source_id, version_number, previous_version_id, state, original_manifest_hash,
|
||||
source_content_hash, processing_fingerprint, metadata_hash, tags, activate_requested,
|
||||
base_active_version_id, embedding_provider, embedding_model, embedding_dimensions,
|
||||
qdrant_collection, expected_document_count, expected_point_count, artifact_state
|
||||
) VALUES (COALESCE($1::uuid, gen_random_uuid()), $2, $3, $4, 'pending', $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17,
|
||||
CASE WHEN $6::char(64) IS NULL THEN 'present' ELSE 'none' END)
|
||||
RETURNING *`,
|
||||
[
|
||||
input.versionId ?? null,
|
||||
input.sourceId,
|
||||
versionNumber,
|
||||
baseActiveVersionId,
|
||||
input.originalManifestHash,
|
||||
input.sourceContentHash,
|
||||
|
|
@ -436,34 +622,51 @@ export class CatalogRepository {
|
|||
input.embeddingProvider,
|
||||
input.embeddingModel,
|
||||
input.embeddingDimensions,
|
||||
input.qdrantCollection,
|
||||
input.expectedDocumentCount,
|
||||
input.expectedPointCount
|
||||
input.qdrantCollection,
|
||||
input.expectedDocumentCount,
|
||||
input.expectedPointCount
|
||||
]
|
||||
);
|
||||
|
||||
const version = toVersion(versionResult.rows[0]);
|
||||
for (const document of input.documents) {
|
||||
await client.query(
|
||||
`INSERT INTO rag_version_documents(
|
||||
version_id, document_id, document_key, original_hash, original_hash_kind,
|
||||
content_hash, mime_type, title, index_state, chunk_count
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending', $9)`,
|
||||
[
|
||||
await client.query(
|
||||
`INSERT INTO rag_version_documents(
|
||||
version_id, document_id, document_key, original_hash, original_hash_kind,
|
||||
content_hash, mime_type, title, extraction_method, index_state, chunk_count, artifact_manifest_path
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'pending', $10, $11)`,
|
||||
[
|
||||
version.versionId,
|
||||
document.documentId,
|
||||
document.documentKey,
|
||||
document.originalHash,
|
||||
document.originalHashKind,
|
||||
document.contentHash,
|
||||
document.mimeType,
|
||||
document.title,
|
||||
document.chunkCount
|
||||
]
|
||||
document.mimeType,
|
||||
document.title,
|
||||
document.extractionMethod ?? "native",
|
||||
document.chunkCount,
|
||||
document.artifactManifestPath ?? null
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
for (const page of input.ocrPages ?? []) {
|
||||
await client.query(
|
||||
`INSERT INTO rag_document_pages(version_id, document_id, page_number, extraction_method, native_text_hash)
|
||||
VALUES ($1, $2, $3, $4, $5)`,
|
||||
[version.versionId, page.documentId, page.page, page.extractionMethod, page.nativeTextHash]
|
||||
);
|
||||
}
|
||||
for (const job of input.ocrJobs ?? []) {
|
||||
await client.query(
|
||||
`INSERT INTO rag_ocr_jobs(version_id, document_id, remote_idempotency_key, state, requested_pages, config_version)
|
||||
VALUES ($1, $2, $3, 'queued', $4, $5)`,
|
||||
[version.versionId, job.documentId, job.remoteIdempotencyKey, job.requestedPages, job.configVersion]
|
||||
);
|
||||
}
|
||||
|
||||
return version;
|
||||
return version;
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -501,7 +704,7 @@ export class CatalogRepository {
|
|||
}
|
||||
const result = await this.pool.query(
|
||||
`UPDATE rag_source_versions
|
||||
SET state = 'failed', error_code = $2, error_detail = $3
|
||||
SET state = 'failed', error_code = $2, error_detail = $3, retention_due_at = now() + interval '7 days'
|
||||
WHERE version_id = $1 AND state IN ('pending', 'indexing')`,
|
||||
[versionId, code, detail.slice(0, 2000)]
|
||||
);
|
||||
|
|
@ -538,6 +741,9 @@ export class CatalogRepository {
|
|||
if (!["ready", "superseded", "active"].includes(version.rows[0].state)) {
|
||||
throw new CatalogError("Only ready or superseded versions can be activated", 409, "VERSION_NOT_ACTIVATABLE");
|
||||
}
|
||||
if (["retention_deleting", "retention_deleted"].includes(version.rows[0].artifact_state ?? "none")) {
|
||||
throw new CatalogError("Version artifacts are unavailable", 409, "VERSION_ARTIFACTS_UNAVAILABLE");
|
||||
}
|
||||
if (version.rows[0].state === "active") {
|
||||
if (currentActive !== versionId) {
|
||||
throw new CatalogError("Active version invariant is inconsistent", 503, "ACTIVE_VERSION_INVARIANT_FAILED");
|
||||
|
|
@ -547,7 +753,7 @@ export class CatalogRepository {
|
|||
|
||||
if (currentActive) {
|
||||
await client.query(
|
||||
"UPDATE rag_source_versions SET state = 'superseded', superseded_at = now() WHERE source_id = $1 AND version_id = $2",
|
||||
"UPDATE rag_source_versions SET state = 'superseded', superseded_at = now(), retention_due_at = now() + interval '30 days' WHERE source_id = $1 AND version_id = $2",
|
||||
[sourceId, currentActive]
|
||||
);
|
||||
}
|
||||
|
|
@ -663,6 +869,50 @@ export class CatalogRepository {
|
|||
await this.pool.query("UPDATE rag_sources SET needs_reingest = true WHERE source_id = $1", [sourceId]);
|
||||
}
|
||||
|
||||
async listOcrRetentionCandidates(now: Date): Promise<OcrRetentionCandidate[]> {
|
||||
const result = await this.pool.query<{ version_id: string; source_id: string; state: SourceVersionState; artifact_state: OcrRetentionCandidate["artifactState"] }>(
|
||||
`SELECT v.version_id, v.source_id, v.state, v.artifact_state FROM rag_source_versions v
|
||||
JOIN rag_sources s ON s.source_id = v.source_id
|
||||
WHERE v.artifact_state IN ('present', 'retention_deleting')
|
||||
AND s.active_version_id IS DISTINCT FROM v.version_id
|
||||
AND (v.artifact_state = 'retention_deleting'
|
||||
OR (v.state = 'review_required' AND v.created_at + interval '30 days' <= $1)
|
||||
OR (v.state IN ('failed', 'rejected') AND COALESCE(v.retention_due_at, v.reviewed_at + interval '7 days', v.created_at + interval '7 days') <= $1)
|
||||
OR (v.state = 'superseded' AND COALESCE(v.retention_due_at, v.superseded_at + interval '30 days') <= $1))
|
||||
ORDER BY v.created_at, v.version_id`, [now]
|
||||
);
|
||||
return result.rows.map((row) => ({ versionId: row.version_id, sourceId: row.source_id, state: row.state, artifactState: row.artifact_state }));
|
||||
}
|
||||
|
||||
async expireOcrReview(sourceId: string, versionId: string, now: Date): Promise<boolean> {
|
||||
const result = await this.pool.query(
|
||||
`UPDATE rag_source_versions v SET state = 'rejected', reviewed_at = $3, error_code = 'REVIEW_EXPIRED',
|
||||
error_detail = 'OCR review expired after 30 days', retention_due_at = $3 + interval '7 days'
|
||||
FROM rag_sources s WHERE v.source_id = $1 AND v.version_id = $2 AND v.source_id = s.source_id
|
||||
AND v.state = 'review_required' AND v.created_at + interval '30 days' <= $3
|
||||
AND s.active_version_id IS DISTINCT FROM v.version_id`, [sourceId, versionId, now]
|
||||
);
|
||||
return result.rowCount === 1;
|
||||
}
|
||||
|
||||
async claimOcrRetentionDeletion(sourceId: string, versionId: string, state: SourceVersionState, artifactState: OcrRetentionCandidate["artifactState"]): Promise<boolean> {
|
||||
const result = await this.pool.query(
|
||||
`UPDATE rag_source_versions v SET artifact_state = 'retention_deleting' FROM rag_sources s
|
||||
WHERE v.source_id = $1 AND v.version_id = $2 AND v.state = $3 AND v.artifact_state = $4
|
||||
AND v.source_id = s.source_id AND s.active_version_id IS DISTINCT FROM v.version_id`, [sourceId, versionId, state, artifactState]
|
||||
);
|
||||
return result.rowCount === 1;
|
||||
}
|
||||
|
||||
async completeOcrRetentionDeletion(sourceId: string, versionId: string, state: SourceVersionState): Promise<boolean> {
|
||||
const result = await this.pool.query(
|
||||
`UPDATE rag_source_versions v SET artifact_state = 'retention_deleted' FROM rag_sources s
|
||||
WHERE v.source_id = $1 AND v.version_id = $2 AND v.state = $3 AND v.artifact_state = 'retention_deleting'
|
||||
AND v.source_id = s.source_id AND s.active_version_id IS DISTINCT FROM v.version_id`, [sourceId, versionId, state]
|
||||
);
|
||||
return result.rowCount === 1;
|
||||
}
|
||||
|
||||
async rollback(sourceId: string, targetVersionId: string, expectedActiveVersionId: string): Promise<CatalogVersionRow> {
|
||||
return this.withSourceLock(sourceId, async () => this.withVersionExclusiveLock(targetVersionId, async () => {
|
||||
const version = await this.activateVersion(sourceId, targetVersionId, expectedActiveVersionId);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import path from "node:path";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { parseDocument, isSupportedDocument } from "../parsers/parser-registry.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { parseDocument, isSupportedDocument, parsePdfPages, type ParsedPdfPage } from "../parsers/parser-registry.js";
|
||||
import { chunkDocument, codeChunkingPolicy, documentalChunkingPolicy } from "../process/chunking.js";
|
||||
import type { EmbeddingProvider } from "../embeddings/provider.js";
|
||||
import type { VectorStoreClient } from "../vectorstore/client.js";
|
||||
|
|
@ -21,6 +22,8 @@ import {
|
|||
import { listFilesRecursively } from "../../shared/utils/files.js";
|
||||
import { env } from "../../config/env.js";
|
||||
import { CatalogError, type CatalogDocumentInput, type CatalogRepository } from "../catalog/repository.js";
|
||||
import { selectPdfPagesForOcr } from "../ocr/detection.js";
|
||||
import { stageOcrArtifacts } from "../ocr/artifacts.js";
|
||||
|
||||
type PreparedDocument = CatalogDocumentInput & {
|
||||
chunks: ReturnType<typeof chunkDocument>;
|
||||
|
|
@ -36,11 +39,38 @@ type OriginalDocument = {
|
|||
originalHash: string;
|
||||
};
|
||||
|
||||
interface OcrPlan {
|
||||
original: OriginalDocument;
|
||||
pages: ParsedPdfPage[];
|
||||
requestedPages: number[];
|
||||
contentHash: string;
|
||||
mimeType: string;
|
||||
title: string;
|
||||
}
|
||||
|
||||
export interface OcrAccepted {
|
||||
accepted: true;
|
||||
sourceId: string;
|
||||
versionId: string;
|
||||
versionNumber: number;
|
||||
state: "indexing";
|
||||
phase: "ocr_queued";
|
||||
statusUrl: string;
|
||||
reviewUrl: null;
|
||||
activated: false;
|
||||
}
|
||||
|
||||
interface OcrIngestOptions {
|
||||
enabled: boolean;
|
||||
artifactRoot: string;
|
||||
}
|
||||
|
||||
export class IngestService {
|
||||
constructor(
|
||||
private readonly embeddingProvider: EmbeddingProvider,
|
||||
private readonly vectorStore: VectorStoreClient,
|
||||
private readonly catalog?: CatalogRepository
|
||||
private readonly catalog?: CatalogRepository,
|
||||
private readonly ocr?: OcrIngestOptions
|
||||
) {}
|
||||
|
||||
async cleanup(scope: RetrieveScope): Promise<{ deleted: number }> {
|
||||
|
|
@ -51,7 +81,7 @@ export class IngestService {
|
|||
return { deleted: count };
|
||||
}
|
||||
|
||||
async ingest(source: IngestSourceInput): Promise<IngestResult> {
|
||||
async ingest(source: IngestSourceInput): Promise<IngestResult | OcrAccepted> {
|
||||
if (!env.ingestWritesEnabled) {
|
||||
throw new CatalogError("Ingest writes are disabled during knowledge catalog maintenance", 503, "INGEST_WRITES_DISABLED");
|
||||
}
|
||||
|
|
@ -135,7 +165,7 @@ export class IngestService {
|
|||
};
|
||||
}
|
||||
|
||||
private async ingestWithLifecycle(source: IngestSourceInput, catalog: CatalogRepository): Promise<IngestResult> {
|
||||
private async ingestWithLifecycle(source: IngestSourceInput, catalog: CatalogRepository): Promise<IngestResult | OcrAccepted> {
|
||||
const activate = source.activate ?? true;
|
||||
if (activate && source.expectedActiveVersionId === undefined) {
|
||||
throw new CatalogError("expectedActiveVersionId is required when activate is true", 400, "EXPECTED_ACTIVE_VERSION_REQUIRED");
|
||||
|
|
@ -158,16 +188,24 @@ export class IngestService {
|
|||
if (!originalManifestHash) {
|
||||
throw new CatalogError("Original manifest hash could not be calculated", 422, "ORIGINAL_MANIFEST_HASH_UNAVAILABLE");
|
||||
}
|
||||
const tags = source.tags ?? [];
|
||||
const embeddingDimensions = this.embeddingProvider.dimensions ?? 4096;
|
||||
const metadataHash = buildMetadataHash(tags, { sourceType: source.sourceType, sourceRef: source.sourceRef });
|
||||
if (this.ocr?.enabled) {
|
||||
const plans = await this.planOcr(originalDocuments);
|
||||
if (plans.some(({ requestedPages }) => requestedPages.length > 0)) {
|
||||
return await this.acceptOcr(source, sourceId, attemptId, originalManifestHash, plans, tags, metadataHash, embeddingDimensions, catalog);
|
||||
}
|
||||
}
|
||||
|
||||
const preparedDocuments = await this.prepareDocuments(originalDocuments);
|
||||
|
||||
if (preparedDocuments.length === 0) {
|
||||
throw new CatalogError("The source has no supported documents with useful content", 422, "EMPTY_SOURCE");
|
||||
}
|
||||
|
||||
const tags = source.tags ?? [];
|
||||
const sourceContentHash = hashOrderedPairs(preparedDocuments.map((document) => [document.documentKey, document.contentHash]));
|
||||
|
||||
const embeddingDimensions = this.embeddingProvider.dimensions ?? 4096;
|
||||
const processingFingerprint = buildProcessingFingerprint({
|
||||
parserVersion: "native-v1",
|
||||
normalizationPolicy: "bom-crlf-trim-final-lf-v1",
|
||||
|
|
@ -179,8 +217,6 @@ export class IngestService {
|
|||
embeddingModel: this.embeddingProvider.modelName,
|
||||
embeddingDimensions
|
||||
});
|
||||
const metadataHash = buildMetadataHash(tags, { sourceType: source.sourceType, sourceRef: source.sourceRef });
|
||||
|
||||
return await catalog.withSourceLock(sourceId, async () => {
|
||||
const reusable = await catalog.findReusableVersion({ sourceId, sourceContentHash, processingFingerprint, metadataHash });
|
||||
if (reusable?.state === "active") {
|
||||
|
|
@ -311,6 +347,146 @@ export class IngestService {
|
|||
}
|
||||
}
|
||||
|
||||
private async planOcr(originals: OriginalDocument[]): Promise<OcrPlan[]> {
|
||||
const plans: OcrPlan[] = [];
|
||||
for (const original of originals) {
|
||||
if (path.extname(original.filePath).toLowerCase() === ".pdf") {
|
||||
const pages = await parsePdfPages(original.filePath);
|
||||
plans.push({
|
||||
original,
|
||||
pages,
|
||||
requestedPages: selectPdfPagesForOcr(original.filePath, pages),
|
||||
contentHash: sha256Hex(normalizeContentForHash(pages.map(({ text }) => text).join("\n"))),
|
||||
mimeType: "application/pdf",
|
||||
title: path.basename(original.filePath)
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const parsed = await parseDocument(original.filePath);
|
||||
plans.push({
|
||||
original,
|
||||
pages: [],
|
||||
requestedPages: [],
|
||||
contentHash: sha256Hex(normalizeContentForHash(parsed.content)),
|
||||
mimeType: parsed.mimeType,
|
||||
title: parsed.title
|
||||
});
|
||||
}
|
||||
return plans;
|
||||
}
|
||||
|
||||
private async acceptOcr(
|
||||
source: IngestSourceInput,
|
||||
sourceId: string,
|
||||
attemptId: string,
|
||||
originalManifestHash: string,
|
||||
plans: OcrPlan[],
|
||||
tags: string[],
|
||||
metadataHash: string,
|
||||
embeddingDimensions: number,
|
||||
catalog: CatalogRepository
|
||||
): Promise<OcrAccepted> {
|
||||
const processingFingerprint = buildProcessingFingerprint({
|
||||
parserVersion: "native-pages-v1+ocr-v1",
|
||||
detectionPolicyVersion: "pdf-detection-v1",
|
||||
normalizationPolicy: "bom-crlf-trim-final-lf-v1",
|
||||
chunking: { code: codeChunkingPolicy, documental: documentalChunkingPolicy },
|
||||
embeddingProvider: this.embeddingProvider.providerName,
|
||||
embeddingModel: this.embeddingProvider.modelName,
|
||||
embeddingDimensions
|
||||
});
|
||||
return catalog.withSourceLock(sourceId, async () => {
|
||||
const pending = await catalog.findPendingOcrVersion({ sourceId, originalManifestHash, processingFingerprint, metadataHash });
|
||||
if (pending) {
|
||||
await catalog.updateAttempt(attemptId, { state: "completed", versionId: pending.version.versionId, inputHash: originalManifestHash });
|
||||
return this.ocrAccepted(sourceId, pending.version.versionId, pending.version.versionNumber);
|
||||
}
|
||||
|
||||
const versionId = randomUUID();
|
||||
const staged = await stageOcrArtifacts({
|
||||
rootDirectory: this.ocr!.artifactRoot,
|
||||
versionId,
|
||||
createdAt: new Date().toISOString(),
|
||||
documents: plans.map(({ original }) => ({
|
||||
documentId: original.documentId,
|
||||
documentKey: original.documentKey,
|
||||
bytes: original.bytes
|
||||
}))
|
||||
});
|
||||
if (staged.originalManifestHash !== originalManifestHash) {
|
||||
throw new CatalogError("Durable OCR manifest does not match the source", 500, "OCR_ARTIFACT_INTEGRITY_FAILED");
|
||||
}
|
||||
let created;
|
||||
try {
|
||||
created = await catalog.createPendingVersion({
|
||||
versionId,
|
||||
sourceId,
|
||||
sourceType: source.sourceType,
|
||||
sourceRef: source.sourceRef,
|
||||
originalManifestHash,
|
||||
sourceContentHash: null,
|
||||
processingFingerprint,
|
||||
metadataHash,
|
||||
tags,
|
||||
activateRequested: source.activate ?? true,
|
||||
embeddingProvider: this.embeddingProvider.providerName,
|
||||
embeddingModel: this.embeddingProvider.modelName,
|
||||
embeddingDimensions,
|
||||
qdrantCollection: env.qdrantCollection,
|
||||
expectedDocumentCount: plans.length,
|
||||
expectedPointCount: 0,
|
||||
documents: plans.map(({ original, requestedPages, contentHash, mimeType, title }) => ({
|
||||
documentId: original.documentId,
|
||||
documentKey: original.documentKey,
|
||||
originalHash: original.originalHash,
|
||||
originalHashKind: "bytes",
|
||||
contentHash: requestedPages.length > 0 ? null : contentHash,
|
||||
mimeType,
|
||||
title,
|
||||
chunkCount: 0,
|
||||
extractionMethod: requestedPages.length > 0 ? "ocr" : "native",
|
||||
artifactManifestPath: staged.manifestPath
|
||||
})),
|
||||
ocrPages: plans.flatMap(({ original, pages, requestedPages }) => pages.map((page) => ({
|
||||
documentId: original.documentId,
|
||||
page: page.page,
|
||||
extractionMethod: requestedPages.includes(page.page) ? "ocr" as const : "native" as const,
|
||||
nativeTextHash: page.textSha256
|
||||
}))),
|
||||
ocrJobs: plans.filter(({ requestedPages }) => requestedPages.length > 0).map(({ original, requestedPages }) => ({
|
||||
documentId: original.documentId,
|
||||
remoteIdempotencyKey: `${original.originalHash}:ocr-v1:${sha256Hex(JSON.stringify(requestedPages))}`,
|
||||
requestedPages,
|
||||
configVersion: "ocr-v1"
|
||||
}))
|
||||
});
|
||||
} catch (error) {
|
||||
if ((error as { code?: unknown }).code !== "23505") throw error;
|
||||
const winner = await catalog.findPendingOcrVersion({ sourceId, originalManifestHash, processingFingerprint, metadataHash });
|
||||
if (!winner) throw error;
|
||||
await catalog.updateAttempt(attemptId, { state: "completed", versionId: winner.version.versionId, inputHash: originalManifestHash });
|
||||
return this.ocrAccepted(sourceId, winner.version.versionId, winner.version.versionNumber);
|
||||
}
|
||||
await catalog.markIndexing(created.versionId);
|
||||
await catalog.updateAttempt(attemptId, { state: "completed", versionId: created.versionId, inputHash: originalManifestHash });
|
||||
return this.ocrAccepted(sourceId, created.versionId, created.versionNumber);
|
||||
});
|
||||
}
|
||||
|
||||
private ocrAccepted(sourceId: string, versionId: string, versionNumber: number): OcrAccepted {
|
||||
return {
|
||||
accepted: true,
|
||||
sourceId,
|
||||
versionId,
|
||||
versionNumber,
|
||||
state: "indexing",
|
||||
phase: "ocr_queued",
|
||||
statusUrl: `/ingestions/${versionId}`,
|
||||
reviewUrl: null,
|
||||
activated: false
|
||||
};
|
||||
}
|
||||
|
||||
private async readOriginalDocuments(source: IngestSourceInput, sourceId: string, files: string[]): Promise<OriginalDocument[]> {
|
||||
const originals: OriginalDocument[] = [];
|
||||
for (const filePath of files) {
|
||||
|
|
|
|||
|
|
@ -69,11 +69,12 @@ export class OcrClient {
|
|||
this.sleep = options.sleep ?? ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
|
||||
}
|
||||
|
||||
async submit(file: Buffer, expected: { documentSha256: string; pages: number[] }): Promise<OcrAck> {
|
||||
async submit(file: Buffer, expected: { documentSha256: string; pages: number[] }, persistedIdempotencyKey?: string): Promise<OcrAck> {
|
||||
assertExpected(expected);
|
||||
if (sha256Hex(file) !== expected.documentSha256) throw integrityError();
|
||||
const payload = { documentSha256: expected.documentSha256, pages: expected.pages, ...OCR_CONFIG };
|
||||
const idempotencyKey = `${expected.documentSha256}:ocr-v1:${sha256Hex(JSON.stringify(expected.pages))}`;
|
||||
const idempotencyKey = persistedIdempotencyKey ?? `${expected.documentSha256}:ocr-v1:${sha256Hex(JSON.stringify(expected.pages))}`;
|
||||
if (!idempotencyKey.trim() || /[\r\n]/u.test(idempotencyKey)) throw new TypeError("OCR idempotency key is invalid");
|
||||
const value = await this.requestJson("/v1/jobs", () => {
|
||||
const form = new FormData();
|
||||
form.set("file", new Blob([new Uint8Array(file)], { type: "application/pdf" }), "original.pdf");
|
||||
|
|
|
|||
108
src/modules/ocr/dispatcher.ts
Normal file
108
src/modules/ocr/dispatcher.ts
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
import type { OcrJobRow } from "../catalog/repository.js";
|
||||
import { OcrClientError, type OcrClient, type OcrResult } from "./client.js";
|
||||
|
||||
export interface OcrDispatchStore {
|
||||
claimNextOcrJob(leaseMs: number): Promise<OcrJobRow | undefined>;
|
||||
claimOcrJob(jobId: string, leaseMs: number): Promise<OcrJobRow | undefined>;
|
||||
recoverExpiredOcrLeases(): Promise<OcrJobRow[]>;
|
||||
setOcrRemoteJob(jobId: string, remoteJobId: string, leaseMs: number): Promise<void>;
|
||||
requeueOcrJob(jobId: string, code: string, detail: string, delayMs: number): Promise<void>;
|
||||
completeOcrJob(jobId: string, result: OcrResult): Promise<boolean>;
|
||||
failOcrJob(jobId: string, code: string, detail: string): Promise<void>;
|
||||
markReviewRequired(versionId: string): Promise<void>;
|
||||
markFailed(versionId: string, code: string, detail: string): Promise<void>;
|
||||
}
|
||||
|
||||
export type OcrDispatchInput = { bytes: Buffer; documentSha256: string };
|
||||
export type OcrDispatchResult = "idle" | "pending" | "succeeded" | "failed";
|
||||
|
||||
export class OcrDispatcher {
|
||||
private activeDrain: Promise<number> | undefined;
|
||||
|
||||
constructor(
|
||||
private readonly store: OcrDispatchStore,
|
||||
private readonly client: OcrClient,
|
||||
private readonly loadInput: (job: OcrJobRow) => Promise<OcrDispatchInput>,
|
||||
private readonly leaseMs = 30_000
|
||||
) {}
|
||||
|
||||
async runOnce(): Promise<OcrDispatchResult> {
|
||||
const job = await this.store.claimNextOcrJob(this.leaseMs);
|
||||
if (!job) return "idle";
|
||||
|
||||
return this.dispatch(job);
|
||||
}
|
||||
|
||||
private async dispatch(job: OcrJobRow): Promise<OcrDispatchResult> {
|
||||
try {
|
||||
const input = await this.loadInput(job);
|
||||
let remoteJobId = job.remoteJobId;
|
||||
if (!remoteJobId) {
|
||||
const acknowledgement = await this.client.submit(input.bytes, {
|
||||
documentSha256: input.documentSha256,
|
||||
pages: job.requestedPages
|
||||
}, job.remoteIdempotencyKey);
|
||||
remoteJobId = acknowledgement.jobId;
|
||||
await this.store.setOcrRemoteJob(job.jobId, remoteJobId, this.leaseMs);
|
||||
}
|
||||
|
||||
const status = await this.client.getStatus(remoteJobId);
|
||||
if (status.status === "queued" || status.status === "running") {
|
||||
const delayMs = Math.min(2_000 * 2 ** Math.max(0, job.attemptCount - 1), 15_000);
|
||||
await this.store.requeueOcrJob(job.jobId, "OCR_PENDING", status.status, delayMs);
|
||||
return "pending";
|
||||
}
|
||||
if (status.status === "failed") {
|
||||
const code = status.error?.code ?? "OCR_REMOTE_FAILED";
|
||||
await this.fail(job, code, status.error?.message ?? "OCR processing failed");
|
||||
return "failed";
|
||||
}
|
||||
|
||||
const result = await this.client.getResult(remoteJobId, {
|
||||
documentSha256: input.documentSha256,
|
||||
pages: job.requestedPages
|
||||
});
|
||||
const versionComplete = await this.store.completeOcrJob(job.jobId, result);
|
||||
if (versionComplete) await this.store.markReviewRequired(job.versionId);
|
||||
return "succeeded";
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? error.message : "Unknown OCR dispatch failure";
|
||||
if (error instanceof OcrClientError && error.retryable) {
|
||||
await this.store.requeueOcrJob(job.jobId, error.code, detail, 15_000);
|
||||
return "pending";
|
||||
}
|
||||
const code = error instanceof OcrClientError ? error.code : "OCR_DISPATCH_FAILED";
|
||||
await this.fail(job, code, detail);
|
||||
return "failed";
|
||||
}
|
||||
}
|
||||
|
||||
async recoverExpiredLeases(): Promise<number> {
|
||||
const recovered = await this.store.recoverExpiredOcrLeases();
|
||||
for (const job of recovered) {
|
||||
const claimed = await this.store.claimOcrJob(job.jobId, this.leaseMs);
|
||||
if (claimed) await this.dispatch(claimed);
|
||||
}
|
||||
return recovered.length;
|
||||
}
|
||||
|
||||
dispatchAvailable(): Promise<number> {
|
||||
if (this.activeDrain) return this.activeDrain;
|
||||
const drain = this.drainAvailable();
|
||||
this.activeDrain = drain;
|
||||
const clear = () => { if (this.activeDrain === drain) this.activeDrain = undefined; };
|
||||
void drain.then(clear, clear);
|
||||
return drain;
|
||||
}
|
||||
|
||||
private async drainAvailable(): Promise<number> {
|
||||
let processed = 0;
|
||||
while (await this.runOnce() !== "idle") processed += 1;
|
||||
return processed;
|
||||
}
|
||||
|
||||
private async fail(job: OcrJobRow, code: string, detail: string): Promise<void> {
|
||||
await this.store.failOcrJob(job.jobId, code, detail);
|
||||
await this.store.markFailed(job.versionId, code, detail);
|
||||
}
|
||||
}
|
||||
67
src/modules/ocr/indexing.ts
Normal file
67
src/modules/ocr/indexing.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
import { CatalogError } from "../catalog/errors.js";
|
||||
|
||||
export interface ApprovedOcrCandidate {
|
||||
versionId: string;
|
||||
sourceId: string;
|
||||
state: "indexing" | "rejected";
|
||||
activateRequested: boolean;
|
||||
expectedActiveVersionId: string | null;
|
||||
reviewedText: string;
|
||||
reviewedTextSha256: string;
|
||||
processingFingerprint: string;
|
||||
metadataHash: string;
|
||||
}
|
||||
|
||||
export interface OcrIndexingStore {
|
||||
findReusableVersion(candidate: ApprovedOcrCandidate): Promise<{ versionId: string } | undefined>;
|
||||
indexReviewed(candidate: ApprovedOcrCandidate): Promise<number>;
|
||||
markReady(versionId: string, verifiedPointCount: number): Promise<void>;
|
||||
settleReusable(candidate: ApprovedOcrCandidate, reusableVersionId: string, activate: boolean): Promise<boolean>;
|
||||
activateVersion(sourceId: string, versionId: string, expectedActiveVersionId: string | null): Promise<string>;
|
||||
}
|
||||
|
||||
export type OcrIndexingResult = {
|
||||
versionId: string;
|
||||
state: "ready" | "active" | "rejected";
|
||||
activated: boolean;
|
||||
activatedVersionId?: string;
|
||||
errorCode?: "DUPLICATE_REUSABLE_VERSION";
|
||||
};
|
||||
|
||||
function activeRace(error: unknown): never {
|
||||
if (error instanceof CatalogError && error.statusCode === 409) {
|
||||
throw new CatalogError("Active version changed during OCR approval", 409, "ACTIVE_VERSION_CHANGED");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
export class OcrIndexingService {
|
||||
constructor(private readonly store: OcrIndexingStore) {}
|
||||
|
||||
async index(candidate: ApprovedOcrCandidate): Promise<OcrIndexingResult> {
|
||||
if (candidate.state !== "indexing") throw new CatalogError("Only approved OCR candidates can be indexed", 409, "INVALID_VERSION_STATE");
|
||||
const reusable = await this.store.findReusableVersion(candidate);
|
||||
if (reusable) {
|
||||
let activated: boolean;
|
||||
try {
|
||||
activated = await this.store.settleReusable(candidate, reusable.versionId, candidate.activateRequested);
|
||||
} catch (error) {
|
||||
return activeRace(error);
|
||||
}
|
||||
return {
|
||||
versionId: candidate.versionId, state: "rejected", activated,
|
||||
...(activated ? { activatedVersionId: reusable.versionId } : {}), errorCode: "DUPLICATE_REUSABLE_VERSION"
|
||||
};
|
||||
}
|
||||
|
||||
const verifiedPointCount = await this.store.indexReviewed(candidate);
|
||||
await this.store.markReady(candidate.versionId, verifiedPointCount);
|
||||
if (!candidate.activateRequested) return { versionId: candidate.versionId, state: "ready", activated: false };
|
||||
try {
|
||||
const activatedVersionId = await this.store.activateVersion(candidate.sourceId, candidate.versionId, candidate.expectedActiveVersionId);
|
||||
return { versionId: candidate.versionId, state: "active", activated: true, activatedVersionId };
|
||||
} catch (error) {
|
||||
return activeRace(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
45
src/modules/ocr/retention.ts
Normal file
45
src/modules/ocr/retention.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { rm } from "node:fs/promises";
|
||||
import type { SourceVersionState } from "../../shared/types/rag.js";
|
||||
import { resolveArtifactPath } from "./artifacts.js";
|
||||
|
||||
export interface OcrRetentionCandidate {
|
||||
versionId: string;
|
||||
sourceId: string;
|
||||
state: SourceVersionState;
|
||||
artifactState: "present" | "retention_deleting";
|
||||
}
|
||||
|
||||
export interface OcrRetentionStore {
|
||||
listOcrRetentionCandidates(now: Date): Promise<OcrRetentionCandidate[]>;
|
||||
withVersionTryLock<T>(versionId: string, handler: () => Promise<T>): Promise<T | undefined>;
|
||||
expireOcrReview(sourceId: string, versionId: string, now: Date): Promise<boolean>;
|
||||
claimOcrRetentionDeletion(sourceId: string, versionId: string, state: SourceVersionState, artifactState: OcrRetentionCandidate["artifactState"]): Promise<boolean>;
|
||||
completeOcrRetentionDeletion(sourceId: string, versionId: string, state: SourceVersionState): Promise<boolean>;
|
||||
}
|
||||
|
||||
export class OcrRetentionService {
|
||||
constructor(private readonly store: OcrRetentionStore, private readonly artifactRoot: string) {}
|
||||
|
||||
async runOnce(now = new Date()): Promise<{ expired: number; deleted: number; resumed: number; skipped: number }> {
|
||||
const result = { expired: 0, deleted: 0, resumed: 0, skipped: 0 };
|
||||
for (const candidate of await this.store.listOcrRetentionCandidates(now)) {
|
||||
if (candidate.state === "active") {
|
||||
result.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
const outcome = await this.store.withVersionTryLock(candidate.versionId, async () => {
|
||||
if (candidate.state === "review_required") {
|
||||
return await this.store.expireOcrReview(candidate.sourceId, candidate.versionId, now) ? "expired" : "skipped";
|
||||
}
|
||||
const claimed = await this.store.claimOcrRetentionDeletion(candidate.sourceId, candidate.versionId, candidate.state, candidate.artifactState);
|
||||
if (!claimed) return "skipped";
|
||||
await rm(resolveArtifactPath(this.artifactRoot, candidate.versionId), { recursive: true, force: true });
|
||||
return await this.store.completeOcrRetentionDeletion(candidate.sourceId, candidate.versionId, candidate.state)
|
||||
? candidate.artifactState === "retention_deleting" ? "resumed" : "deleted"
|
||||
: "skipped";
|
||||
});
|
||||
result[outcome ?? "skipped"] += 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
143
src/modules/ocr/review.ts
Normal file
143
src/modules/ocr/review.ts
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
import { CatalogError } from "../catalog/errors.js";
|
||||
import { sha256Hex } from "../../shared/utils/ids.js";
|
||||
|
||||
export interface OcrReviewLine {
|
||||
lineId: string;
|
||||
text: string;
|
||||
confidence: number;
|
||||
bbox: [number, number, number, number];
|
||||
lineSha256: string;
|
||||
}
|
||||
|
||||
export interface OcrReviewCandidate {
|
||||
versionId: string;
|
||||
sourceId: string;
|
||||
state: "review_required" | "indexing" | "rejected";
|
||||
candidateSha256: string;
|
||||
baseActiveVersionId: string | null;
|
||||
currentActiveVersionId: string | null;
|
||||
activateRequested: boolean;
|
||||
processingFingerprint: string;
|
||||
metadataHash: string;
|
||||
documents: Array<{ documentId: string; pages: Array<{
|
||||
page: number;
|
||||
imageUrl: string;
|
||||
nativeText: string;
|
||||
ocr: { text: string; lines: OcrReviewLine[] };
|
||||
candidateText: string;
|
||||
differences: string[];
|
||||
risks: string[];
|
||||
}> }>;
|
||||
}
|
||||
|
||||
export interface OcrCorrection {
|
||||
documentId: string;
|
||||
page: number;
|
||||
lineId: string;
|
||||
expectedLineSha256: string;
|
||||
replacementText: string;
|
||||
}
|
||||
|
||||
export interface OcrReviewStore {
|
||||
loadCandidate(versionId: string): Promise<OcrReviewCandidate | undefined>;
|
||||
commitApproval(input: {
|
||||
candidate: OcrReviewCandidate;
|
||||
candidateSha256: string;
|
||||
expectedActiveVersionId: string | null;
|
||||
reviewedBy: string;
|
||||
corrections: OcrCorrection[];
|
||||
reviewedText: string;
|
||||
reviewedTextSha256: string;
|
||||
}): Promise<void>;
|
||||
commitRejection(input: {
|
||||
candidate: OcrReviewCandidate;
|
||||
candidateSha256: string;
|
||||
reviewedBy: string;
|
||||
reason: string;
|
||||
}): Promise<void>;
|
||||
}
|
||||
|
||||
export interface ApprovedOcrReview {
|
||||
versionId: string;
|
||||
sourceId: string;
|
||||
state: "indexing";
|
||||
activateRequested: boolean;
|
||||
expectedActiveVersionId: string | null;
|
||||
reviewedText: string;
|
||||
reviewedTextSha256: string;
|
||||
processingFingerprint: string;
|
||||
metadataHash: string;
|
||||
}
|
||||
|
||||
function conflict(message: string, code = "REVIEW_CONFLICT"): CatalogError {
|
||||
return new CatalogError(message, 409, code);
|
||||
}
|
||||
|
||||
export class OcrReviewService {
|
||||
constructor(private readonly store: OcrReviewStore) {}
|
||||
|
||||
async view(versionId: string): Promise<OcrReviewCandidate> {
|
||||
const candidate = await this.store.loadCandidate(versionId);
|
||||
if (!candidate) throw new CatalogError("OCR review candidate not found", 404, "REVIEW_NOT_FOUND");
|
||||
if (candidate.state !== "review_required") throw conflict("Version is not awaiting OCR review", "INVALID_VERSION_STATE");
|
||||
return candidate;
|
||||
}
|
||||
|
||||
async approve(versionId: string, input: {
|
||||
candidateSha256: string;
|
||||
expectedActiveVersionId: string | null;
|
||||
reviewedBy: string;
|
||||
corrections: OcrCorrection[];
|
||||
}): Promise<ApprovedOcrReview> {
|
||||
const candidate = await this.view(versionId);
|
||||
if (input.candidateSha256 !== candidate.candidateSha256) throw conflict("Candidate hash is stale");
|
||||
if (input.expectedActiveVersionId !== candidate.baseActiveVersionId || input.expectedActiveVersionId !== candidate.currentActiveVersionId) {
|
||||
throw conflict("Active version changed during review", "ACTIVE_VERSION_CHANGED");
|
||||
}
|
||||
|
||||
const reviewedCandidate = structuredClone(candidate);
|
||||
const lines = new Map<string, OcrReviewLine>();
|
||||
for (const document of reviewedCandidate.documents) for (const page of document.pages) for (const line of page.ocr.lines) {
|
||||
const key = `${document.documentId}:${page.page}:${line.lineId}`;
|
||||
if (lines.has(key)) throw conflict("Candidate line identities are duplicated", "CORRECTION_CONFLICT");
|
||||
lines.set(key, line);
|
||||
}
|
||||
const targets = new Set<string>();
|
||||
for (const correction of input.corrections) {
|
||||
const key = `${correction.documentId}:${correction.page}:${correction.lineId}`;
|
||||
const line = lines.get(key);
|
||||
if (targets.has(key) || !line || line.lineSha256 !== correction.expectedLineSha256) {
|
||||
throw conflict("Correction target is stale or duplicated", "CORRECTION_CONFLICT");
|
||||
}
|
||||
targets.add(key);
|
||||
line.text = correction.replacementText;
|
||||
line.lineSha256 = sha256Hex(line.text);
|
||||
}
|
||||
for (const document of reviewedCandidate.documents) for (const page of document.pages) {
|
||||
if (page.ocr.lines.length > 0) page.ocr.text = page.candidateText = page.ocr.lines.map(({ text }) => text).join("\n");
|
||||
}
|
||||
const reviewedText = reviewedCandidate.documents.flatMap(({ pages }) => pages.filter(({ candidateText }) => candidateText).map(({ candidateText }) => candidateText)).join("\n\n");
|
||||
const reviewedTextSha256 = sha256Hex(reviewedText);
|
||||
await this.store.commitApproval({ candidate: reviewedCandidate, ...input, reviewedText, reviewedTextSha256 });
|
||||
return {
|
||||
versionId, sourceId: candidate.sourceId, state: "indexing", activateRequested: candidate.activateRequested,
|
||||
expectedActiveVersionId: input.expectedActiveVersionId, reviewedText, reviewedTextSha256,
|
||||
processingFingerprint: candidate.processingFingerprint, metadataHash: candidate.metadataHash
|
||||
};
|
||||
}
|
||||
|
||||
async reject(versionId: string, input: { candidateSha256: string; reviewedBy: string; reason: string }): Promise<{
|
||||
versionId: string; state: "rejected"; activated: false;
|
||||
}> {
|
||||
if (!input || typeof input.candidateSha256 !== "string" || typeof input.reviewedBy !== "string" || typeof input.reason !== "string") {
|
||||
throw new CatalogError("Candidate hash, reviewer, and rejection reason are required", 400, "INVALID_REJECTION");
|
||||
}
|
||||
const reviewedBy = input.reviewedBy?.trim();
|
||||
const reason = input.reason?.trim();
|
||||
if (!input.candidateSha256 || !reviewedBy || !reason) throw new CatalogError("Candidate hash, reviewer, and rejection reason are required", 400, "INVALID_REJECTION");
|
||||
const candidate = await this.view(versionId);
|
||||
if (input.candidateSha256 !== candidate.candidateSha256) throw conflict("Candidate hash is stale");
|
||||
await this.store.commitRejection({ candidate, candidateSha256: input.candidateSha256, reviewedBy, reason });
|
||||
return { versionId, state: "rejected", activated: false };
|
||||
}
|
||||
}
|
||||
71
tests/ocr/contracts-deploy.test.ts
Normal file
71
tests/ocr/contracts-deploy.test.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { buildApiHelp, openApiDocument } from "../../src/api/openapi.js";
|
||||
import { env } from "../../src/config/env.js";
|
||||
|
||||
const api = openApiDocument as unknown as {
|
||||
paths: Record<string, Record<string, Record<string, unknown>>>;
|
||||
components: { schemas: Record<string, Record<string, unknown>> };
|
||||
};
|
||||
|
||||
test("OpenAPI describes authenticated OCR ingestion, status, review, approval, and rejection", () => {
|
||||
const ingestResponses = api.paths["/ingest"]!.post!.responses as Record<string, { content?: Record<string, { schema?: unknown }> }>;
|
||||
const uploadResponses = api.paths["/ingest/upload"]!.post!.responses as typeof ingestResponses;
|
||||
assert.deepEqual(ingestResponses["202"]!.content!["application/json"]!.schema, { oneOf: [{ $ref: "#/components/schemas/OcrAccepted" }, { $ref: "#/components/schemas/IngestResponse" }] });
|
||||
assert.deepEqual(uploadResponses["202"]!.content!["application/json"]!.schema, { oneOf: [{ $ref: "#/components/schemas/OcrUploadAccepted" }, { $ref: "#/components/schemas/UploadIngestResponse" }] });
|
||||
|
||||
for (const [path, method] of [
|
||||
["/ingestions/{versionId}", "get"],
|
||||
["/ingestions/{versionId}/review", "get"],
|
||||
["/ingestions/{versionId}/approve", "post"],
|
||||
["/ingestions/{versionId}/reject", "post"]
|
||||
]) {
|
||||
const operation = api.paths[path]![method]!;
|
||||
assert.deepEqual(operation.security, [{ bearerAuth: [] }], `${method.toUpperCase()} ${path} must require bearer auth`);
|
||||
const responses = operation.responses as Record<string, unknown>;
|
||||
assert.ok("200" in responses && "401" in responses && "404" in responses && "503" in responses);
|
||||
}
|
||||
|
||||
const approve = api.paths["/ingestions/{versionId}/approve"]!.post!;
|
||||
const reject = api.paths["/ingestions/{versionId}/reject"]!.post!;
|
||||
assert.ok("409" in (approve.responses as Record<string, unknown>));
|
||||
assert.ok("400" in (reject.responses as Record<string, unknown>));
|
||||
assert.ok("409" in (reject.responses as Record<string, unknown>));
|
||||
assert.deepEqual((approve.requestBody as { content: Record<string, { schema: unknown }> }).content["application/json"]!.schema,
|
||||
{ $ref: "#/components/schemas/OcrApprovalRequest" });
|
||||
assert.deepEqual((reject.requestBody as { content: Record<string, { schema: unknown }> }).content["application/json"]!.schema,
|
||||
{ $ref: "#/components/schemas/OcrRejectionRequest" });
|
||||
assert.match(buildApiHelp().authentication, /Bearer.*OCR/u);
|
||||
});
|
||||
|
||||
test("OpenAPI OCR schemas preserve progress, review evidence, corrections, and decision states", () => {
|
||||
const schemas = api.components.schemas;
|
||||
assert.deepEqual(schemas.OcrPhase!.enum, ["native_extracting", "ocr_queued", "ocr_running", "review_required", "indexing", "ready", "active", "failed", "rejected"]);
|
||||
assert.deepEqual(schemas.OcrAccepted!.required, ["accepted", "sourceId", "versionId", "versionNumber", "state", "phase", "statusUrl", "reviewUrl", "activated"]);
|
||||
assert.deepEqual(schemas.IngestionStatus!.required, ["sourceId", "versionId", "state", "phase", "activated", "documents", "error", "statusUrl", "reviewUrl"]);
|
||||
assert.deepEqual(schemas.OcrReview!.required, ["versionId", "sourceId", "state", "candidateSha256", "baseActiveVersionId", "currentActiveVersionId", "activateRequested", "processingFingerprint", "metadataHash", "documents"]);
|
||||
const approvalProperties = schemas.OcrApprovalRequest!.properties as Record<string, { items?: unknown }>;
|
||||
assert.deepEqual(approvalProperties.corrections!.items, { $ref: "#/components/schemas/OcrCorrection" });
|
||||
assert.deepEqual(schemas.OcrDecisionResponse!.properties, {
|
||||
versionId: { type: "string", format: "uuid" }, state: { type: "string", enum: ["ready", "active", "rejected"] }, activated: { type: "boolean" },
|
||||
activatedVersionId: { type: "string", format: "uuid" }, errorCode: { type: "string", enum: ["DUPLICATE_REUSABLE_VERSION"] }
|
||||
});
|
||||
});
|
||||
|
||||
test("OCR deployment defaults and container wiring match the private durable contract", async () => {
|
||||
assert.deepEqual({
|
||||
root: env.ocrArtifactRoot,
|
||||
uploadBytes: (env as unknown as Record<string, unknown>).ocrMaxUploadBytes,
|
||||
pages: (env as unknown as Record<string, unknown>).ocrMaxPages,
|
||||
pageTimeoutMs: (env as unknown as Record<string, unknown>).ocrPageTimeoutMs,
|
||||
totalTimeoutMs: (env as unknown as Record<string, unknown>).ocrTotalTimeoutMs
|
||||
}, { root: "/data/ingestions", uploadBytes: 50 * 1024 * 1024, pages: 100, pageTimeoutMs: 60_000, totalTimeoutMs: 15 * 60_000 });
|
||||
|
||||
const dockerfile = await readFile(new URL("../../Dockerfile", import.meta.url), "utf8");
|
||||
assert.match(dockerfile, /VOLUME \["\/data\/ingestions"\]/u);
|
||||
assert.match(dockerfile, /ENV NODE_ENV=production OCR_ARTIFACT_ROOT=\/data\/ingestions/u);
|
||||
assert.match(dockerfile, /USER node/u);
|
||||
assert.match(dockerfile, /dist\/modules\/catalog\/migrations\.js.*dist\/server\.js/u);
|
||||
assert.doesNotMatch(dockerfile, /(?:COPY|ADD)\s+\.env|ENV\s+(?:OCR_INTERNAL_TOKEN|LIFECYCLE_ADMIN_TOKEN)=/u);
|
||||
});
|
||||
397
tests/ocr/dispatcher.test.ts
Normal file
397
tests/ocr/dispatcher.test.ts
Normal file
|
|
@ -0,0 +1,397 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { createApp } from "../../src/app.js";
|
||||
import { env } from "../../src/config/env.js";
|
||||
import { CatalogError } from "../../src/modules/catalog/errors.js";
|
||||
import { CatalogRepository } from "../../src/modules/catalog/repository.js";
|
||||
import { KnowledgeLifecycleReconciler } from "../../src/modules/catalog/reconciler.js";
|
||||
import { OcrDispatcher } from "../../src/modules/ocr/dispatcher.js";
|
||||
import { IngestService } from "../../src/modules/ingest/service.js";
|
||||
import type { EmbeddingProvider } from "../../src/modules/embeddings/provider.js";
|
||||
import type { VectorStoreClient } from "../../src/modules/vectorstore/client.js";
|
||||
import { sha256Hex } from "../../src/shared/utils/ids.js";
|
||||
|
||||
function setEnvFlag(name: keyof typeof env, value: unknown): void {
|
||||
(env as unknown as Record<string, unknown>)[name] = value;
|
||||
}
|
||||
|
||||
function buildPdf(text: string): Buffer {
|
||||
const stream = text ? `BT /F1 10 Tf 40 760 Td (${text.replace(/([\\()])/g, "\\$1")}) Tj ET` : "";
|
||||
const objects = [
|
||||
"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
|
||||
"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
|
||||
`<< /Length ${Buffer.byteLength(stream)} >>\nstream\n${stream}\nendstream`,
|
||||
"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"
|
||||
];
|
||||
let pdf = "%PDF-1.4\n";
|
||||
const offsets = [0];
|
||||
objects.forEach((object, index) => {
|
||||
offsets.push(Buffer.byteLength(pdf));
|
||||
pdf += `${index + 1} 0 obj\n${object}\nendobj\n`;
|
||||
});
|
||||
const xref = Buffer.byteLength(pdf);
|
||||
pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
|
||||
pdf += offsets.slice(1).map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`).join("");
|
||||
pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
|
||||
return Buffer.from(pdf);
|
||||
}
|
||||
|
||||
function embeddingProvider(): EmbeddingProvider {
|
||||
return {
|
||||
providerName: "test-provider",
|
||||
modelName: "test-model",
|
||||
dimensions: 3,
|
||||
async embed(input) { return input.map(() => [0.1, 0.2, 0.3]); }
|
||||
};
|
||||
}
|
||||
|
||||
function vectorStore(): VectorStoreClient {
|
||||
return {
|
||||
kind: "fake",
|
||||
async upsert() {},
|
||||
async countVersionPoints() { return 0; }
|
||||
} as VectorStoreClient;
|
||||
}
|
||||
|
||||
test("OCR-disabled textual PDFs retain the native synchronous path", async (context) => {
|
||||
const previousLifecycle = env.knowledgeLifecycleEnforced;
|
||||
setEnvFlag("knowledgeLifecycleEnforced", true);
|
||||
context.after(() => setEnvFlag("knowledgeLifecycleEnforced", previousLifecycle));
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "rag-ocr-disabled-"));
|
||||
context.after(() => rm(directory, { recursive: true, force: true }));
|
||||
const filePath = path.join(directory, "native.pdf");
|
||||
const sufficient = Array.from({ length: 24 }, (_, index) => `Alpha${index} beta${index}`).join(" ");
|
||||
await writeFile(filePath, buildPdf(sufficient));
|
||||
const calls: string[] = [];
|
||||
const catalog = {
|
||||
async beginAttempt() { return "attempt-1"; },
|
||||
async updateAttempt() {},
|
||||
async withSourceLock(_sourceId: string, handler: () => Promise<unknown>) { return handler(); },
|
||||
async findReusableVersion() { return { versionId: "native-1", versionNumber: 1, state: "active", previousVersionId: null, expectedDocumentCount: 1 }; },
|
||||
async assertActiveVersionPrecondition() { calls.push("native"); }
|
||||
};
|
||||
const service = new IngestService(embeddingProvider(), vectorStore(), catalog as never, {
|
||||
enabled: false,
|
||||
artifactRoot: directory
|
||||
});
|
||||
|
||||
const result = await service.ingest({ sourceType: "file", sourceRef: "native.pdf", readPath: filePath, activate: true, expectedActiveVersionId: null });
|
||||
|
||||
assert.equal(result.state, "active");
|
||||
assert.equal("phase" in result, false);
|
||||
assert.deepEqual(calls, ["native"]);
|
||||
});
|
||||
|
||||
test("OCR-required PDFs are durably accepted once and duplicate pending ingestion reuses status identity", async (context) => {
|
||||
const previousLifecycle = env.knowledgeLifecycleEnforced;
|
||||
setEnvFlag("knowledgeLifecycleEnforced", true);
|
||||
context.after(() => setEnvFlag("knowledgeLifecycleEnforced", previousLifecycle));
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "rag-ocr-accept-"));
|
||||
context.after(() => rm(directory, { recursive: true, force: true }));
|
||||
const filePath = path.join(directory, "scanned.pdf");
|
||||
await writeFile(filePath, buildPdf(""));
|
||||
let createdInput: Record<string, unknown> | undefined;
|
||||
let pending: Record<string, unknown> | undefined;
|
||||
const catalog = {
|
||||
async beginAttempt() { return "attempt-1"; },
|
||||
async updateAttempt() {},
|
||||
async withSourceLock(_sourceId: string, handler: () => Promise<unknown>) { return handler(); },
|
||||
async findPendingOcrVersion() { return pending; },
|
||||
async createPendingVersion(input: Record<string, unknown>) {
|
||||
createdInput = input;
|
||||
pending = { version: { versionId: input.versionId, versionNumber: 4, state: "indexing" }, jobs: [{ jobId: "job-1" }] };
|
||||
return (pending as { version: unknown }).version;
|
||||
},
|
||||
async markIndexing() {}
|
||||
};
|
||||
const service = new IngestService(embeddingProvider(), vectorStore(), catalog as never, {
|
||||
enabled: true,
|
||||
artifactRoot: path.join(directory, "artifacts")
|
||||
});
|
||||
const source = { sourceId: "src:scan", sourceType: "file" as const, sourceRef: "scanned.pdf", readPath: filePath, activate: true, expectedActiveVersionId: null };
|
||||
|
||||
const accepted = await service.ingest(source);
|
||||
const duplicate = await service.ingest(source);
|
||||
|
||||
assert.deepEqual(accepted, {
|
||||
accepted: true,
|
||||
sourceId: "src:scan",
|
||||
versionId: accepted.versionId,
|
||||
versionNumber: 4,
|
||||
state: "indexing",
|
||||
phase: "ocr_queued",
|
||||
statusUrl: `/ingestions/${accepted.versionId}`,
|
||||
reviewUrl: null,
|
||||
activated: false
|
||||
});
|
||||
assert.equal(duplicate.versionId, accepted.versionId);
|
||||
assert.equal((createdInput?.sourceContentHash), null);
|
||||
assert.deepEqual((createdInput?.ocrJobs as Array<{ requestedPages: number[] }>)[0]?.requestedPages, [1]);
|
||||
const originalPath = path.join(directory, "artifacts", String(accepted.versionId), "documents");
|
||||
assert.equal((await readFile(path.join(directory, "artifacts", String(accepted.versionId), "manifest.json"), "utf8")).includes("scanned.pdf"), true);
|
||||
assert.equal(path.isAbsolute(originalPath), true);
|
||||
});
|
||||
|
||||
test("OCR acceptance retains every supported native document and reloads a concurrent winner", async (context) => {
|
||||
const previousLifecycle = env.knowledgeLifecycleEnforced;
|
||||
setEnvFlag("knowledgeLifecycleEnforced", true);
|
||||
context.after(() => setEnvFlag("knowledgeLifecycleEnforced", previousLifecycle));
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "rag-ocr-multi-"));
|
||||
context.after(() => rm(directory, { recursive: true, force: true }));
|
||||
await writeFile(path.join(directory, "scanned.pdf"), buildPdf(""));
|
||||
await writeFile(path.join(directory, "notes.txt"), "Native companion document\n");
|
||||
const winner = { version: { versionId: "22222222-2222-4222-8222-222222222222", versionNumber: 7, state: "indexing" }, jobs: [{ jobId: "winner-job" }] };
|
||||
let lookupCount = 0;
|
||||
let candidateInput: Record<string, unknown> | undefined;
|
||||
const catalog = {
|
||||
async beginAttempt() { return "attempt-multi"; },
|
||||
async updateAttempt() {},
|
||||
async withSourceLock(_sourceId: string, handler: () => Promise<unknown>) { return handler(); },
|
||||
async findPendingOcrVersion() { lookupCount += 1; return lookupCount === 1 ? undefined : winner; },
|
||||
async createPendingVersion(input: Record<string, unknown>) {
|
||||
candidateInput = input;
|
||||
throw Object.assign(new Error("duplicate pending identity"), { code: "23505" });
|
||||
}
|
||||
};
|
||||
const service = new IngestService(embeddingProvider(), vectorStore(), catalog as never, { enabled: true, artifactRoot: path.join(directory, "artifacts") });
|
||||
|
||||
const accepted = await service.ingest({ sourceId: "src:multi", sourceType: "folder", sourceRef: "bundle", readPath: directory, activate: true, expectedActiveVersionId: null });
|
||||
|
||||
assert.equal(accepted.versionId, winner.version.versionId);
|
||||
assert.equal(candidateInput?.expectedDocumentCount, 2);
|
||||
assert.deepEqual((candidateInput?.documents as Array<{ documentKey: string }>).map(({ documentKey }) => documentKey), ["notes.txt", "scanned.pdf"]);
|
||||
assert.equal((candidateInput?.ocrJobs as unknown[]).length, 1);
|
||||
});
|
||||
|
||||
test("application wiring dispatches a newly accepted OCR job from durable artifacts", async (context) => {
|
||||
const previous = {
|
||||
lifecycle: env.knowledgeLifecycleEnforced,
|
||||
enabled: (env as unknown as Record<string, unknown>).ocrIngestEnabled,
|
||||
root: (env as unknown as Record<string, unknown>).ocrArtifactRoot
|
||||
};
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "rag-ocr-runtime-"));
|
||||
context.after(() => rm(directory, { recursive: true, force: true }));
|
||||
setEnvFlag("knowledgeLifecycleEnforced", true);
|
||||
setEnvFlag("ocrIngestEnabled" as keyof typeof env, true);
|
||||
setEnvFlag("ocrArtifactRoot" as keyof typeof env, path.join(directory, "artifacts"));
|
||||
context.after(() => {
|
||||
setEnvFlag("knowledgeLifecycleEnforced", previous.lifecycle);
|
||||
setEnvFlag("ocrIngestEnabled" as keyof typeof env, previous.enabled);
|
||||
setEnvFlag("ocrArtifactRoot" as keyof typeof env, previous.root);
|
||||
});
|
||||
const pdf = buildPdf("");
|
||||
let queuedJob: Record<string, unknown> | undefined;
|
||||
let persistedKey: unknown;
|
||||
const dispatchCalls: string[] = [];
|
||||
const catalog = {
|
||||
async beginAttempt() { return "runtime-attempt"; }, async updateAttempt() {},
|
||||
async withSourceLock(_sourceId: string, handler: () => Promise<unknown>) { return handler(); },
|
||||
async findPendingOcrVersion() { return undefined; },
|
||||
async createPendingVersion(input: Record<string, unknown>) {
|
||||
const job = (input.ocrJobs as Array<Record<string, unknown>>)[0]!;
|
||||
persistedKey = job.remoteIdempotencyKey;
|
||||
queuedJob = { ...job, jobId: "runtime-job", versionId: input.versionId, remoteJobId: null, state: "queued", completedPages: 0, attemptCount: 0, heartbeatAt: null, leaseExpiresAt: null, nextAttemptAt: null, errorCode: null, errorDetail: null };
|
||||
return { versionId: input.versionId, versionNumber: 3 };
|
||||
},
|
||||
async markIndexing() {},
|
||||
async claimNextOcrJob() { const job = queuedJob; queuedJob = undefined; return job ? { ...job, state: "running", attemptCount: 1 } : undefined; },
|
||||
async setOcrRemoteJob() {},
|
||||
async requeueOcrJob() { dispatchCalls.push("requeued"); },
|
||||
async completeOcrJob() { return false; }, async failOcrJob() {}, async markReviewRequired() {}, async markFailed() {}
|
||||
};
|
||||
const client = {
|
||||
async submit(bytes: Buffer, _expected: unknown, key: string) { assert.equal(key, persistedKey); dispatchCalls.push(`${bytes.length}:${key}`); return { jobId: "runtime-remote" }; },
|
||||
async getStatus() { return { jobId: "runtime-remote", status: "queued", completedPages: 0, totalPages: 1, error: null }; }
|
||||
};
|
||||
const server = createApp({ catalog: catalog as never, ocrClient: client as never, startReconciler: false }).listen(0);
|
||||
context.after(() => server.close());
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
|
||||
const form = new FormData();
|
||||
form.set("file", new Blob([new Uint8Array(pdf)], { type: "application/pdf" }), "runtime.pdf");
|
||||
form.set("sourceId", "src:runtime");
|
||||
form.set("activate", "true");
|
||||
form.set("expectedActiveVersionId", "null");
|
||||
const response = await fetch(`http://127.0.0.1:${address.port}/ingest/upload`, { method: "POST", body: form });
|
||||
assert.equal(response.status, 202);
|
||||
assert.equal((await response.json() as { uploadedResource: string }).uploadedResource, "runtime.pdf");
|
||||
for (let attempt = 0; attempt < 20 && !dispatchCalls.includes("requeued"); attempt += 1) await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
assert.equal(dispatchCalls.length, 2);
|
||||
assert.match(dispatchCalls[0]!, /^\d+:.+:ocr-v1:.+$/u);
|
||||
assert.equal(dispatchCalls[1], "requeued");
|
||||
});
|
||||
|
||||
test("dispatcher recovers only expired work, reuses its remote job, and completes it once", async () => {
|
||||
const calls: string[] = [];
|
||||
const job = {
|
||||
jobId: "job-expired",
|
||||
versionId: "version-1",
|
||||
documentId: "document-1",
|
||||
remoteJobId: "remote-1",
|
||||
remoteIdempotencyKey: "persisted-key",
|
||||
state: "queued" as const,
|
||||
requestedPages: [1],
|
||||
completedPages: 0,
|
||||
configVersion: "ocr-v1",
|
||||
attemptCount: 1,
|
||||
heartbeatAt: null,
|
||||
leaseExpiresAt: null,
|
||||
nextAttemptAt: null,
|
||||
errorCode: null,
|
||||
errorDetail: null
|
||||
};
|
||||
let available = true;
|
||||
const repository = {
|
||||
async recoverExpiredOcrLeases() { calls.push("recover"); return [job]; },
|
||||
async claimNextOcrJob() { throw new Error("recovery must not claim unrelated queued work"); },
|
||||
async claimOcrJob(jobId: string) { assert.equal(jobId, job.jobId); if (!available) return undefined; available = false; calls.push("claim-exact"); return { ...job, state: "running" as const }; },
|
||||
async setOcrRemoteJob() { calls.push("submit-persist"); },
|
||||
async requeueOcrJob() { calls.push("requeue"); },
|
||||
async completeOcrJob() { calls.push("complete"); return true; },
|
||||
async failOcrJob() { calls.push("job-failed"); },
|
||||
async markReviewRequired() { calls.push("review-required"); },
|
||||
async markFailed() { calls.push("version-failed"); }
|
||||
};
|
||||
const client = {
|
||||
async submit() { calls.push("submit"); throw new Error("existing remote work must not be duplicated"); },
|
||||
async getStatus() { calls.push("status"); return { jobId: "remote-1", status: "succeeded", completedPages: 1, totalPages: 1, error: null }; },
|
||||
async getResult() { calls.push("result"); return { pages: [{ page: 1 }] }; }
|
||||
};
|
||||
const dispatcher = new OcrDispatcher(repository, client as never, async () => ({ bytes: Buffer.from("pdf"), documentSha256: sha256Hex("pdf") }));
|
||||
|
||||
assert.equal(await dispatcher.recoverExpiredLeases(), 1);
|
||||
assert.deepEqual(calls, ["recover", "claim-exact", "status", "result", "complete", "review-required"]);
|
||||
});
|
||||
|
||||
test("OCR exhaustion fails the leased candidate without activation or engine substitution", async () => {
|
||||
const calls: string[] = [];
|
||||
const repository = {
|
||||
async claimNextOcrJob() {
|
||||
return { jobId: "job-1", versionId: "version-1", documentId: "document-1", remoteJobId: null, remoteIdempotencyKey: "same-key", state: "running", requestedPages: [1], completedPages: 0, configVersion: "ocr-v1", attemptCount: 1, heartbeatAt: null, leaseExpiresAt: null, nextAttemptAt: null, errorCode: null, errorDetail: null };
|
||||
},
|
||||
async setOcrRemoteJob() { calls.push("remote"); },
|
||||
async requeueOcrJob() { calls.push("requeue"); },
|
||||
async completeOcrJob() { calls.push("complete"); return false; },
|
||||
async failOcrJob(_jobId: string, code: string) { calls.push(`job:${code}`); },
|
||||
async markReviewRequired() { calls.push("review"); },
|
||||
async markFailed(_versionId: string, code: string) { calls.push(`version:${code}`); }
|
||||
};
|
||||
const client = { async submit() { throw new Error("OCR unavailable after retries"); } };
|
||||
const dispatcher = new OcrDispatcher(repository as never, client as never, async () => ({ bytes: Buffer.from("pdf"), documentSha256: sha256Hex("pdf") }));
|
||||
|
||||
assert.equal(await dispatcher.runOnce(), "failed");
|
||||
assert.deepEqual(calls, ["job:OCR_DISPATCH_FAILED", "version:OCR_DISPATCH_FAILED"]);
|
||||
});
|
||||
|
||||
test("reconciler makes expired OCR leases dispatchable while leaving live leases to PostgreSQL", async () => {
|
||||
const calls: string[] = [];
|
||||
const catalog = {
|
||||
async withGlobalTryLock(_name: string, handler: () => Promise<unknown>) { return handler(); },
|
||||
async resolveActiveVersions() { return []; },
|
||||
async validateActiveInvariant() { return []; },
|
||||
async listOrphanedIndexingCandidates() { return []; }
|
||||
};
|
||||
const dispatcher = {
|
||||
async recoverExpiredLeases() { calls.push("ocr-recovery"); return 1; },
|
||||
async dispatchAvailable() { calls.push("ocr-dispatch"); return 2; }
|
||||
};
|
||||
const reconciler = new KnowledgeLifecycleReconciler(catalog as never, vectorStore(), dispatcher as never);
|
||||
|
||||
const status = await reconciler.runOnce();
|
||||
|
||||
assert.equal(status.ocrLeasesRecovered, 1);
|
||||
assert.deepEqual(calls, ["ocr-recovery", "ocr-dispatch"]);
|
||||
});
|
||||
|
||||
test("runtime HTTP routing returns native 201, OCR 202/status, and catalog-down 503", async () => {
|
||||
const previous = { knowledgeLifecycleEnforced: env.knowledgeLifecycleEnforced, lifecycleAdminToken: env.lifecycleAdminToken, ocrIngestEnabled: env.ocrIngestEnabled };
|
||||
setEnvFlag("knowledgeLifecycleEnforced", true);
|
||||
setEnvFlag("lifecycleAdminToken", "unit-8-token");
|
||||
setEnvFlag("ocrIngestEnabled", true);
|
||||
const versionId = "11111111-1111-4111-8111-111111111111";
|
||||
const ingestService = {
|
||||
async ingest(input: Record<string, unknown>) {
|
||||
return input.sourceRef === "native.pdf"
|
||||
? { accepted: true, sourceId: "src:native", versionId: "native-1", versionNumber: 1, state: "active", filesDiscovered: 1, documentsProcessed: 1, chunksStored: 1, activated: true, noOp: false, collectionName: "rag" }
|
||||
: { accepted: true, sourceId: "src:scan", versionId, versionNumber: 2, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false };
|
||||
},
|
||||
async cleanup() { return { deleted: 0 }; }
|
||||
};
|
||||
const catalog = {
|
||||
async getIngestionStatus(id: string) {
|
||||
assert.equal(id, versionId);
|
||||
return { sourceId: "src:scan", versionId, state: "indexing", phase: "ocr_running", activated: false, documents: [{ documentId: "doc:scan", state: "ocr_running", completedPages: 0, totalPages: 1, pages: [{ page: 1, method: "ocr", state: "ocr_running" }] }], error: null, statusUrl: `/ingestions/${versionId}`, reviewUrl: null };
|
||||
}
|
||||
};
|
||||
const app = createApp({ ingestService: ingestService as never, catalog: catalog as never, startReconciler: false });
|
||||
const server = app.listen(0);
|
||||
try {
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
const baseUrl = `http://127.0.0.1:${address.port}`;
|
||||
const request = (sourceRef: string) => fetch(`${baseUrl}/ingest`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ sourceType: "file", sourceRef }) });
|
||||
|
||||
assert.equal((await request("native.pdf")).status, 201);
|
||||
const accepted = await request("scan.pdf");
|
||||
assert.equal(accepted.status, 202);
|
||||
assert.equal((await accepted.json() as { statusUrl: string }).statusUrl, `/ingestions/${versionId}`);
|
||||
assert.equal((await fetch(`${baseUrl}/ingestions/${versionId}`)).status, 401);
|
||||
const status = await fetch(`${baseUrl}/ingestions/${versionId}`, { headers: { authorization: "Bearer unit-8-token" } });
|
||||
assert.equal(status.status, 200);
|
||||
assert.equal((await status.json() as { phase: string }).phase, "ocr_running");
|
||||
|
||||
const unavailable = createApp({ ingestService: ingestService as never, catalog: undefined, startReconciler: false }).listen(0);
|
||||
try {
|
||||
const unavailableAddress = unavailable.address();
|
||||
assert.ok(unavailableAddress && typeof unavailableAddress === "object");
|
||||
const response = await fetch(`http://127.0.0.1:${unavailableAddress.port}/ingestions/${versionId}`, { headers: { authorization: "Bearer unit-8-token" } });
|
||||
assert.equal(response.status, 503);
|
||||
assert.equal((await response.json() as { code: string }).code, "CATALOG_UNAVAILABLE");
|
||||
const retrieval = await fetch(`http://127.0.0.1:${unavailableAddress.port}/retrieve`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ query: "previous active content", mode: "documental", intent: "specific" })
|
||||
});
|
||||
assert.equal(retrieval.status, 503);
|
||||
assert.equal((await retrieval.json() as { code: string }).code, "CATALOG_UNAVAILABLE");
|
||||
} finally {
|
||||
unavailable.close();
|
||||
}
|
||||
} finally {
|
||||
server.close();
|
||||
setEnvFlag("knowledgeLifecycleEnforced", previous.knowledgeLifecycleEnforced);
|
||||
setEnvFlag("lifecycleAdminToken", previous.lifecycleAdminToken);
|
||||
setEnvFlag("ocrIngestEnabled", previous.ocrIngestEnabled);
|
||||
}
|
||||
});
|
||||
|
||||
test("status reports all native and OCR documents and fails the version when one document fails", async () => {
|
||||
let query = 0;
|
||||
const pool = { async query() {
|
||||
query += 1;
|
||||
if (query === 1) return { rowCount: 1, rows: [{ source_id: "src:mixed", state: "failed", error_code: "OCR_QUALITY_BLOCKED", error_detail: "Visible ink failed OCR quality" }] };
|
||||
if (query === 2) return { rowCount: 2, rows: [
|
||||
{ document_id: "doc:native", index_state: "indexing", job_state: null, completed_pages: null, requested_pages: null },
|
||||
{ document_id: "doc:ocr", index_state: "failed", job_state: "failed", completed_pages: 0, requested_pages: [2] }
|
||||
] };
|
||||
return { rowCount: 3, rows: [
|
||||
{ document_id: "doc:native", page_number: 1, extraction_method: "native", blocked_reason: null },
|
||||
{ document_id: "doc:ocr", page_number: 1, extraction_method: "native", blocked_reason: null },
|
||||
{ document_id: "doc:ocr", page_number: 2, extraction_method: "ocr", blocked_reason: "OCR_QUALITY_BLOCKED" }
|
||||
] };
|
||||
} };
|
||||
const status = await new CatalogRepository(pool as never).getIngestionStatus("version-mixed") as {
|
||||
phase: string; documents: Array<{ documentId: string; completedPages: number; totalPages: number }>; error: { code: string }
|
||||
};
|
||||
|
||||
assert.equal(status.phase, "failed");
|
||||
assert.deepEqual(status.documents.map(({ documentId, completedPages, totalPages }) => [documentId, completedPages, totalPages]), [
|
||||
["doc:native", 1, 1], ["doc:ocr", 1, 2]
|
||||
]);
|
||||
assert.equal(status.error.code, "OCR_QUALITY_BLOCKED");
|
||||
});
|
||||
183
tests/ocr/e2e.test.ts
Normal file
183
tests/ocr/e2e.test.ts
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { createApp } from "../../src/app.js";
|
||||
import { env } from "../../src/config/env.js";
|
||||
import { OcrDispatcher } from "../../src/modules/ocr/dispatcher.js";
|
||||
import { OcrIndexingService } from "../../src/modules/ocr/indexing.js";
|
||||
import { OcrReviewService, type OcrReviewCandidate } from "../../src/modules/ocr/review.js";
|
||||
import { sha256Hex } from "../../src/shared/utils/ids.js";
|
||||
|
||||
const token = "local-e2e-token";
|
||||
const scanVersion = "11111111-1111-4111-8111-111111111111";
|
||||
const mixedVersion = "22222222-2222-4222-8222-222222222222";
|
||||
|
||||
function headers(json = false): Record<string, string> {
|
||||
return { authorization: `Bearer ${token}`, ...(json ? { "content-type": "application/json" } : {}) };
|
||||
}
|
||||
|
||||
function candidate(): OcrReviewCandidate {
|
||||
const text = "CBGO4a";
|
||||
return {
|
||||
versionId: scanVersion,
|
||||
sourceId: "src:scan",
|
||||
state: "review_required",
|
||||
candidateSha256: "candidate-hash",
|
||||
baseActiveVersionId: "previous-active",
|
||||
currentActiveVersionId: "previous-active",
|
||||
activateRequested: true,
|
||||
processingFingerprint: "fingerprint",
|
||||
metadataHash: "metadata",
|
||||
documents: [{
|
||||
documentId: "doc:scan",
|
||||
pages: [{
|
||||
page: 1,
|
||||
imageUrl: `/ingestions/${scanVersion}/documents/doc:scan/pages/1/image`,
|
||||
nativeText: "",
|
||||
ocr: { text, lines: [{ lineId: "line-1", text, confidence: 0.72, bbox: [1, 2, 30, 12], lineSha256: sha256Hex(text) }] },
|
||||
candidateText: text,
|
||||
differences: ["native text is empty"],
|
||||
risks: [text]
|
||||
}]
|
||||
}]
|
||||
};
|
||||
}
|
||||
|
||||
test("local HTTP flow keeps native ingestion synchronous and activates only reviewed OCR content", async (context) => {
|
||||
const previous = { lifecycle: env.knowledgeLifecycleEnforced, enabled: env.ocrIngestEnabled, token: env.lifecycleAdminToken };
|
||||
Object.assign(env, { knowledgeLifecycleEnforced: true, ocrIngestEnabled: true, lifecycleAdminToken: token });
|
||||
context.after(() => Object.assign(env, {
|
||||
knowledgeLifecycleEnforced: previous.lifecycle,
|
||||
ocrIngestEnabled: previous.enabled,
|
||||
lifecycleAdminToken: previous.token
|
||||
}));
|
||||
let reviewCandidate = candidate();
|
||||
let reviewedText = "";
|
||||
const states = new Map([[scanVersion, "review_required"], [mixedVersion, "review_required"]]);
|
||||
const ingestService = {
|
||||
async ingest(input: { sourceRef: string }) {
|
||||
if (input.sourceRef === "native.pdf") {
|
||||
return { accepted: true, sourceId: "src:native", versionId: "native-active", versionNumber: 1, state: "active", filesDiscovered: 1, documentsProcessed: 1, chunksStored: 1, activated: true, noOp: false, collectionName: "rag" };
|
||||
}
|
||||
const versionId = input.sourceRef === "mixed.pdf" ? mixedVersion : scanVersion;
|
||||
return { accepted: true, sourceId: `src:${input.sourceRef}`, versionId, versionNumber: 2, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false };
|
||||
},
|
||||
async cleanup() { return { deleted: 0 }; }
|
||||
};
|
||||
const catalog = {
|
||||
async claimNextOcrJob() { return undefined; },
|
||||
async getIngestionStatus(versionId: string) {
|
||||
const state = states.get(versionId);
|
||||
if (!state) return undefined;
|
||||
const mixed = versionId === mixedVersion;
|
||||
return {
|
||||
sourceId: mixed ? "src:mixed.pdf" : "src:scan.pdf", versionId, state, phase: state,
|
||||
activated: state === "active",
|
||||
documents: mixed
|
||||
? [{ documentId: "doc:mixed", state: "ocr_complete", completedPages: 2, totalPages: 2, pages: [
|
||||
{ page: 1, method: "native", state: "native_complete" }, { page: 2, method: "ocr", state: "ocr_complete" }
|
||||
] }]
|
||||
: [{ documentId: "doc:scan", state: "ocr_complete", completedPages: 1, totalPages: 1, pages: [{ page: 1, method: "ocr", state: "ocr_complete" }] }],
|
||||
error: null, statusUrl: `/ingestions/${versionId}`, reviewUrl: state === "review_required" ? `/ingestions/${versionId}/review` : null
|
||||
};
|
||||
}
|
||||
};
|
||||
const review = new OcrReviewService({
|
||||
async loadCandidate(versionId) { return versionId === scanVersion ? reviewCandidate : undefined; },
|
||||
async commitApproval(input) {
|
||||
reviewedText = input.reviewedText;
|
||||
reviewCandidate = { ...reviewCandidate, state: "indexing" };
|
||||
states.set(scanVersion, "indexing");
|
||||
},
|
||||
async commitRejection() { throw new Error("rejection is outside this flow"); }
|
||||
});
|
||||
const indexing = new OcrIndexingService({
|
||||
async findReusableVersion() { return undefined; },
|
||||
async indexReviewed(value) { assert.equal(value.reviewedText, "CBG04a"); return 1; },
|
||||
async markReady(versionId) { states.set(versionId, "ready"); },
|
||||
async settleReusable() { throw new Error("candidate is not reusable"); },
|
||||
async activateVersion(_sourceId, versionId, expected) {
|
||||
assert.equal(expected, "previous-active");
|
||||
states.set(versionId, "active");
|
||||
return versionId;
|
||||
}
|
||||
});
|
||||
const server = createApp({ ingestService: ingestService as never, catalog: catalog as never, ocrClient: {} as never, reviewService: review, indexingService: indexing, startReconciler: false }).listen(0);
|
||||
context.after(() => server.close());
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
const base = `http://127.0.0.1:${address.port}`;
|
||||
const ingest = (sourceRef: string) => fetch(`${base}/ingest`, { method: "POST", headers: headers(true), body: JSON.stringify({ sourceType: "file", sourceRef }) });
|
||||
|
||||
assert.equal((await ingest("native.pdf")).status, 201);
|
||||
const accepted = await ingest("scanned.pdf");
|
||||
assert.equal(accepted.status, 202);
|
||||
assert.deepEqual(await accepted.json(), {
|
||||
accepted: true, sourceId: "src:scanned.pdf", versionId: scanVersion, versionNumber: 2,
|
||||
state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${scanVersion}`, reviewUrl: null, activated: false
|
||||
});
|
||||
const reviewResponse = await fetch(`${base}/ingestions/${scanVersion}/review`, { headers: headers() });
|
||||
assert.equal(reviewResponse.status, 200);
|
||||
assert.deepEqual((await reviewResponse.json() as OcrReviewCandidate).documents[0]?.pages[0]?.risks, ["CBGO4a"]);
|
||||
const approval = await fetch(`${base}/ingestions/${scanVersion}/approve`, {
|
||||
method: "POST", headers: headers(true), body: JSON.stringify({
|
||||
candidateSha256: "candidate-hash", expectedActiveVersionId: "previous-active", reviewedBy: "local-reviewer",
|
||||
corrections: [{ documentId: "doc:scan", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("CBGO4a"), replacementText: "CBG04a" }]
|
||||
})
|
||||
});
|
||||
assert.equal(approval.status, 200);
|
||||
assert.deepEqual(await approval.json(), { versionId: scanVersion, state: "active", activated: true, activatedVersionId: scanVersion });
|
||||
assert.equal(reviewedText, "CBG04a");
|
||||
assert.equal((await fetch(`${base}/ingestions/${scanVersion}`, { headers: headers() }).then((response) => response.json()) as { state: string }).state, "active");
|
||||
|
||||
assert.equal((await ingest("mixed.pdf")).status, 202);
|
||||
const mixed = await fetch(`${base}/ingestions/${mixedVersion}`, { headers: headers() }).then((response) => response.json()) as { documents: Array<{ pages: Array<{ method: string }> }> };
|
||||
assert.deepEqual(mixed.documents[0]?.pages.map(({ method }) => method), ["native", "ocr"]);
|
||||
});
|
||||
|
||||
test("resends reuse identity, OCR exhaustion fails closed, and catalog absence returns 503", async (context) => {
|
||||
const previous = { lifecycle: env.knowledgeLifecycleEnforced, enabled: env.ocrIngestEnabled, token: env.lifecycleAdminToken };
|
||||
Object.assign(env, { knowledgeLifecycleEnforced: true, ocrIngestEnabled: true, lifecycleAdminToken: token });
|
||||
context.after(() => Object.assign(env, {
|
||||
knowledgeLifecycleEnforced: previous.lifecycle,
|
||||
ocrIngestEnabled: previous.enabled,
|
||||
lifecycleAdminToken: previous.token
|
||||
}));
|
||||
const identities = new Map<string, string>();
|
||||
let creations = 0;
|
||||
const ingestService = {
|
||||
async ingest(input: { sourceRef: string }) {
|
||||
let versionId = identities.get(input.sourceRef);
|
||||
if (!versionId) { versionId = input.sourceRef === "scan.pdf" ? scanVersion : mixedVersion; identities.set(input.sourceRef, versionId); creations += 1; }
|
||||
return { accepted: true, sourceId: `src:${input.sourceRef}`, versionId, versionNumber: 1, state: "indexing", phase: "ocr_queued", statusUrl: `/ingestions/${versionId}`, reviewUrl: null, activated: false };
|
||||
},
|
||||
async cleanup() { return { deleted: 0 }; }
|
||||
};
|
||||
const server = createApp({ ingestService: ingestService as never, catalog: undefined, startReconciler: false }).listen(0);
|
||||
context.after(() => server.close());
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
const base = `http://127.0.0.1:${address.port}`;
|
||||
const resend = () => fetch(`${base}/ingest`, { method: "POST", headers: headers(true), body: JSON.stringify({ sourceType: "file", sourceRef: "scan.pdf" }) }).then((response) => response.json()) as Promise<{ versionId: string }>;
|
||||
assert.deepEqual([(await resend()).versionId, (await resend()).versionId], [scanVersion, scanVersion]);
|
||||
assert.equal(creations, 1);
|
||||
|
||||
const calls: string[] = [];
|
||||
let candidateState = "indexing";
|
||||
const dispatcher = new OcrDispatcher({
|
||||
async claimNextOcrJob() { return { jobId: "job", versionId: scanVersion, documentId: "doc", remoteJobId: null, remoteIdempotencyKey: "stable-key", state: "running", requestedPages: [1], completedPages: 0, configVersion: "ocr-v1", attemptCount: 1, heartbeatAt: null, leaseExpiresAt: null, nextAttemptAt: null, errorCode: null, errorDetail: null }; },
|
||||
async claimOcrJob() { return undefined; }, async recoverExpiredOcrLeases() { return []; }, async setOcrRemoteJob() { calls.push("remote"); },
|
||||
async requeueOcrJob() { calls.push("requeue"); }, async completeOcrJob() { calls.push("complete"); return false; },
|
||||
async failOcrJob() { calls.push("job-failed"); }, async markReviewRequired() { calls.push("review"); },
|
||||
async markFailed() { candidateState = "failed"; calls.push("version-failed"); }
|
||||
}, { async submit() { throw new Error("OCR connection unavailable after bounded retries"); } } as never, async () => ({ bytes: Buffer.from("pdf"), documentSha256: sha256Hex("pdf") }));
|
||||
assert.equal(await dispatcher.runOnce(), "failed");
|
||||
assert.deepEqual(calls, ["job-failed", "version-failed"]);
|
||||
assert.equal(candidateState, "failed");
|
||||
|
||||
const unavailableStatus = await fetch(`${base}/ingestions/${scanVersion}`, { headers: headers() });
|
||||
assert.equal(unavailableStatus.status, 503);
|
||||
assert.equal((await unavailableStatus.json() as { code: string }).code, "CATALOG_UNAVAILABLE");
|
||||
const unavailableRetrieval = await fetch(`${base}/retrieve`, { method: "POST", headers: headers(true), body: JSON.stringify({ query: "prior active", mode: "documental", intent: "specific" }) });
|
||||
assert.equal(unavailableRetrieval.status, 503);
|
||||
assert.equal((await unavailableRetrieval.json() as { code: string }).code, "CATALOG_UNAVAILABLE");
|
||||
});
|
||||
134
tests/ocr/retention.test.ts
Normal file
134
tests/ocr/retention.test.ts
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { access, mkdir, mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { createApp } from "../../src/app.js";
|
||||
import { env } from "../../src/config/env.js";
|
||||
import { KnowledgeLifecycleReconciler } from "../../src/modules/catalog/reconciler.js";
|
||||
import { CatalogRepository } from "../../src/modules/catalog/repository.js";
|
||||
import { OcrRetentionService, type OcrRetentionCandidate, type OcrRetentionStore } from "../../src/modules/ocr/retention.js";
|
||||
|
||||
const activeId = "11111111-1111-4111-8111-111111111111";
|
||||
const failedId = "22222222-2222-4222-8222-222222222222";
|
||||
const deletingId = "33333333-3333-4333-8333-333333333333";
|
||||
|
||||
function candidate(versionId: string, state: OcrRetentionCandidate["state"], artifactState: OcrRetentionCandidate["artifactState"] = "present"): OcrRetentionCandidate {
|
||||
return { versionId, sourceId: `source:${versionId}`, state, artifactState };
|
||||
}
|
||||
|
||||
test("repository retention selection applies exact TTLs and CAS-protects active versions", async () => {
|
||||
const queries: Array<{ sql: string; params?: unknown[] }> = [];
|
||||
const pool = { async query(sql: string, params?: unknown[]) {
|
||||
queries.push({ sql, params });
|
||||
if (sql.includes("SELECT v.version_id")) return { rowCount: 1, rows: [{ version_id: failedId, source_id: "source:failed", state: "failed", artifact_state: "present" }] };
|
||||
return { rowCount: 1, rows: [] };
|
||||
} };
|
||||
const repository = new CatalogRepository(pool as never);
|
||||
const now = new Date("2026-09-15T12:00:00.000Z");
|
||||
|
||||
assert.deepEqual(await repository.listOcrRetentionCandidates(now), [{ ...candidate(failedId, "failed"), sourceId: "source:failed" }]);
|
||||
assert.equal(await repository.expireOcrReview("source:review", activeId, now), true);
|
||||
assert.equal(await repository.claimOcrRetentionDeletion("source:failed", failedId, "failed", "present"), true);
|
||||
assert.equal(await repository.completeOcrRetentionDeletion("source:failed", failedId, "failed"), true);
|
||||
|
||||
assert.match(queries[0]!.sql, /review_required.*30 days.*failed.*rejected.*7 days.*superseded.*30 days/s);
|
||||
assert.match(queries[0]!.sql, /active_version_id IS DISTINCT FROM v\.version_id/);
|
||||
assert.deepEqual(queries[0]!.params, [now]);
|
||||
assert.match(queries[1]!.sql, /state = 'rejected'.*REVIEW_EXPIRED.*retention_due_at/s);
|
||||
assert.match(queries[2]!.sql, /SET artifact_state = 'retention_deleting'.*state = \$3.*artifact_state = \$4/s);
|
||||
assert.match(queries[2]!.sql, /active_version_id IS DISTINCT FROM v\.version_id/);
|
||||
assert.match(queries[3]!.sql, /artifact_state = 'retention_deleted'.*artifact_state = 'retention_deleting'/s);
|
||||
});
|
||||
|
||||
test("activation cannot race an artifact deletion already claimed by retention", async () => {
|
||||
const pool = {
|
||||
async query(sql: string) {
|
||||
if (sql.includes("maintenance")) return { rowCount: 1, rows: [{ maintenance: false }] };
|
||||
if (sql.includes("FROM rag_sources")) return { rowCount: 1, rows: [{ active_version_id: null }] };
|
||||
return { rowCount: 1, rows: [{ version_id: failedId, source_id: "source:failed", version_number: 2, previous_version_id: null, state: "ready", tags: [], source_content_hash: "hash", processing_fingerprint: "fingerprint", metadata_hash: "metadata", embedding_provider: "test", embedding_model: "test", embedding_dimensions: 3, expected_document_count: 1, expected_point_count: 1, verified_point_count: 1, qdrant_collection: "rag", artifact_state: "retention_deleting" }] };
|
||||
},
|
||||
async connect() { return { query: this.query, release() {} }; }
|
||||
};
|
||||
const repository = new CatalogRepository(pool as never);
|
||||
|
||||
await assert.rejects(repository.activateVersion("source:failed", failedId, null), (error) => error instanceof Error && (error as { code?: string }).code === "VERSION_ARTIFACTS_UNAVAILABLE");
|
||||
});
|
||||
|
||||
test("retention expires review before removal and preserves active artifacts", async (context) => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "rag-retention-"));
|
||||
context.after(() => rm(root, { recursive: true, force: true }));
|
||||
await Promise.all([activeId, failedId].map((id) => mkdir(path.join(root, id))));
|
||||
const calls: string[] = [];
|
||||
const store: OcrRetentionStore = {
|
||||
async listOcrRetentionCandidates() { return [candidate(activeId, "active"), candidate(failedId, "failed"), candidate(deletingId, "review_required")]; },
|
||||
async withVersionTryLock(versionId, handler) { calls.push(`lock:${versionId}`); return handler(); },
|
||||
async expireOcrReview(_sourceId, versionId) { calls.push(`expire:${versionId}`); return true; },
|
||||
async claimOcrRetentionDeletion(_sourceId, versionId) { calls.push(`claim:${versionId}`); return true; },
|
||||
async completeOcrRetentionDeletion(_sourceId, versionId) { calls.push(`complete:${versionId}`); return true; }
|
||||
};
|
||||
|
||||
assert.deepEqual(await new OcrRetentionService(store, root).runOnce(), { expired: 1, deleted: 1, resumed: 0, skipped: 1 });
|
||||
await access(path.join(root, activeId));
|
||||
await assert.rejects(access(path.join(root, failedId)));
|
||||
assert.equal(calls.some((call) => call.includes(activeId)), false);
|
||||
assert.equal(calls.includes(`expire:${deletingId}`), true);
|
||||
});
|
||||
|
||||
test("interrupted deletion resumes idempotently without affecting another version", async (context) => {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "rag-retention-resume-"));
|
||||
context.after(() => rm(root, { recursive: true, force: true }));
|
||||
await Promise.all([deletingId, activeId].map((id) => mkdir(path.join(root, id))));
|
||||
let completed = false;
|
||||
let failCompletion = true;
|
||||
const store: OcrRetentionStore = {
|
||||
async listOcrRetentionCandidates() { return completed ? [] : [candidate(deletingId, "rejected", "retention_deleting")]; },
|
||||
async withVersionTryLock(_versionId, handler) { return handler(); }, async expireOcrReview() { return false; },
|
||||
async claimOcrRetentionDeletion() { return true; },
|
||||
async completeOcrRetentionDeletion() { if (failCompletion) { failCompletion = false; throw new Error("simulated restart"); } completed = true; return true; }
|
||||
};
|
||||
const retention = new OcrRetentionService(store, root);
|
||||
|
||||
await assert.rejects(retention.runOnce(), /simulated restart/);
|
||||
await assert.rejects(access(path.join(root, deletingId)));
|
||||
assert.deepEqual(await retention.runOnce(), { expired: 0, deleted: 0, resumed: 1, skipped: 0 });
|
||||
assert.deepEqual(await retention.runOnce(), { expired: 0, deleted: 0, resumed: 0, skipped: 0 });
|
||||
await access(path.join(root, activeId));
|
||||
});
|
||||
|
||||
test("OCR flag off keeps native HTTP synchronous and hides candidate surfaces", async (context) => {
|
||||
const previous = { enabled: env.ocrIngestEnabled, lifecycle: env.knowledgeLifecycleEnforced, token: env.lifecycleAdminToken };
|
||||
Object.assign(env, { ocrIngestEnabled: false, knowledgeLifecycleEnforced: true, lifecycleAdminToken: "retention-token" });
|
||||
context.after(() => Object.assign(env, previous));
|
||||
let candidateReads = 0;
|
||||
const native = { accepted: true, sourceId: "source:native", versionId: "native-1", state: "active" };
|
||||
const app = createApp({
|
||||
ingestService: { async ingest() { return native; }, async cleanup() { return { deleted: 0 }; } } as never,
|
||||
catalog: { async getIngestionStatus() { candidateReads += 1; return {}; } } as never,
|
||||
reviewService: { async view() { candidateReads += 1; return {}; }, async approve() { candidateReads += 1; return {} as never; }, async reject() { candidateReads += 1; return {} as never; } },
|
||||
indexingService: { async index() { candidateReads += 1; return {} as never; } }, startReconciler: false
|
||||
});
|
||||
const server = app.listen(0);
|
||||
context.after(() => server.close());
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
const base = `http://127.0.0.1:${address.port}`;
|
||||
|
||||
assert.equal((await fetch(`${base}/ingest`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" })).status, 201);
|
||||
for (const [suffix, method] of [["", "GET"], ["/review", "GET"], ["/approve", "POST"], ["/reject", "POST"]] as const) {
|
||||
assert.equal((await fetch(`${base}/ingestions/${failedId}${suffix}`, { method, headers: { authorization: "Bearer retention-token", "content-type": "application/json" }, body: method === "POST" ? "{}" : undefined })).status, 404);
|
||||
}
|
||||
assert.equal(candidateReads, 0);
|
||||
});
|
||||
|
||||
test("reconciler runs retention inside its exclusive global lock", async () => {
|
||||
let locked = false;
|
||||
const catalog = {
|
||||
async withGlobalTryLock(_name: string, handler: () => Promise<unknown>) { locked = true; try { return await handler(); } finally { locked = false; } },
|
||||
async resolveActiveVersions() { return []; }, async validateActiveInvariant() { return []; }, async listOrphanedIndexingCandidates() { return []; }
|
||||
};
|
||||
const retention = { async runOnce() { assert.equal(locked, true); return { expired: 0, deleted: 0, resumed: 0, skipped: 0 }; } };
|
||||
const reconciler = new KnowledgeLifecycleReconciler(catalog as never, {} as never, undefined, retention);
|
||||
|
||||
assert.equal((await reconciler.runOnce()).ocrRetentionDeleted, 0);
|
||||
});
|
||||
185
tests/ocr/review.test.ts
Normal file
185
tests/ocr/review.test.ts
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { createApp } from "../../src/app.js";
|
||||
import { env } from "../../src/config/env.js";
|
||||
import { CatalogError } from "../../src/modules/catalog/errors.js";
|
||||
import { OcrIndexingService, type ApprovedOcrCandidate, type OcrIndexingStore } from "../../src/modules/ocr/indexing.js";
|
||||
import { OcrReviewService, type OcrReviewCandidate } from "../../src/modules/ocr/review.js";
|
||||
import { sha256Hex } from "../../src/shared/utils/ids.js";
|
||||
|
||||
function candidate(state: OcrReviewCandidate["state"] = "review_required"): OcrReviewCandidate {
|
||||
const lines = ["CBGO4a", "FATo7"].map((text, index) => ({
|
||||
lineId: `line-${index + 1}`, text, confidence: 0.7, bbox: [0, index * 10, 50, index * 10 + 8] as [number, number, number, number], lineSha256: sha256Hex(text)
|
||||
}));
|
||||
return {
|
||||
versionId: "version-2", sourceId: "source-1", state, candidateSha256: "candidate-hash",
|
||||
baseActiveVersionId: "version-1", currentActiveVersionId: "version-1", activateRequested: true,
|
||||
processingFingerprint: "fingerprint", metadataHash: "metadata", documents: [{ documentId: "document-1", pages: [{
|
||||
page: 1, imageUrl: "/private/page-1.png", nativeText: "", ocr: { text: "CBGO4a\nFATo7", lines },
|
||||
candidateText: "CBGO4a\nFATo7", differences: ["native text is empty"], risks: ["CBGO4a", "FATo7"]
|
||||
}] }]
|
||||
};
|
||||
}
|
||||
|
||||
test("review HTTP rejects unauthorized and premature access without exposing artifacts", async (context) => {
|
||||
const previous = { token: env.lifecycleAdminToken, enabled: env.ocrIngestEnabled };
|
||||
Object.assign(env, { lifecycleAdminToken: "review-token", ocrIngestEnabled: true });
|
||||
context.after(() => Object.assign(env, { lifecycleAdminToken: previous.token, ocrIngestEnabled: previous.enabled }));
|
||||
let value = candidate();
|
||||
let reads = 0;
|
||||
const review = new OcrReviewService({ async loadCandidate() { reads += 1; return value; }, async commitApproval() { throw new Error("not called"); } });
|
||||
const server = createApp({ reviewService: review, startReconciler: false }).listen(0);
|
||||
context.after(() => server.close());
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
const url = `http://127.0.0.1:${address.port}/ingestions/version-2/review`;
|
||||
|
||||
const unauthorized = await fetch(url);
|
||||
assert.equal(unauthorized.status, 401);
|
||||
assert.equal((await fetch(url.replace("/review", "/approve"), { method: "POST" })).status, 401);
|
||||
assert.equal(reads, 0);
|
||||
value = candidate("indexing");
|
||||
const premature = await fetch(url, { headers: { authorization: "Bearer review-token" } });
|
||||
assert.equal(premature.status, 409);
|
||||
assert.deepEqual(await premature.json(), { ok: false, error: "Version is not awaiting OCR review", code: "INVALID_VERSION_STATE" });
|
||||
});
|
||||
|
||||
test("review exposes audit detail and commits current corrections as one immutable set", async () => {
|
||||
const commits: unknown[] = [];
|
||||
const service = new OcrReviewService({ async loadCandidate() { return candidate(); }, async commitApproval(value) { commits.push(value); } });
|
||||
const review = await service.view("version-2");
|
||||
assert.deepEqual(review.documents[0]?.pages[0]?.ocr.lines.map(({ text, confidence, bbox }) => [text, confidence, bbox]), [
|
||||
["CBGO4a", 0.7, [0, 0, 50, 8]], ["FATo7", 0.7, [0, 10, 50, 18]]
|
||||
]);
|
||||
assert.deepEqual(review.documents[0]?.pages[0]?.risks, ["CBGO4a", "FATo7"]);
|
||||
|
||||
const approved = await service.approve("version-2", {
|
||||
candidateSha256: "candidate-hash", expectedActiveVersionId: "version-1", reviewedBy: "admin",
|
||||
corrections: [
|
||||
{ documentId: "document-1", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("CBGO4a"), replacementText: "CBG04a" },
|
||||
{ documentId: "document-1", page: 1, lineId: "line-2", expectedLineSha256: sha256Hex("FATo7"), replacementText: "FAT07" }
|
||||
]
|
||||
});
|
||||
assert.equal(commits.length, 1);
|
||||
assert.deepEqual((commits[0] as { corrections: Array<{ replacementText: string }> }).corrections.map(({ replacementText }) => replacementText), ["CBG04a", "FAT07"]);
|
||||
assert.equal(approved.reviewedText, "CBG04a\nFAT07");
|
||||
assert.equal(approved.reviewedTextSha256, sha256Hex("CBG04a\nFAT07"));
|
||||
});
|
||||
|
||||
test("stale or duplicate corrections conflict before any correction or transition", async () => {
|
||||
let commits = 0;
|
||||
const service = new OcrReviewService({ async loadCandidate() { return candidate(); }, async commitApproval() { commits += 1; } });
|
||||
const base = { candidateSha256: "stale", expectedActiveVersionId: "version-1", reviewedBy: "admin", corrections: [] };
|
||||
await assert.rejects(service.approve("version-2", base), (error) => error instanceof CatalogError && error.statusCode === 409);
|
||||
const duplicate = { ...base, candidateSha256: "candidate-hash", corrections: Array(2).fill({
|
||||
documentId: "document-1", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("CBGO4a"), replacementText: "CBG04a"
|
||||
}) };
|
||||
await assert.rejects(service.approve("version-2", duplicate), (error) => error instanceof CatalogError && error.code === "CORRECTION_CONFLICT");
|
||||
await assert.rejects(service.approve("version-2", { ...base, candidateSha256: "candidate-hash", expectedActiveVersionId: "version-old" }),
|
||||
(error) => error instanceof CatalogError && error.code === "ACTIVE_VERSION_CHANGED");
|
||||
await assert.rejects(service.approve("version-2", { ...base, candidateSha256: "candidate-hash", corrections: [{
|
||||
documentId: "document-1", page: 1, lineId: "line-1", expectedLineSha256: sha256Hex("stale"), replacementText: "CBG04a"
|
||||
}] }), (error) => error instanceof CatalogError && error.code === "CORRECTION_CONFLICT");
|
||||
assert.equal(commits, 0);
|
||||
});
|
||||
|
||||
function approved(activateRequested: boolean, state: ApprovedOcrCandidate["state"] = "indexing"): ApprovedOcrCandidate {
|
||||
return { versionId: "version-2", sourceId: "source-1", state, activateRequested, expectedActiveVersionId: "version-1", reviewedText: "reviewed", reviewedTextSha256: sha256Hex("reviewed"), processingFingerprint: "fingerprint", metadataHash: "metadata" };
|
||||
}
|
||||
|
||||
test("indexing activates only when requested and leaves a new candidate ready on an activation race", async () => {
|
||||
const calls: string[] = [];
|
||||
const store: OcrIndexingStore = {
|
||||
async findReusableVersion() { return undefined; }, async indexReviewed() { calls.push("embed"); return 1; },
|
||||
async markReady() { calls.push("ready"); }, async settleReusable() { throw new Error("not reusable"); },
|
||||
async activateVersion() { calls.push("activate"); return "version-2"; }
|
||||
};
|
||||
const service = new OcrIndexingService(store);
|
||||
assert.deepEqual(await service.index(approved(true)), { versionId: "version-2", state: "active", activated: true, activatedVersionId: "version-2" });
|
||||
assert.deepEqual(calls.splice(0), ["embed", "ready", "activate"]);
|
||||
assert.deepEqual(await service.index(approved(false)), { versionId: "version-2", state: "ready", activated: false });
|
||||
assert.deepEqual(calls.splice(0), ["embed", "ready"]);
|
||||
store.activateVersion = async () => { throw new CatalogError("race", 409, "ACTIVE_VERSION_PRECONDITION_FAILED"); };
|
||||
await assert.rejects(service.index(approved(true)), (error) => error instanceof CatalogError && error.code === "ACTIVE_VERSION_CHANGED");
|
||||
assert.deepEqual(calls, ["embed", "ready"]);
|
||||
});
|
||||
|
||||
test("reusable and rejected candidates create no embeddings and reusable activation obeys CAS intent", async () => {
|
||||
const calls: string[] = [];
|
||||
const store: OcrIndexingStore = {
|
||||
async findReusableVersion() { return { versionId: "version-existing" }; }, async indexReviewed() { calls.push("embed"); return 1; },
|
||||
async markReady() { calls.push("ready"); }, async settleReusable(_candidate, _reusable, activate) { calls.push(`settle:${activate}`); return activate; },
|
||||
async activateVersion() { throw new Error("new activation must not run"); }
|
||||
};
|
||||
const service = new OcrIndexingService(store);
|
||||
assert.deepEqual(await service.index(approved(true)), { versionId: "version-2", state: "rejected", activated: true, activatedVersionId: "version-existing", errorCode: "DUPLICATE_REUSABLE_VERSION" });
|
||||
assert.deepEqual(await service.index(approved(false)), { versionId: "version-2", state: "rejected", activated: false, errorCode: "DUPLICATE_REUSABLE_VERSION" });
|
||||
store.settleReusable = async () => { throw new CatalogError("race", 409, "ACTIVE_VERSION_PRECONDITION_FAILED"); };
|
||||
await assert.rejects(service.index(approved(true)), (error) => error instanceof CatalogError && error.code === "ACTIVE_VERSION_CHANGED");
|
||||
await assert.rejects(service.index(approved(true, "rejected")), (error) => error instanceof CatalogError && error.code === "INVALID_VERSION_STATE");
|
||||
assert.deepEqual(calls, ["settle:true", "settle:false"]);
|
||||
});
|
||||
|
||||
test("authenticated rejection is durable, conflict-safe, and never indexes or activates", async (context) => {
|
||||
const previous = { token: env.lifecycleAdminToken, enabled: env.ocrIngestEnabled };
|
||||
Object.assign(env, { lifecycleAdminToken: "review-token", ocrIngestEnabled: true });
|
||||
context.after(() => Object.assign(env, { lifecycleAdminToken: previous.token, ocrIngestEnabled: previous.enabled }));
|
||||
let value = candidate();
|
||||
let reads = 0;
|
||||
const audits: Array<{ reviewedBy: string; reason: string }> = [];
|
||||
const review = new OcrReviewService({
|
||||
async loadCandidate() { reads += 1; return value; },
|
||||
async commitApproval() { throw new Error("approval must not run"); },
|
||||
async commitRejection({ reviewedBy, reason }) {
|
||||
audits.push({ reviewedBy, reason });
|
||||
value = { ...value, state: "rejected" };
|
||||
}
|
||||
});
|
||||
let indexingCalls = 0;
|
||||
const catalog = { async getIngestionStatus() {
|
||||
return { versionId: value.versionId, state: value.state, phase: value.state, activated: false, error: value.state === "rejected" ? { code: "OCR_REJECTED", message: audits[0]?.reason, retryable: false } : null };
|
||||
} };
|
||||
const server = createApp({ catalog: catalog as never, reviewService: review, indexingService: { async index() { indexingCalls += 1; throw new Error("indexing must not run"); } }, startReconciler: false }).listen(0);
|
||||
context.after(() => server.close());
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
const url = `http://127.0.0.1:${address.port}/ingestions/version-2`;
|
||||
|
||||
const unauthorized = await fetch(`${url}/reject`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ candidateSha256: "candidate-hash", reviewedBy: "admin", reason: "Unreadable code" }) });
|
||||
assert.equal(unauthorized.status, 401);
|
||||
assert.equal(reads, 0);
|
||||
const invalid = await fetch(`${url}/reject`, { method: "POST", headers: { authorization: "Bearer review-token", "content-type": "application/json" }, body: "{}" });
|
||||
assert.equal(invalid.status, 400);
|
||||
assert.deepEqual(await invalid.json(), { ok: false, error: "Candidate hash, reviewer, and rejection reason are required", code: "INVALID_REJECTION" });
|
||||
assert.equal(reads, 0);
|
||||
const stale = await fetch(`${url}/reject`, { method: "POST", headers: { authorization: "Bearer review-token", "content-type": "application/json" }, body: JSON.stringify({ candidateSha256: "stale", reviewedBy: "admin", reason: "Unreadable code" }) });
|
||||
assert.equal(stale.status, 409);
|
||||
assert.equal(audits.length, 0);
|
||||
const rejected = await fetch(`${url}/reject`, { method: "POST", headers: { authorization: "Bearer review-token", "content-type": "application/json" }, body: JSON.stringify({ candidateSha256: "candidate-hash", reviewedBy: "admin", reason: "Unreadable code" }) });
|
||||
assert.equal(rejected.status, 200);
|
||||
assert.deepEqual(await rejected.json(), { versionId: "version-2", state: "rejected", activated: false });
|
||||
assert.deepEqual(audits, [{ reviewedBy: "admin", reason: "Unreadable code" }]);
|
||||
const status = await fetch(url, { headers: { authorization: "Bearer review-token" } });
|
||||
assert.equal(status.status, 200);
|
||||
assert.deepEqual(await status.json(), { versionId: "version-2", state: "rejected", phase: "rejected", activated: false, error: { code: "OCR_REJECTED", message: "Unreadable code", retryable: false } });
|
||||
const repeated = await fetch(`${url}/reject`, { method: "POST", headers: { authorization: "Bearer review-token", "content-type": "application/json" }, body: JSON.stringify({ candidateSha256: "candidate-hash", reviewedBy: "admin", reason: "Again" }) });
|
||||
assert.equal(repeated.status, 409);
|
||||
assert.equal(indexingCalls, 0);
|
||||
assert.equal(audits.length, 1);
|
||||
});
|
||||
|
||||
test("playground serves the authenticated OCR review controls and audit fields", async (context) => {
|
||||
const server = createApp({ startReconciler: false }).listen(0);
|
||||
context.after(() => server.close());
|
||||
const address = server.address();
|
||||
assert.ok(address && typeof address === "object");
|
||||
const base = `http://127.0.0.1:${address.port}`;
|
||||
const [html, script, styles] = await Promise.all([
|
||||
fetch(`${base}/playground`).then((response) => response.text()),
|
||||
fetch(`${base}/playground/app.js`).then((response) => response.text()),
|
||||
fetch(`${base}/playground/styles.css`).then((response) => response.text())
|
||||
]);
|
||||
for (const marker of ["data-tab=\"review\"", "reviewVersionId", "reviewToken", "loadReviewButton", "approveReviewButton", "rejectReviewButton", "reviewCandidate"]) assert.match(html, new RegExp(marker));
|
||||
for (const marker of ["Authorization", "/review", "/approve", "/reject", "candidateSha256", "expectedLineSha256", "imageUrl", "nativeText", "confidence", "bbox", "differences", "risks"]) assert.match(script, new RegExp(marker));
|
||||
assert.match(styles, /\.review-page/);
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue