import hashlib import json import os import sqlite3 import sys import time from datetime import datetime, timedelta, timezone from pathlib import Path import pytest from fastapi.testclient import TestClient sys.path.insert(0, str(Path(__file__).parents[1])) from app.main import create_app from app.engine import EngineLine from app.jobs import JobQueue from app.version import release_version TOKEN = "unit-5-test-token" PDF = b"%PDF-1.4\nunit five\n%%EOF" FIXTURE = Path(__file__).parents[2] / "tests" / "fixtures" / "ocr" / "native-three-pages.pdf" class FakeEngine: def recognize(self, _image: object) -> list[EngineLine]: return [EngineLine("Factura FAT07", 0.97, (10, 20, 160, 50))] def request_for(pdf: bytes = PDF, pages: list[int] | None = None) -> dict: return { "documentSha256": hashlib.sha256(pdf).hexdigest(), "pages": pages or [1, 2], "languages": ["es", "en"], "dpi": 200, "engine": "paddleocr", "engineVersion": "3.4.0", "runtimeVersion": "3.2.2", "configVersion": "ocr-v2", "returnLayout": True, } def key_for(request: dict) -> str: pages = json.dumps(request["pages"], separators=(",", ":")).encode() return f'{request["documentSha256"]}:ocr-v2:{hashlib.sha256(pages).hexdigest()}' def submit(client: TestClient, request: dict, pdf: bytes = PDF, key: str | None = None): return client.post( "/v1/jobs", headers={"Authorization": f"Bearer {TOKEN}", "Idempotency-Key": key or key_for(request)}, files={ "file": ("input.pdf", pdf, "application/pdf"), "request": (None, json.dumps(request), "application/json"), }, ) @pytest.fixture def client(tmp_path: Path) -> TestClient: return TestClient(create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True)) def wait_for_status(client: TestClient, job_id: str, expected: str) -> dict: headers = {"Authorization": f"Bearer {TOKEN}"} deadline = time.monotonic() + 5 while time.monotonic() < deadline: status = client.get(f"/v1/jobs/{job_id}", headers=headers).json() if status["status"] == expected: return status time.sleep(0.02) pytest.fail(f"Job {job_id} did not reach {expected}") def test_auth_rejects_missing_and_wrong_bearer_and_health_exposes_no_secret(client: TestClient): revision = os.getenv("BUILD_REVISION", "unknown") live = client.get("/health/live") ready = client.get("/health/ready") assert live.json() == {"status": "ok", "service": "ocr", "version": release_version(), "revision": revision} assert ready.status_code == 503 assert ready.json() == { "ready": False, "engineLoaded": True, "storageAvailable": True, "queueDepth": 0, "queueCapacity": 3, "queueStates": {"queued": 0, "running": 0, "succeeded": 0, "failed": 0}, "concurrency": 1, "workerState": "stopped", "workerOperational": False, "sweeperOperational": False, "recoveryAttempts": 0, "lastSweepAt": None, "service": "ocr", "version": release_version(), "revision": revision, } for authorization in (None, "Bearer wrong-token"): headers = {"Idempotency-Key": key_for(request_for())} if authorization: headers["Authorization"] = authorization response = client.post( "/v1/jobs", headers=headers, files={"file": ("input.pdf", PDF, "application/pdf"), "request": (None, json.dumps(request_for()))}, ) assert response.status_code == 401 assert response.json()["detail"]["retryable"] is False assert TOKEN not in response.text unavailable = TestClient(create_app(TOKEN, ":memory:", engine_ready=False)).get("/health/ready") assert unavailable.status_code == 503 assert unavailable.json()["ready"] is False def test_declared_ocr_version_must_match_version_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): monkeypatch.setenv("OCR_VERSION", "incorrect") with pytest.raises(RuntimeError, match="OCR_VERSION must match VERSION"): create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True) def test_auth_submission_is_idempotent_and_conflicting_payload_is_terminal(client: TestClient): request = request_for() first = submit(client, request) repeated = submit(client, request) conflict = request_for(pages=[1]) conflicting = submit(client, conflict, key=key_for(request)) assert first.status_code == repeated.status_code == 202 assert first.json() == repeated.json() assert first.json()["requestedPages"] == [1, 2] assert conflicting.status_code == 409 assert conflicting.json()["detail"] == { "code": "IDEMPOTENCY_CONFLICT", "message": "The idempotency key is already bound to another request", "retryable": False, } def test_auth_duplicates_preserve_each_known_remote_state_and_reject_ocr_v1(tmp_path: Path): db_path = tmp_path / "states.db" client = TestClient(create_app(TOKEN, db_path, engine_ready=True)) request = request_for() accepted = submit(client, request) job_id = accepted.json()["jobId"] for state in ("queued", "running", "succeeded", "failed"): with sqlite3.connect(db_path) as connection: connection.execute("UPDATE jobs SET status=? WHERE job_id=?", (state, job_id)) repeated = submit(client, request) assert repeated.status_code == 202 assert repeated.json()["jobId"] == job_id assert repeated.json()["status"] == state legacy = request_for() legacy["configVersion"] = "ocr-v1" rejected = submit(client, legacy, key=key_for(legacy).replace(":ocr-v2:", ":ocr-v1:")) assert rejected.status_code == 400 assert rejected.json()["detail"]["code"] == "CONFIG_NOT_ALLOWED" @pytest.mark.parametrize( ("field", "value"), [("languages", ["en"]), ("dpi", 300), ("engine", "tesseract"), ("returnLayout", False)], ) def test_auth_allowlist_rejects_client_selected_configuration(client: TestClient, field: str, value: object): request = request_for() request[field] = value response = submit(client, request, key="different-key") assert response.status_code == 400 assert response.json()["detail"]["retryable"] is False def test_auth_limits_corrupt_pdf_and_integrity_mismatch_are_terminal(tmp_path: Path): client = TestClient(create_app(TOKEN, tmp_path / "limited.db", max_upload_bytes=8, engine_ready=True)) regular = TestClient(create_app(TOKEN, tmp_path / "regular.db", engine_ready=True)) oversized = submit(client, request_for(PDF), PDF) too_many = submit(regular, request_for(pages=list(range(1, 102)))) corrupt = submit(regular, request_for(b"not-pdf"), b"not-pdf") wrong_hash = request_for() wrong_hash["documentSha256"] = "0" * 64 mismatch = submit(regular, wrong_hash, key=key_for(wrong_hash)) assert oversized.status_code == too_many.status_code == 413 assert corrupt.status_code == mismatch.status_code == 422 assert mismatch.json()["detail"]["code"] == "INTEGRITY_MISMATCH" for response in (oversized, too_many, corrupt, mismatch): assert response.json()["detail"]["retryable"] is False assert "Retry-After" not in response.headers def test_auth_queue_pressure_is_retryable_and_status_and_delete_are_authenticated(client: TestClient): accepted = [submit(client, request_for(pdf), pdf) for pdf in (PDF, PDF + b"1", PDF + b"2")] pressure = submit(client, request_for(PDF + b"3"), PDF + b"3") assert [response.status_code for response in accepted] == [202, 202, 202] assert pressure.status_code == 429 assert pressure.json()["detail"]["retryable"] is True assert pressure.headers["Retry-After"] == "2" job_id = accepted[0].json()["jobId"] status = client.get(f"/v1/jobs/{job_id}", headers={"Authorization": f"Bearer {TOKEN}"}) assert {key: status.json()[key] for key in ("jobId", "status", "completedPages", "totalPages", "error")} == { "jobId": job_id, "status": "queued", "completedPages": 0, "totalPages": 2, "error": None, } assert status.json()["configVersion"] == "ocr-v2" assert status.json()["requestIdentitySha256"] assert client.delete(f"/v1/jobs/{job_id}").status_code == 401 assert client.delete(f"/v1/jobs/{job_id}", headers={"Authorization": f"Bearer {TOKEN}"}).status_code == 204 assert client.delete(f"/v1/jobs/{job_id}", headers={"Authorization": f"Bearer {TOKEN}"}).status_code == 204 def test_auth_health_sweeps_only_jobs_older_than_24_hours(tmp_path: Path): current = [datetime(2026, 9, 16, tzinfo=timezone.utc)] db_path = tmp_path / "expiry.db" client = TestClient(create_app(TOKEN, db_path, engine_ready=True, now=lambda: current[0])) job_id = submit(client, request_for()).json()["jobId"] headers = {"Authorization": f"Bearer {TOKEN}"} current[0] += timedelta(hours=23, minutes=59) assert client.get("/health/ready").status_code == 503 assert client.get(f"/v1/jobs/{job_id}", headers=headers).status_code == 200 current[0] += timedelta(minutes=2) client.app.state.queue.sweep_expired() assert client.get("/health/ready").status_code == 503 assert client.get(f"/v1/jobs/{job_id}", headers=headers).status_code == 404 with sqlite3.connect(db_path) as connection: assert connection.execute("SELECT count(*) FROM jobs").fetchone()[0] == 0 def test_auth_accepted_job_executes_and_exposes_integrity_bound_result(tmp_path: Path): pdf = FIXTURE.read_bytes() db_path = tmp_path / "executed.db" with sqlite3.connect(db_path) as connection: connection.execute("CREATE TABLE jobs (job_id TEXT PRIMARY KEY, idempotency_key TEXT UNIQUE, payload_hash TEXT, document_sha256 TEXT, pages TEXT, status TEXT, created_at TEXT)") queued_client = TestClient(create_app(TOKEN, db_path, engine_ready=True)) submit(queued_client, request_for(pdf, [1]), pdf) with TestClient(create_app(TOKEN, db_path, engine_ready=True, engine=FakeEngine())) as client: accepted = submit(client, request_for(pdf, [1]), pdf) job_id = accepted.json()["jobId"] status = wait_for_status(client, job_id, "succeeded") result = client.get(f"/v1/jobs/{job_id}/result", headers={"Authorization": f"Bearer {TOKEN}"}) assert {key: status[key] for key in ("jobId", "status", "completedPages", "totalPages", "error")} == { "jobId": job_id, "status": "succeeded", "completedPages": 1, "totalPages": 1, "error": None, } assert status["configVersion"] == "ocr-v2" assert status["requestIdentitySha256"] assert result.status_code == 200 assert (result.json()["jobId"], result.json()["documentSha256"]) == (job_id, hashlib.sha256(pdf).hexdigest()) assert result.json()["configVersion"] == "ocr-v2" assert result.json()["idempotencyKey"] == key_for(request_for(pdf, [1])) assert [(page["page"], page["text"]) for page in result.json()["pages"]] == [(1, "Factura FAT07")] ready = client.get("/health/ready") assert ready.status_code == 200 assert ready.json()["workerState"] == "idle" assert ready.json()["sweeperOperational"] is True assert ready.json()["queueStates"] == {"queued": 0, "running": 0, "succeeded": 1, "failed": 0} image = client.get(f"/v1/jobs/{job_id}/pages/1/image", headers={"Authorization": f"Bearer {TOKEN}"}) assert image.status_code == 200 assert image.headers["content-type"] == "image/png" assert image.headers["x-document-sha256"] == hashlib.sha256(pdf).hexdigest() assert image.headers["x-ocr-job-id"] == job_id assert image.headers["x-ocr-config-version"] == "ocr-v2" assert image.headers["x-ocr-identity-sha256"] == result.json()["requestIdentitySha256"] assert image.headers["x-content-sha256"] == hashlib.sha256(image.content).hexdigest() assert image.content.startswith(b"\x89PNG\r\n\x1a\n") assert client.get(f"/v1/jobs/{job_id}/pages/1/image").status_code == 401 assert client.get(f"/v1/jobs/{job_id}/pages/4/image", headers={"Authorization": f"Bearer {TOKEN}"}).status_code == 422 def test_d2_serves_the_persisted_render_without_rerendering(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): pdf = FIXTURE.read_bytes() db_path = tmp_path / "jobs.db" with TestClient(create_app(TOKEN, db_path, engine_ready=True, engine=FakeEngine())) as client: job_id = submit(client, request_for(pdf, [1]), pdf).json()["jobId"] wait_for_status(client, job_id, "succeeded") image_path = tmp_path / "artifacts" / job_id / "review-images" / "page-0001.png" expected_png = image_path.read_bytes() assert image_path.stat().st_mode & 0o777 == 0o600 def rerender_is_forbidden(*_args: object, **_kwargs: object) -> object: raise AssertionError("review endpoint must not invoke PDFium") monkeypatch.setattr("app.render.render_pdf_pages", rerender_is_forbidden) image = client.get(f"/v1/jobs/{job_id}/pages/1/image", headers={"Authorization": f"Bearer {TOKEN}"}) assert image.status_code == 200 assert image.content == expected_png def test_d2_missing_or_unfinished_review_images_fail_closed(tmp_path: Path): pdf = FIXTURE.read_bytes() with TestClient(create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True, engine=FakeEngine())) as client: job_id = submit(client, request_for(pdf, [1]), pdf).json()["jobId"] headers = {"Authorization": f"Bearer {TOKEN}"} pending = client.get(f"/v1/jobs/{job_id}/pages/1/image", headers=headers) assert (pending.status_code, pending.json()["detail"]["code"]) == (409, "RESULT_NOT_READY") wait_for_status(client, job_id, "succeeded") (tmp_path / "artifacts" / job_id / "review-images" / "page-0001.png").unlink() missing = client.get(f"/v1/jobs/{job_id}/pages/1/image", headers=headers) assert (missing.status_code, missing.json()["detail"]["code"]) == (409, "RESULT_NOT_READY") def test_d1_persists_input_as_an_artifact_not_a_sqlite_blob(tmp_path: Path): queue = JobQueue(tmp_path / "jobs.db", lambda: datetime.now(timezone.utc)) request = request_for() ack, created = queue.submit(key_for(request), request, PDF) assert created is True artifact = tmp_path / "artifacts" / ack["jobId"] / "input.pdf" assert artifact.read_bytes() == PDF assert artifact.stat().st_mode & 0o777 == 0o600 with sqlite3.connect(tmp_path / "jobs.db") as connection: columns = {column[1] for column in connection.execute("PRAGMA table_info(jobs)")} stored_path = connection.execute("SELECT input_path FROM jobs WHERE job_id=?", (ack["jobId"],)).fetchone()[0] assert "pdf" not in columns assert stored_path == f"artifacts/{ack['jobId']}/input.pdf" def test_d1_recovers_one_interrupted_job_then_fails_a_second_interruption(tmp_path: Path): now = datetime(2026, 9, 21, tzinfo=timezone.utc) db_path = tmp_path / "jobs.db" queue = JobQueue(db_path, lambda: now) request = request_for() ack, _ = queue.submit(key_for(request), request, PDF) with sqlite3.connect(db_path) as connection: connection.execute("UPDATE jobs SET status='running' WHERE job_id=?", (ack["jobId"],)) recovered = JobQueue(db_path, lambda: now) assert recovered.status(ack["jobId"])["status"] == "queued" with sqlite3.connect(db_path) as connection: connection.execute("UPDATE jobs SET status='running' WHERE job_id=?", (ack["jobId"],)) exhausted = JobQueue(db_path, lambda: now) status = exhausted.status(ack["jobId"]) assert status["status"] == "failed" assert status["error"]["code"] == "OCR_RECOVERY_EXHAUSTED" def test_d4_storage_pressure_rejects_admission_and_cleans_partial_images(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): application = create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True) client = TestClient(application) queue = application.state.queue monkeypatch.setattr(queue, "_has_storage_capacity", lambda _required=0: False) pressure = submit(client, request_for()) assert pressure.status_code == 503 assert pressure.json()["detail"] == { "code": "OCR_STORAGE_PRESSURE", "message": "OCR storage capacity is unavailable", "retryable": True, } assert pressure.headers["Retry-After"] == "15" assert queue.depth() == 0 monkeypatch.setattr(queue, "_has_storage_capacity", lambda _required=0: True) job_id = submit(client, request_for()).json()["jobId"] def persist_one_page(*_args, **kwargs): kwargs["persist_review_image"](1, b"private-png") return {} monkeypatch.setattr("app.jobs.process_pdf", persist_one_page) monkeypatch.setattr(queue, "_has_storage_capacity", lambda _required=0: False) queue.start_worker(FakeEngine()) try: status = wait_for_status(client, job_id, "failed") finally: queue.stop() assert status["error"]["code"] == "OCR_STORAGE_PRESSURE" assert not (tmp_path / "artifacts" / job_id / "review-images").exists() def test_d4_lease_recovery_timeout_and_active_delete_are_fail_closed(tmp_path: Path, caplog: pytest.LogCaptureFixture): current = [datetime(2026, 9, 21, tzinfo=timezone.utc)] application = create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True, now=lambda: current[0]) client = TestClient(application) queue = application.state.queue job_id = submit(client, request_for()).json()["jobId"] queue._claim() headers = {"Authorization": f"Bearer {TOKEN}"} active = client.delete(f"/v1/jobs/{job_id}", headers=headers) assert (active.status_code, active.json()["detail"]["code"]) == (409, "JOB_ACTIVE") with queue.connection: queue.connection.execute( "UPDATE jobs SET lease_expires_at=? WHERE job_id=?", (queue._iso(current[0] - timedelta(seconds=1)), job_id), ) queue.sweep_expired() assert queue.status(job_id)["status"] == "queued" assert queue.operational_state()["recoveryAttempts"] == 1 queue._claim() current[0] += timedelta(minutes=16) queue.sweep_expired() assert queue.status(job_id)["error"]["code"] == "OCR_PROCESSING_TIMEOUT" assert job_id in caplog.text assert "category=ocr_processing_timeout" in caplog.text stale_id = queue.submit(key_for(request_for(PDF + b"stale")), request_for(PDF + b"stale"), PDF + b"stale")[0]["jobId"] with queue.connection: queue.connection.execute( "UPDATE jobs SET status='running',started_at=? WHERE job_id=?", (queue._iso(current[0] - timedelta(minutes=16)), stale_id), ) restarted = JobQueue(tmp_path / "jobs.db", lambda: current[0]) assert restarted.status(stale_id)["error"]["code"] == "OCR_PROCESSING_TIMEOUT" def test_d4_periodic_sweeper_purges_ttl_and_maintains_incremental_vacuum(tmp_path: Path): current = [datetime(2026, 9, 21, tzinfo=timezone.utc)] queue = JobQueue(tmp_path / "jobs.db", lambda: current[0], sweep_interval=0.01) job_id = queue.submit(key_for(request_for()), request_for(), PDF)[0]["jobId"] queue.start_worker(None) current[0] += timedelta(hours=25) try: deadline = time.monotonic() + 1 while queue.status(job_id) is not None and time.monotonic() < deadline: time.sleep(0.01) assert queue.status(job_id) is None assert queue.operational_state()["sweeperOperational"] is True assert queue.last_sweep_at is not None assert queue.connection.execute("PRAGMA auto_vacuum").fetchone()[0] == 2 finally: queue.stop() def test_auth_result_rejects_unknown_not_ready_and_unauthorized_jobs(client: TestClient): job_id = submit(client, request_for()).json()["jobId"] headers = {"Authorization": f"Bearer {TOKEN}"} assert client.get(f"/v1/jobs/{job_id}/result").status_code == 401 assert client.get("/v1/jobs/missing/result", headers=headers).json()["detail"]["code"] == "JOB_NOT_FOUND" pending = client.get(f"/v1/jobs/{job_id}/result", headers=headers) assert (pending.status_code, pending.json()["detail"]["code"]) == (409, "RESULT_NOT_READY")