380 lines
18 KiB
Python
380 lines
18 KiB
Python
import hashlib
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
sys.path.insert(0, str(Path(__file__).parents[1]))
|
|
|
|
from app.main import create_app
|
|
from app.engine import EngineLine
|
|
from app.jobs import JobQueue
|
|
|
|
|
|
TOKEN = "unit-5-test-token"
|
|
PDF = b"%PDF-1.4\nunit five\n%%EOF"
|
|
FIXTURE = Path(__file__).parents[2] / "tests" / "fixtures" / "ocr" / "native-three-pages.pdf"
|
|
|
|
|
|
class FakeEngine:
|
|
def recognize(self, _image: object) -> list[EngineLine]:
|
|
return [EngineLine("Factura FAT07", 0.97, (10, 20, 160, 50))]
|
|
|
|
|
|
def request_for(pdf: bytes = PDF, pages: list[int] | None = None) -> dict:
|
|
return {
|
|
"documentSha256": hashlib.sha256(pdf).hexdigest(),
|
|
"pages": pages or [1, 2],
|
|
"languages": ["es", "en"],
|
|
"dpi": 200,
|
|
"engine": "paddleocr",
|
|
"engineVersion": "3.4.0",
|
|
"runtimeVersion": "3.2.2",
|
|
"configVersion": "ocr-v1",
|
|
"returnLayout": True,
|
|
}
|
|
|
|
|
|
def key_for(request: dict) -> str:
|
|
pages = json.dumps(request["pages"], separators=(",", ":")).encode()
|
|
return f'{request["documentSha256"]}:ocr-v1:{hashlib.sha256(pages).hexdigest()}'
|
|
|
|
|
|
def submit(client: TestClient, request: dict, pdf: bytes = PDF, key: str | None = None):
|
|
return client.post(
|
|
"/v1/jobs",
|
|
headers={"Authorization": f"Bearer {TOKEN}", "Idempotency-Key": key or key_for(request)},
|
|
files={
|
|
"file": ("input.pdf", pdf, "application/pdf"),
|
|
"request": (None, json.dumps(request), "application/json"),
|
|
},
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def client(tmp_path: Path) -> TestClient:
|
|
return TestClient(create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True))
|
|
|
|
|
|
def wait_for_status(client: TestClient, job_id: str, expected: str) -> dict:
|
|
headers = {"Authorization": f"Bearer {TOKEN}"}
|
|
deadline = time.monotonic() + 5
|
|
while time.monotonic() < deadline:
|
|
status = client.get(f"/v1/jobs/{job_id}", headers=headers).json()
|
|
if status["status"] == expected:
|
|
return status
|
|
time.sleep(0.02)
|
|
pytest.fail(f"Job {job_id} did not reach {expected}")
|
|
|
|
|
|
def test_auth_rejects_missing_and_wrong_bearer_and_health_exposes_no_secret(client: TestClient):
|
|
live = client.get("/health/live")
|
|
ready = client.get("/health/ready")
|
|
assert live.json() == {"status": "ok", "service": "ocr", "version": "0.2.0", "revision": "unknown"}
|
|
assert ready.status_code == 503
|
|
assert ready.json() == {
|
|
"ready": False, "engineLoaded": True, "storageAvailable": True,
|
|
"queueDepth": 0, "queueCapacity": 3,
|
|
"queueStates": {"queued": 0, "running": 0, "succeeded": 0, "failed": 0},
|
|
"concurrency": 1, "workerState": "stopped", "workerOperational": False,
|
|
"sweeperOperational": False, "recoveryAttempts": 0,
|
|
"lastSweepAt": None,
|
|
"service": "ocr", "version": "0.2.0", "revision": "unknown",
|
|
}
|
|
for authorization in (None, "Bearer wrong-token"):
|
|
headers = {"Idempotency-Key": key_for(request_for())}
|
|
if authorization:
|
|
headers["Authorization"] = authorization
|
|
response = client.post(
|
|
"/v1/jobs",
|
|
headers=headers,
|
|
files={"file": ("input.pdf", PDF, "application/pdf"), "request": (None, json.dumps(request_for()))},
|
|
)
|
|
assert response.status_code == 401
|
|
assert response.json()["detail"]["retryable"] is False
|
|
assert TOKEN not in response.text
|
|
|
|
unavailable = TestClient(create_app(TOKEN, ":memory:", engine_ready=False)).get("/health/ready")
|
|
assert unavailable.status_code == 503
|
|
assert unavailable.json()["ready"] is False
|
|
|
|
|
|
def test_auth_submission_is_idempotent_and_conflicting_payload_is_terminal(client: TestClient):
|
|
request = request_for()
|
|
first = submit(client, request)
|
|
repeated = submit(client, request)
|
|
conflict = request_for(pages=[1])
|
|
conflicting = submit(client, conflict, key=key_for(request))
|
|
assert first.status_code == repeated.status_code == 202
|
|
assert first.json() == repeated.json()
|
|
assert first.json()["requestedPages"] == [1, 2]
|
|
assert conflicting.status_code == 409
|
|
assert conflicting.json()["detail"] == {
|
|
"code": "IDEMPOTENCY_CONFLICT",
|
|
"message": "The idempotency key is already bound to another request",
|
|
"retryable": False,
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("field", "value"),
|
|
[("languages", ["en"]), ("dpi", 300), ("engine", "tesseract"), ("returnLayout", False)],
|
|
)
|
|
def test_auth_allowlist_rejects_client_selected_configuration(client: TestClient, field: str, value: object):
|
|
request = request_for()
|
|
request[field] = value
|
|
response = submit(client, request, key="different-key")
|
|
assert response.status_code == 400
|
|
assert response.json()["detail"]["retryable"] is False
|
|
|
|
|
|
def test_auth_limits_corrupt_pdf_and_integrity_mismatch_are_terminal(tmp_path: Path):
|
|
client = TestClient(create_app(TOKEN, tmp_path / "limited.db", max_upload_bytes=8, engine_ready=True))
|
|
regular = TestClient(create_app(TOKEN, tmp_path / "regular.db", engine_ready=True))
|
|
oversized = submit(client, request_for(PDF), PDF)
|
|
too_many = submit(regular, request_for(pages=list(range(1, 102))))
|
|
corrupt = submit(regular, request_for(b"not-pdf"), b"not-pdf")
|
|
wrong_hash = request_for()
|
|
wrong_hash["documentSha256"] = "0" * 64
|
|
mismatch = submit(regular, wrong_hash, key=key_for(wrong_hash))
|
|
assert oversized.status_code == too_many.status_code == 413
|
|
assert corrupt.status_code == mismatch.status_code == 422
|
|
assert mismatch.json()["detail"]["code"] == "INTEGRITY_MISMATCH"
|
|
for response in (oversized, too_many, corrupt, mismatch):
|
|
assert response.json()["detail"]["retryable"] is False
|
|
assert "Retry-After" not in response.headers
|
|
|
|
|
|
def test_auth_queue_pressure_is_retryable_and_status_and_delete_are_authenticated(client: TestClient):
|
|
accepted = [submit(client, request_for(pdf), pdf) for pdf in (PDF, PDF + b"1", PDF + b"2")]
|
|
pressure = submit(client, request_for(PDF + b"3"), PDF + b"3")
|
|
assert [response.status_code for response in accepted] == [202, 202, 202]
|
|
assert pressure.status_code == 429
|
|
assert pressure.json()["detail"]["retryable"] is True
|
|
assert pressure.headers["Retry-After"] == "2"
|
|
job_id = accepted[0].json()["jobId"]
|
|
status = client.get(f"/v1/jobs/{job_id}", headers={"Authorization": f"Bearer {TOKEN}"})
|
|
assert status.json() == {"jobId": job_id, "status": "queued", "completedPages": 0, "totalPages": 2, "error": None}
|
|
assert client.delete(f"/v1/jobs/{job_id}").status_code == 401
|
|
assert client.delete(f"/v1/jobs/{job_id}", headers={"Authorization": f"Bearer {TOKEN}"}).status_code == 204
|
|
assert client.delete(f"/v1/jobs/{job_id}", headers={"Authorization": f"Bearer {TOKEN}"}).status_code == 204
|
|
|
|
|
|
def test_auth_health_sweeps_only_jobs_older_than_24_hours(tmp_path: Path):
|
|
current = [datetime(2026, 9, 16, tzinfo=timezone.utc)]
|
|
db_path = tmp_path / "expiry.db"
|
|
client = TestClient(create_app(TOKEN, db_path, engine_ready=True, now=lambda: current[0]))
|
|
job_id = submit(client, request_for()).json()["jobId"]
|
|
headers = {"Authorization": f"Bearer {TOKEN}"}
|
|
|
|
current[0] += timedelta(hours=23, minutes=59)
|
|
assert client.get("/health/ready").status_code == 503
|
|
assert client.get(f"/v1/jobs/{job_id}", headers=headers).status_code == 200
|
|
current[0] += timedelta(minutes=2)
|
|
client.app.state.queue.sweep_expired()
|
|
assert client.get("/health/ready").status_code == 503
|
|
assert client.get(f"/v1/jobs/{job_id}", headers=headers).status_code == 404
|
|
with sqlite3.connect(db_path) as connection:
|
|
assert connection.execute("SELECT count(*) FROM jobs").fetchone()[0] == 0
|
|
|
|
|
|
def test_auth_accepted_job_executes_and_exposes_integrity_bound_result(tmp_path: Path):
|
|
pdf = FIXTURE.read_bytes()
|
|
db_path = tmp_path / "executed.db"
|
|
with sqlite3.connect(db_path) as connection:
|
|
connection.execute("CREATE TABLE jobs (job_id TEXT PRIMARY KEY, idempotency_key TEXT UNIQUE, payload_hash TEXT, document_sha256 TEXT, pages TEXT, status TEXT, created_at TEXT)")
|
|
queued_client = TestClient(create_app(TOKEN, db_path, engine_ready=True))
|
|
submit(queued_client, request_for(pdf, [1]), pdf)
|
|
with TestClient(create_app(TOKEN, db_path, engine_ready=True, engine=FakeEngine())) as client:
|
|
accepted = submit(client, request_for(pdf, [1]), pdf)
|
|
job_id = accepted.json()["jobId"]
|
|
status = wait_for_status(client, job_id, "succeeded")
|
|
result = client.get(f"/v1/jobs/{job_id}/result", headers={"Authorization": f"Bearer {TOKEN}"})
|
|
assert status == {"jobId": job_id, "status": "succeeded", "completedPages": 1, "totalPages": 1, "error": None}
|
|
assert result.status_code == 200
|
|
assert (result.json()["jobId"], result.json()["documentSha256"]) == (job_id, hashlib.sha256(pdf).hexdigest())
|
|
assert [(page["page"], page["text"]) for page in result.json()["pages"]] == [(1, "Factura FAT07")]
|
|
ready = client.get("/health/ready")
|
|
assert ready.status_code == 200
|
|
assert ready.json()["workerState"] == "idle"
|
|
assert ready.json()["sweeperOperational"] is True
|
|
assert ready.json()["queueStates"] == {"queued": 0, "running": 0, "succeeded": 1, "failed": 0}
|
|
|
|
image = client.get(f"/v1/jobs/{job_id}/pages/1/image", headers={"Authorization": f"Bearer {TOKEN}"})
|
|
assert image.status_code == 200
|
|
assert image.headers["content-type"] == "image/png"
|
|
assert image.headers["x-document-sha256"] == hashlib.sha256(pdf).hexdigest()
|
|
assert image.headers["x-content-sha256"] == hashlib.sha256(image.content).hexdigest()
|
|
assert image.content.startswith(b"\x89PNG\r\n\x1a\n")
|
|
assert client.get(f"/v1/jobs/{job_id}/pages/1/image").status_code == 401
|
|
assert client.get(f"/v1/jobs/{job_id}/pages/4/image", headers={"Authorization": f"Bearer {TOKEN}"}).status_code == 422
|
|
|
|
|
|
def test_d2_serves_the_persisted_render_without_rerendering(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
|
pdf = FIXTURE.read_bytes()
|
|
db_path = tmp_path / "jobs.db"
|
|
with TestClient(create_app(TOKEN, db_path, engine_ready=True, engine=FakeEngine())) as client:
|
|
job_id = submit(client, request_for(pdf, [1]), pdf).json()["jobId"]
|
|
wait_for_status(client, job_id, "succeeded")
|
|
image_path = tmp_path / "artifacts" / job_id / "review-images" / "page-0001.png"
|
|
expected_png = image_path.read_bytes()
|
|
assert image_path.stat().st_mode & 0o777 == 0o600
|
|
|
|
def rerender_is_forbidden(*_args: object, **_kwargs: object) -> object:
|
|
raise AssertionError("review endpoint must not invoke PDFium")
|
|
|
|
monkeypatch.setattr("app.render.render_pdf_pages", rerender_is_forbidden)
|
|
image = client.get(f"/v1/jobs/{job_id}/pages/1/image", headers={"Authorization": f"Bearer {TOKEN}"})
|
|
|
|
assert image.status_code == 200
|
|
assert image.content == expected_png
|
|
|
|
|
|
def test_d2_missing_or_unfinished_review_images_fail_closed(tmp_path: Path):
|
|
pdf = FIXTURE.read_bytes()
|
|
with TestClient(create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True, engine=FakeEngine())) as client:
|
|
job_id = submit(client, request_for(pdf, [1]), pdf).json()["jobId"]
|
|
headers = {"Authorization": f"Bearer {TOKEN}"}
|
|
pending = client.get(f"/v1/jobs/{job_id}/pages/1/image", headers=headers)
|
|
assert (pending.status_code, pending.json()["detail"]["code"]) == (409, "RESULT_NOT_READY")
|
|
|
|
wait_for_status(client, job_id, "succeeded")
|
|
(tmp_path / "artifacts" / job_id / "review-images" / "page-0001.png").unlink()
|
|
missing = client.get(f"/v1/jobs/{job_id}/pages/1/image", headers=headers)
|
|
assert (missing.status_code, missing.json()["detail"]["code"]) == (409, "RESULT_NOT_READY")
|
|
|
|
|
|
def test_d1_persists_input_as_an_artifact_not_a_sqlite_blob(tmp_path: Path):
|
|
queue = JobQueue(tmp_path / "jobs.db", lambda: datetime.now(timezone.utc))
|
|
request = request_for()
|
|
ack, created = queue.submit(key_for(request), request, PDF)
|
|
|
|
assert created is True
|
|
artifact = tmp_path / "artifacts" / ack["jobId"] / "input.pdf"
|
|
assert artifact.read_bytes() == PDF
|
|
assert artifact.stat().st_mode & 0o777 == 0o600
|
|
with sqlite3.connect(tmp_path / "jobs.db") as connection:
|
|
columns = {column[1] for column in connection.execute("PRAGMA table_info(jobs)")}
|
|
stored_path = connection.execute("SELECT input_path FROM jobs WHERE job_id=?", (ack["jobId"],)).fetchone()[0]
|
|
assert "pdf" not in columns
|
|
assert stored_path == f"artifacts/{ack['jobId']}/input.pdf"
|
|
|
|
|
|
def test_d1_recovers_one_interrupted_job_then_fails_a_second_interruption(tmp_path: Path):
|
|
now = datetime(2026, 9, 21, tzinfo=timezone.utc)
|
|
db_path = tmp_path / "jobs.db"
|
|
queue = JobQueue(db_path, lambda: now)
|
|
request = request_for()
|
|
ack, _ = queue.submit(key_for(request), request, PDF)
|
|
with sqlite3.connect(db_path) as connection:
|
|
connection.execute("UPDATE jobs SET status='running' WHERE job_id=?", (ack["jobId"],))
|
|
recovered = JobQueue(db_path, lambda: now)
|
|
assert recovered.status(ack["jobId"])["status"] == "queued"
|
|
with sqlite3.connect(db_path) as connection:
|
|
connection.execute("UPDATE jobs SET status='running' WHERE job_id=?", (ack["jobId"],))
|
|
exhausted = JobQueue(db_path, lambda: now)
|
|
status = exhausted.status(ack["jobId"])
|
|
assert status["status"] == "failed"
|
|
assert status["error"]["code"] == "OCR_RECOVERY_EXHAUSTED"
|
|
|
|
|
|
def test_d4_storage_pressure_rejects_admission_and_cleans_partial_images(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
|
application = create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True)
|
|
client = TestClient(application)
|
|
queue = application.state.queue
|
|
monkeypatch.setattr(queue, "_has_storage_capacity", lambda _required=0: False)
|
|
|
|
pressure = submit(client, request_for())
|
|
assert pressure.status_code == 503
|
|
assert pressure.json()["detail"] == {
|
|
"code": "OCR_STORAGE_PRESSURE", "message": "OCR storage capacity is unavailable", "retryable": True,
|
|
}
|
|
assert pressure.headers["Retry-After"] == "15"
|
|
assert queue.depth() == 0
|
|
|
|
monkeypatch.setattr(queue, "_has_storage_capacity", lambda _required=0: True)
|
|
job_id = submit(client, request_for()).json()["jobId"]
|
|
|
|
def persist_one_page(*_args, **kwargs):
|
|
kwargs["persist_review_image"](1, b"private-png")
|
|
return {}
|
|
|
|
monkeypatch.setattr("app.jobs.process_pdf", persist_one_page)
|
|
monkeypatch.setattr(queue, "_has_storage_capacity", lambda _required=0: False)
|
|
queue.start_worker(FakeEngine())
|
|
try:
|
|
status = wait_for_status(client, job_id, "failed")
|
|
finally:
|
|
queue.stop()
|
|
assert status["error"]["code"] == "OCR_STORAGE_PRESSURE"
|
|
assert not (tmp_path / "artifacts" / job_id / "review-images").exists()
|
|
|
|
|
|
def test_d4_lease_recovery_timeout_and_active_delete_are_fail_closed(tmp_path: Path, caplog: pytest.LogCaptureFixture):
|
|
current = [datetime(2026, 9, 21, tzinfo=timezone.utc)]
|
|
application = create_app(TOKEN, tmp_path / "jobs.db", engine_ready=True, now=lambda: current[0])
|
|
client = TestClient(application)
|
|
queue = application.state.queue
|
|
job_id = submit(client, request_for()).json()["jobId"]
|
|
queue._claim()
|
|
headers = {"Authorization": f"Bearer {TOKEN}"}
|
|
|
|
active = client.delete(f"/v1/jobs/{job_id}", headers=headers)
|
|
assert (active.status_code, active.json()["detail"]["code"]) == (409, "JOB_ACTIVE")
|
|
|
|
with queue.connection:
|
|
queue.connection.execute(
|
|
"UPDATE jobs SET lease_expires_at=? WHERE job_id=?",
|
|
(queue._iso(current[0] - timedelta(seconds=1)), job_id),
|
|
)
|
|
queue.sweep_expired()
|
|
assert queue.status(job_id)["status"] == "queued"
|
|
assert queue.operational_state()["recoveryAttempts"] == 1
|
|
|
|
queue._claim()
|
|
current[0] += timedelta(minutes=16)
|
|
queue.sweep_expired()
|
|
assert queue.status(job_id)["error"]["code"] == "OCR_PROCESSING_TIMEOUT"
|
|
assert job_id in caplog.text
|
|
assert "category=ocr_processing_timeout" in caplog.text
|
|
|
|
stale_id = queue.submit(key_for(request_for(PDF + b"stale")), request_for(PDF + b"stale"), PDF + b"stale")[0]["jobId"]
|
|
with queue.connection:
|
|
queue.connection.execute(
|
|
"UPDATE jobs SET status='running',started_at=? WHERE job_id=?",
|
|
(queue._iso(current[0] - timedelta(minutes=16)), stale_id),
|
|
)
|
|
restarted = JobQueue(tmp_path / "jobs.db", lambda: current[0])
|
|
assert restarted.status(stale_id)["error"]["code"] == "OCR_PROCESSING_TIMEOUT"
|
|
|
|
|
|
def test_d4_periodic_sweeper_purges_ttl_and_maintains_incremental_vacuum(tmp_path: Path):
|
|
current = [datetime(2026, 9, 21, tzinfo=timezone.utc)]
|
|
queue = JobQueue(tmp_path / "jobs.db", lambda: current[0], sweep_interval=0.01)
|
|
job_id = queue.submit(key_for(request_for()), request_for(), PDF)[0]["jobId"]
|
|
queue.start_worker(None)
|
|
current[0] += timedelta(hours=25)
|
|
try:
|
|
deadline = time.monotonic() + 1
|
|
while queue.status(job_id) is not None and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
assert queue.status(job_id) is None
|
|
assert queue.operational_state()["sweeperOperational"] is True
|
|
assert queue.last_sweep_at is not None
|
|
assert queue.connection.execute("PRAGMA auto_vacuum").fetchone()[0] == 2
|
|
finally:
|
|
queue.stop()
|
|
|
|
|
|
def test_auth_result_rejects_unknown_not_ready_and_unauthorized_jobs(client: TestClient):
|
|
job_id = submit(client, request_for()).json()["jobId"]
|
|
headers = {"Authorization": f"Bearer {TOKEN}"}
|
|
assert client.get(f"/v1/jobs/{job_id}/result").status_code == 401
|
|
assert client.get("/v1/jobs/missing/result", headers=headers).json()["detail"]["code"] == "JOB_NOT_FOUND"
|
|
pending = client.get(f"/v1/jobs/{job_id}/result", headers=headers)
|
|
assert (pending.status_code, pending.json()["detail"]["code"]) == (409, "RESULT_NOT_READY")
|