diff --git a/scripts/spike-pdfjs.ts b/scripts/spike-pdfjs.ts new file mode 100644 index 0000000..e05850a --- /dev/null +++ b/scripts/spike-pdfjs.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import pdf from "pdf-parse"; + +interface PdfPageData { + getTextContent(options: { normalizeWhitespace: boolean; disableCombineTextItems: boolean }): Promise<{ + items: Array<{ str?: string }>; + }>; +} + +const fixtureUrl = new URL("../tests/fixtures/ocr/native-three-pages.pdf", import.meta.url); +const input = process.argv[2] ? new URL(`file://${process.argv[2]}`) : fixtureUrl; +const nodeMajor = Number.parseInt(process.versions.node.split(".")[0] ?? "0", 10); + +assert.ok(nodeMajor >= 22, `Node 22 or newer is required; received ${process.versions.node}`); + +const pages: Array<{ page: number; text: string }> = []; +const bytes = Uint8Array.from(await readFile(input)); +const result = await pdf(bytes as Buffer, { + version: "v2.0.550", + pagerender: async (pageData: PdfPageData) => { + const textContent = await pageData.getTextContent({ normalizeWhitespace: false, disableCombineTextItems: false }); + const text = textContent.items.flatMap((item) => item.str ?? []).join(" ").trim(); + pages.push({ page: pages.length + 1, text }); + return text; + } +}); + +assert.equal(result.numpages, pages.length); +assert.deepEqual(pages, [ + { page: 1, text: "Native page one." }, + { page: 2, text: "" }, + { page: 3, text: "Native page three." } +]); + +console.log(JSON.stringify({ node: process.versions.node, parser: `pdf-parse/pdfjs-${result.version}`, pages })); diff --git a/src/modules/parsers/parser-registry.ts b/src/modules/parsers/parser-registry.ts index be9268c..c4127d0 100644 --- a/src/modules/parsers/parser-registry.ts +++ b/src/modules/parsers/parser-registry.ts @@ -1,7 +1,9 @@ import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; import path from "node:path"; import pdf from "pdf-parse"; import type { ChunkingMode } from "../process/chunking.js"; +import { sha256Hex } from "../../shared/utils/ids.js"; export interface ParsedDocument { title: string; @@ -10,6 +12,18 @@ export interface ParsedDocument { chunkMode: ChunkingMode; } +export interface ParsedPdfPage { + page: number; + text: string; + textSha256: string; +} + +interface PdfPageData { + getTextContent(options: { normalizeWhitespace: boolean; disableCombineTextItems: boolean }): Promise<{ + items: Array<{ str?: string }>; + }>; +} + const documentalExtensions = [".md", ".txt", ".pdf"] as const; const codeExtensions = [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".json", ".yml", ".yaml"] as const; const parserExtensions = [...documentalExtensions, ...codeExtensions] as const; @@ -19,6 +33,9 @@ export function supportedParserExtensions(): string[] { } export function isSupportedDocument(filePath: string): boolean { + if (path.basename(filePath).toLowerCase() === "cmakelists.txt") { + return false; + } return parserExtensions.includes(path.extname(filePath).toLowerCase() as (typeof parserExtensions)[number]); } @@ -43,16 +60,42 @@ function inferMimeType(extension: string, chunkMode: ChunkingMode): string { return "text/plain"; } +export async function parsePdfPages(filePath: string | URL): Promise { + const resolvedPath = filePath instanceof URL ? fileURLToPath(filePath) : filePath; + if (path.extname(resolvedPath).toLowerCase() !== ".pdf") { + throw new Error("Only PDF documents support page extraction"); + } + + const pages: ParsedPdfPage[] = []; + const bytes = Uint8Array.from(await readFile(filePath)); + const result = await pdf(bytes as Buffer, { + version: "v2.0.550", + pagerender: async (pageData: PdfPageData) => { + const textContent = await pageData.getTextContent({ + normalizeWhitespace: false, + disableCombineTextItems: false + }); + const text = textContent.items.flatMap((item) => item.str ?? []).join(" ").trim(); + pages.push({ page: pages.length + 1, text, textSha256: sha256Hex(text) }); + return text; + } + }); + + if (result.numrender !== result.numpages || pages.length !== result.numpages) { + throw new Error("PDF page extraction did not render every page"); + } + return pages; +} + export async function parseDocument(filePath: string): Promise { const extension = path.extname(filePath).toLowerCase(); const chunkMode = inferChunkMode(filePath); if (extension === ".pdf") { - const buffer = await readFile(filePath); - const result = await pdf(buffer); + const pages = await parsePdfPages(filePath); return { title: path.basename(filePath), - content: result.text.trim(), + content: pages.map((page) => page.text).filter(Boolean).join("\n\n"), mimeType: inferMimeType(extension, chunkMode), chunkMode }; diff --git a/tests/fixtures/ocr/native-three-pages.pdf b/tests/fixtures/ocr/native-three-pages.pdf new file mode 100644 index 0000000..210702d --- /dev/null +++ b/tests/fixtures/ocr/native-three-pages.pdf @@ -0,0 +1,63 @@ +%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [4 0 R 5 0 R 6 0 R] /Count 3 >> +endobj +3 0 obj +<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> +endobj +4 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents 7 0 R >> +endobj +5 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents 8 0 R >> +endobj +6 0 obj +<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents 9 0 R >> +endobj +7 0 obj +<< /Length 48 >> +stream +BT +/F1 12 Tf +72 720 Td +(Native page one.) Tj +ET +endstream +endobj +8 0 obj +<< /Length 6 >> +stream +BT +ET +endstream +endobj +9 0 obj +<< /Length 50 >> +stream +BT +/F1 12 Tf +72 720 Td +(Native page three.) Tj +ET +endstream +endobj +xref +0 10 +0000000000 65535 f +0000000009 00000 n +0000000058 00000 n +0000000127 00000 n +0000000197 00000 n +0000000323 00000 n +0000000449 00000 n +0000000575 00000 n +0000000672 00000 n +0000000726 00000 n +trailer +<< /Size 10 /Root 1 0 R >> +startxref +825 +%%EOF diff --git a/tests/parsers/pdf-pages.test.ts b/tests/parsers/pdf-pages.test.ts new file mode 100644 index 0000000..db889f3 --- /dev/null +++ b/tests/parsers/pdf-pages.test.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import { access, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + isSupportedDocument, + parseDocument, + parsePdfPages +} from "../../src/modules/parsers/parser-registry.js"; + +const fixturePath = new URL("../fixtures/ocr/native-three-pages.pdf", import.meta.url); + +test("parsePdfPages extracts ordered one-based pages with stable native text hashes", async () => { + const pages = await parsePdfPages(fixturePath); + + assert.deepEqual(pages, [ + { + page: 1, + text: "Native page one.", + textSha256: "b9efe3745cacd6c87189435d7b83374ee7e5f77fca9d52e8d4e76f4a12a419f4" + }, + { + page: 2, + text: "", + textSha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + page: 3, + text: "Native page three.", + textSha256: "e50a3249ccbe373afaac6ed06c36692f7190b260f8c55f21b45e4a5066e9352e" + } + ]); +}); + +test("parseDocument preserves the synchronous native PDF content contract", async () => { + const parsed = await parseDocument(fixturePath.pathname); + + assert.equal(parsed.title, "native-three-pages.pdf"); + assert.equal(parsed.content, "Native page one.\n\nNative page three."); + assert.equal(parsed.mimeType, "application/pdf"); + assert.equal(parsed.chunkMode, "documental"); +}); + +test("page extraction rejects non-PDF inputs before parsing bytes", async () => { + await assert.rejects( + parsePdfPages(new URL("../fixtures/ocr/not-a-pdf.txt", import.meta.url)), + /only PDF documents support page extraction/i + ); +}); + +test("supported text documents are read as data and never executed", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "rag-parser-threat-")); + const markerPath = path.join(directory, "executed"); + const payload = `$(touch ${markerPath})`; + + try { + for (const fileName of ["requirements.txt", "executable.md"]) { + const filePath = path.join(directory, fileName); + await writeFile(filePath, payload, "utf8"); + const parsed = await parseDocument(filePath); + + assert.equal(isSupportedDocument(filePath), true); + assert.equal(parsed.content, payload); + } + await assert.rejects(access(markerPath), /ENOENT/); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("build and executable lookalike formats remain unsupported", () => { + assert.deepEqual( + ["CMakeLists.txt", "component.mdx", "README.sh"].map(isSupportedDocument), + [false, false, false] + ); +});