80 lines
2.6 KiB
TypeScript
80 lines
2.6 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { access, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import test from "node:test";
|
|
import {
|
|
isSupportedDocument,
|
|
parseDocument,
|
|
parsePdfPages
|
|
} from "../../src/modules/parsers/parser-registry.js";
|
|
|
|
const fixturePath = new URL("../fixtures/ocr/native-three-pages.pdf", import.meta.url);
|
|
|
|
test("parsePdfPages extracts ordered one-based pages with stable native text hashes", async () => {
|
|
const pages = await parsePdfPages(fixturePath);
|
|
|
|
assert.deepEqual(pages, [
|
|
{
|
|
page: 1,
|
|
text: "Native page one.",
|
|
rasterCoverage: 0,
|
|
textSha256: "b9efe3745cacd6c87189435d7b83374ee7e5f77fca9d52e8d4e76f4a12a419f4"
|
|
},
|
|
{
|
|
page: 2,
|
|
text: "",
|
|
rasterCoverage: 0,
|
|
textSha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
|
},
|
|
{
|
|
page: 3,
|
|
text: "Native page three.",
|
|
rasterCoverage: 0,
|
|
textSha256: "e50a3249ccbe373afaac6ed06c36692f7190b260f8c55f21b45e4a5066e9352e"
|
|
}
|
|
]);
|
|
});
|
|
|
|
test("parseDocument preserves the synchronous native PDF content contract", async () => {
|
|
const parsed = await parseDocument(fixturePath.pathname);
|
|
|
|
assert.equal(parsed.title, "native-three-pages.pdf");
|
|
assert.equal(parsed.content, "Native page one.\n\nNative page three.");
|
|
assert.equal(parsed.mimeType, "application/pdf");
|
|
assert.equal(parsed.chunkMode, "documental");
|
|
});
|
|
|
|
test("page extraction rejects non-PDF inputs before parsing bytes", async () => {
|
|
await assert.rejects(
|
|
parsePdfPages(new URL("../fixtures/ocr/not-a-pdf.txt", import.meta.url)),
|
|
/only PDF documents support page extraction/i
|
|
);
|
|
});
|
|
|
|
test("supported text documents are read as data and never executed", async () => {
|
|
const directory = await mkdtemp(path.join(os.tmpdir(), "rag-parser-threat-"));
|
|
const markerPath = path.join(directory, "executed");
|
|
const payload = `$(touch ${markerPath})`;
|
|
|
|
try {
|
|
for (const fileName of ["requirements.txt", "executable.md"]) {
|
|
const filePath = path.join(directory, fileName);
|
|
await writeFile(filePath, payload, "utf8");
|
|
const parsed = await parseDocument(filePath);
|
|
|
|
assert.equal(isSupportedDocument(filePath), true);
|
|
assert.equal(parsed.content, payload);
|
|
}
|
|
await assert.rejects(access(markerPath), /ENOENT/);
|
|
} finally {
|
|
await rm(directory, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("build and executable lookalike formats remain unsupported", () => {
|
|
assert.deepEqual(
|
|
["CMakeLists.txt", "component.mdx", "README.sh"].map(isSupportedDocument),
|
|
[false, false, false]
|
|
);
|
|
});
|