import { readFile } from "node:fs/promises"; import path from "node:path"; import pdf from "pdf-parse"; import type { ChunkingMode } from "../process/chunking.js"; export interface ParsedDocument { title: string; content: string; mimeType: string; chunkMode: ChunkingMode; } const documentalExtensions = [".md", ".txt", ".pdf"] as const; const codeExtensions = [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".json", ".yml", ".yaml"] as const; const parserExtensions = [...documentalExtensions, ...codeExtensions] as const; export function supportedParserExtensions(): string[] { return [...parserExtensions]; } export function isSupportedDocument(filePath: string): boolean { return parserExtensions.includes(path.extname(filePath).toLowerCase() as (typeof parserExtensions)[number]); } export function inferChunkMode(filePath: string): ChunkingMode { const extension = path.extname(filePath).toLowerCase(); if (codeExtensions.includes(extension as (typeof codeExtensions)[number])) { return "codigo"; } return "documental"; } function inferMimeType(extension: string, chunkMode: ChunkingMode): string { if (extension === ".pdf") { return "application/pdf"; } if (extension === ".md") { return "text/markdown"; } if (chunkMode === "codigo") { return "text/x-code"; } return "text/plain"; } export async function parseDocument(filePath: string): Promise { const extension = path.extname(filePath).toLowerCase(); const chunkMode = inferChunkMode(filePath); if (extension === ".pdf") { const buffer = await readFile(filePath); const result = await pdf(buffer); return { title: path.basename(filePath), content: result.text.trim(), mimeType: inferMimeType(extension, chunkMode), chunkMode }; } const content = await readFile(filePath, "utf8"); return { title: path.basename(filePath), content, mimeType: inferMimeType(extension, chunkMode), chunkMode }; }