import express from "express"; import multer from "multer"; import type { Request } from "express"; import { writeFile, unlink } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { env } from "./config/env.js"; import { AnswerService } from "./modules/answer/service.js"; import { IngestService } from "./modules/ingest/service.js"; import { OpenRouterEmbeddingProvider } from "./modules/embeddings/provider.js"; import { EvaluationLogService } from "./modules/logs/service.js"; import { documentalChunkingPolicy } from "./modules/process/chunking.js"; import { RetrieveService } from "./modules/retrieve/service.js"; import { supportedParserExtensions } from "./modules/parsers/parser-registry.js"; import { QdrantVectorStoreClient } from "./modules/vectorstore/client.js"; import type { ChatMessage, ChunkMode, RetrieveIntent, RetrieveScope } from "./shared/types/rag.js"; type UploadRequest = Request & { file?: Express.Multer.File; }; export function createApp() { const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const publicDir = path.resolve(__dirname, "../public"); const app = express(); const upload = multer({ storage: multer.memoryStorage() }); const embeddingProvider = new OpenRouterEmbeddingProvider(); const vectorStore = new QdrantVectorStoreClient(); const evaluationLogs = new EvaluationLogService(embeddingProvider); const ingestService = new IngestService(embeddingProvider, vectorStore); const retrieveService = new RetrieveService(embeddingProvider, vectorStore); const answerService = new AnswerService(retrieveService); function needsContextLog(summary?: string, itemsCount = 0, answer?: string) { if (itemsCount === 0) { return true; } const text = `${summary ?? ""} ${answer ?? ""}`; return /no se recupero contexto|no hay informacion suficiente|no dispongo de mas detalles|contexto insuficiente/i.test(text); } app.use(express.json({ limit: "5mb" })); app.use(express.static(publicDir)); app.get("/playground", (_req, res) => { res.sendFile(path.join(publicDir, "playground", "index.html")); }); app.get("/health", async (_req, res) => { const vectorHealth = await vectorStore.healthcheck(); res.json({ ok: true, service: "rag", environment: env.nodeEnv, embeddings: { provider: embeddingProvider.providerName, model: embeddingProvider.modelName }, answer: { provider: env.answerProvider, model: env.answerModel }, vectorStore: vectorHealth, parsers: supportedParserExtensions(), chunking: documentalChunkingPolicy }); }); app.get("/sources", async (_req, res) => { try { const scopes = await vectorStore.listScopes(); res.json(scopes); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown sources error" }); } }); app.get("/models/answer", async (_req, res) => { try { const models = await answerService.listAvailableAnswerModels(); res.json({ defaultModel: env.answerModel, models }); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown answer models error" }); } }); app.get("/logs/recent", async (req, res) => { try { const limit = req.query.limit ? Number(req.query.limit) : 20; const logs = await evaluationLogs.listRecent(limit); res.json(logs); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown logs error" }); } }); app.post("/ingest", async (req, res) => { try { const result = await ingestService.ingest(req.body); res.status(202).json(result); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown ingest error" }); } }); app.post("/ingest/upload", upload.single("file"), async (req: UploadRequest, res) => { let tempPath: string | undefined; try { if (!req.file) { res.status(400).json({ ok: false, error: "Missing file upload" }); return; } const tempDir = await os.tmpdir(); tempPath = path.join(tempDir, `${Date.now()}-${req.file.originalname}`); await writeFile(tempPath, req.file.buffer); const tags = typeof req.body.tags === "string" ? req.body.tags.split(",").map((entry: string) => entry.trim()).filter(Boolean) : []; const result = await ingestService.ingest({ sourceId: req.body.sourceId ? String(req.body.sourceId) : undefined, sourceType: "file", sourceRef: req.file.originalname, readPath: tempPath, mode: req.body.mode === "interactive" ? "interactive" : "mechanical", tags }); res.status(202).json({ ...result, uploadedFile: req.file.originalname }); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown upload ingest error" }); } finally { if (tempPath) { await unlink(tempPath).catch(() => undefined); } } }); app.post("/retrieve", async (req, res) => { try { const mode = (req.body.mode ?? "auto") as ChunkMode; const intent = (req.body.intent ?? "specific") as RetrieveIntent; const query = String(req.body.query ?? ""); const model = req.body.model ? String(req.body.model) : undefined; const useModel = Boolean(req.body.useModelInRetrieve); const scope: RetrieveScope | undefined = req.body.scope ? { sourceId: req.body.scope.sourceId, sourceRef: req.body.scope.sourceRef, tags: Array.isArray(req.body.scope.tags) ? req.body.scope.tags.map(String) : undefined } : undefined; const result = await retrieveService.retrieve(mode, intent, query, scope); const items = result.items; if (useModel) { const modelSummary = await answerService.summarizeRetrieve(query, result, model); const payload = { ...result, model: modelSummary.model, modelSummary: modelSummary.summary }; if (needsContextLog(payload.modelSummary, items.length)) { await evaluationLogs.log({ trigger: "automatic", operation: "retrieve", reason: "retrieve_context_insufficient", query, mode, intent, scope, model: payload.model, responseSummary: payload.modelSummary, retrievedItems: items }); } res.json(payload); return; } if (needsContextLog(result.summary, items.length)) { await evaluationLogs.log({ trigger: "automatic", operation: "retrieve", reason: "retrieve_context_insufficient", query, mode, intent, scope, responseSummary: result.summary, retrievedItems: items }); } res.json(result); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown retrieve error" }); } }); app.post("/answer", async (req, res) => { try { const mode = (req.body.mode ?? "auto") as ChunkMode; const intent = (req.body.intent ?? "specific") as RetrieveIntent; const query = String(req.body.query ?? ""); const scope: RetrieveScope | undefined = req.body.scope ? { sourceId: req.body.scope.sourceId, sourceRef: req.body.scope.sourceRef, tags: Array.isArray(req.body.scope.tags) ? req.body.scope.tags.map(String) : undefined } : undefined; const model = req.body.model ? String(req.body.model) : undefined; const preloadedContext = req.body.preloadedContext ? String(req.body.preloadedContext) : undefined; const result = await answerService.answer(mode, intent, query, scope, model, preloadedContext); if (needsContextLog(result.summary, result.citations.length, result.answer)) { await evaluationLogs.log({ trigger: "automatic", operation: "answer", reason: "answer_context_insufficient", query, mode, intent, scope, model: result.model, note: preloadedContext ? "bootstrap_context_present" : undefined, usedBootstrapContext: Boolean(preloadedContext), responseSummary: result.answer, retrievedItems: result.citations.map((citation) => ({ chunkId: citation.chunkId, documentId: citation.documentId, sourceId: scope?.sourceId ?? "", title: citation.title, sectionTitle: citation.sectionTitle, content: "", score: 0, startLine: citation.startLine, endLine: citation.endLine })) }); } res.json(result); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown answer error" }); } }); app.post("/answer/direct", async (req, res) => { try { const query = String(req.body.query ?? ""); const model = req.body.model ? String(req.body.model) : undefined; const preloadedContext = req.body.preloadedContext ? String(req.body.preloadedContext) : undefined; const result = await answerService.answerWithoutRag(query, model, preloadedContext); res.json(result); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown direct answer error" }); } }); app.post("/chat", async (req, res) => { try { const message = String(req.body.message ?? ""); const mode = (req.body.mode ?? "documental") as ChunkMode; const model = req.body.model ? String(req.body.model) : undefined; const preloadedContext = req.body.preloadedContext ? String(req.body.preloadedContext) : undefined; const allowAdditionalRetrieve = Boolean(req.body.allowAdditionalRetrieve); const historyEntries = Array.isArray(req.body.history) ? req.body.history as Array<{ role?: string; content?: unknown }> : []; const history: ChatMessage[] = historyEntries.length > 0 ? historyEntries .map((entry: { role?: string; content?: unknown }): ChatMessage => ({ role: entry.role === "assistant" ? "assistant" : "user", content: String(entry.content ?? "") })) .filter((entry) => entry.content.length > 0) : []; const scope: RetrieveScope | undefined = req.body.scope ? { sourceId: req.body.scope.sourceId, sourceRef: req.body.scope.sourceRef, tags: Array.isArray(req.body.scope.tags) ? req.body.scope.tags.map(String) : undefined } : undefined; const result = await answerService.chat({ message, history, mode, scope, modelOverride: model, preloadedContext, allowAdditionalRetrieve }); if (needsContextLog(result.retrieved?.summary, result.retrieved?.items.length ?? 0, result.answer)) { await evaluationLogs.log({ trigger: "automatic", operation: "chat", reason: "chat_context_insufficient", query: message, mode, intent: "specific", scope, model: result.model, usedBootstrapContext: result.usedBootstrapContext, usedAdditionalRetrieve: result.usedAdditionalRetrieve, responseSummary: result.answer, retrievedItems: result.retrieved?.items ?? [] }); } res.json(result); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown chat error" }); } }); app.post("/logs/manual", async (req, res) => { try { const query = String(req.body.query ?? ""); const entry = await evaluationLogs.log({ trigger: "manual", operation: req.body.operation === "chat" || req.body.operation === "retrieve" ? req.body.operation : "answer", reason: req.body.reason ? String(req.body.reason) : "manual_review_requested", query, mode: req.body.mode, intent: req.body.intent, scope: req.body.scope, model: req.body.model, note: req.body.note ? `Log solicitado por el usuario. Nota: ${String(req.body.note)}` : "Log solicitado por el usuario.", usedBootstrapContext: Boolean(req.body.usedBootstrapContext), usedAdditionalRetrieve: Boolean(req.body.usedAdditionalRetrieve), responseSummary: req.body.responseSummary ? String(req.body.responseSummary) : undefined, retrievedItems: Array.isArray(req.body.retrievedItems) ? req.body.retrievedItems : [] }); res.status(201).json(entry); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown manual log error" }); } }); app.patch("/logs/:id", async (req, res) => { try { const entry = await evaluationLogs.update(String(req.params.id), { reviewStatus: req.body.reviewStatus, severity: req.body.severity, reviewedBy: req.body.reviewedBy ? String(req.body.reviewedBy) : undefined, resolutionNote: req.body.resolutionNote ? String(req.body.resolutionNote) : undefined, fixReference: req.body.fixReference ? String(req.body.fixReference) : undefined, supersedesLogId: req.body.supersedesLogId ? String(req.body.supersedesLogId) : undefined }); if (!entry) { res.status(404).json({ ok: false, error: "Log not found" }); return; } res.json(entry); } catch (error) { res.status(500).json({ ok: false, error: error instanceof Error ? error.message : "Unknown log update error" }); } }); return app; }