diff --git a/RAG/docs/HISTORIAL_SESIONES.md b/RAG/docs/HISTORIAL_SESIONES.md index ec07000..a154ff8 100644 --- a/RAG/docs/HISTORIAL_SESIONES.md +++ b/RAG/docs/HISTORIAL_SESIONES.md @@ -47,3 +47,4 @@ Dar continuidad al RAG en `RAG/` a partir del estado actual documentado. - Limpieza ejecutada exitosamente sobre el `scope` del código fuente antiguo (`RAG/src`). - Reingesta del directorio `RAG/src` con el código actualizado. - Documento de seguimiento `RAG/docs/TASK_LIMPIEZA.md` y documentacion API `RAG/docs/API_RAG.md` actualizados. +- Implementacion de ingesta directa de carpetas locales desde el playground: el navegador respeta `.gitignore`, empaqueta la carpeta en un `.zip` en memoria y el backend usa `adm-zip` para extraerla de forma segura en un directorio temporal antes de la ingesta. diff --git a/RAG/package-lock.json b/RAG/package-lock.json index 3393d17..beb2bb3 100644 --- a/RAG/package-lock.json +++ b/RAG/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "dependencies": { "@qdrant/js-client-rest": "^1.15.0", + "adm-zip": "^0.5.17", "dotenv": "^16.4.5", "express": "^4.21.2", "multer": "^2.0.0", @@ -16,6 +17,7 @@ "pdf-parse": "^1.1.1" }, "devDependencies": { + "@types/adm-zip": "^0.5.8", "@types/express": "^5.0.1", "@types/multer": "^1.4.12", "@types/node": "^22.15.3", @@ -493,6 +495,16 @@ "pnpm": ">=8" } }, + "node_modules/@types/adm-zip": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@types/adm-zip/-/adm-zip-0.5.8.tgz", + "integrity": "sha512-RVVH7QvZYbN+ihqZ4kX/dMiowf6o+Jk1fNwiSdx0NahBJLU787zkULhGhJM8mf/obmLGmgdMM0bXsQTmyfbR7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/body-parser": { "version": "1.19.6", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", @@ -645,6 +657,15 @@ "node": ">= 0.6" } }, + "node_modules/adm-zip": { + "version": "0.5.17", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.17.tgz", + "integrity": "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/agentkeepalive": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", diff --git a/RAG/package.json b/RAG/package.json index e278ba3..e5ea328 100644 --- a/RAG/package.json +++ b/RAG/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@qdrant/js-client-rest": "^1.15.0", + "adm-zip": "^0.5.17", "dotenv": "^16.4.5", "express": "^4.21.2", "multer": "^2.0.0", @@ -18,6 +19,7 @@ "pdf-parse": "^1.1.1" }, "devDependencies": { + "@types/adm-zip": "^0.5.8", "@types/express": "^5.0.1", "@types/multer": "^1.4.12", "@types/node": "^22.15.3", diff --git a/RAG/public/playground/app.js b/RAG/public/playground/app.js index 62abed0..c687983 100644 --- a/RAG/public/playground/app.js +++ b/RAG/public/playground/app.js @@ -31,6 +31,10 @@ const ingestSourceIdWrapper = document.getElementById("ingestSourceIdWrapper"); const ingestSourceId = document.getElementById("ingestSourceId"); const ingestSourceRef = document.getElementById("ingestSourceRef"); const ingestUploadFile = document.getElementById("ingestUploadFile"); +const ingestUploadFolder = document.getElementById("ingestUploadFolder"); +const btnUploadFile = document.getElementById("btnUploadFile"); +const btnUploadFolder = document.getElementById("btnUploadFolder"); +const uploadStatusText = document.getElementById("uploadStatusText"); const ingestMode = document.getElementById("ingestMode"); const ingestTags = document.getElementById("ingestTags"); const ingestModeHint = document.getElementById("ingestModeHint"); @@ -63,6 +67,8 @@ let chatHistory = []; let availableScopes = []; let lastInteraction = null; +let currentUploadType = null; // 'file' o 'folder' + function format(value) { return JSON.stringify(value, null, 2); } @@ -77,17 +83,29 @@ function buildScopeLabel(scope) { } function updateIngestUiState() { - const hasUpload = Boolean(ingestUploadFile.files && ingestUploadFile.files[0]); + const hasFile = Boolean(ingestUploadFile.files && ingestUploadFile.files.length > 0); + const hasFolder = Boolean(ingestUploadFolder.files && ingestUploadFolder.files.length > 0); + const hasUpload = hasFile || hasFolder; + ingestSourceType.value = hasUpload ? "file" : ingestSourceType.value; ingestSourceType.disabled = hasUpload; ingestSourceRef.disabled = hasUpload; ingestSourceIdWrapper.style.display = ingestScopeMode.value === "custom" ? "block" : "none"; - if (hasUpload) { - ingestModeHint.textContent = `Upload directo activo: se ingerira el archivo local "${ingestUploadFile.files[0].name}" y se ignorara la ruta manual.`; + if (hasFile) { + uploadStatusText.textContent = `Archivo seleccionado: ${ingestUploadFile.files[0].name}`; + ingestModeHint.textContent = `Upload directo activo: se ingerira el archivo local "${ingestUploadFile.files[0].name}" y se ignorara la ruta remota.`; + ingestModeHint.classList.add("strong"); + } else if (hasFolder) { + // Al seleccionar carpeta mostramos el nombre del primer archivo padre y cuantos ficheros hay + const firstPath = ingestUploadFolder.files[0].webkitRelativePath || ""; + const folderName = firstPath.split('/')[0] || "Carpeta"; + uploadStatusText.textContent = `Carpeta seleccionada: ${folderName} (${ingestUploadFolder.files.length} archivos totales, se filtraran ignorados)`; + ingestModeHint.textContent = `Upload directo activo: se comprimira y subira la carpeta local "${folderName}" y se ignorara la ruta remota.`; ingestModeHint.classList.add("strong"); } else { - ingestModeHint.textContent = "Si seleccionas un archivo local, el playground usara upload directo y podras aislarlo con un `sourceId` propio para no mezclarlo con otros scopes."; + uploadStatusText.textContent = "Ningun elemento seleccionado"; + ingestModeHint.textContent = "Si seleccionas un archivo o carpeta local, el playground lo subira directamente y podras aislarlo con un `sourceId` propio."; ingestModeHint.classList.remove("strong"); } } @@ -285,7 +303,20 @@ cleanupScopeSelect.addEventListener("change", () => { applySelectedCleanupScope(scope); }); +btnUploadFile.addEventListener("click", () => { + ingestUploadFolder.value = ""; + currentUploadType = 'file'; + ingestUploadFile.click(); +}); + +btnUploadFolder.addEventListener("click", () => { + ingestUploadFile.value = ""; + currentUploadType = 'folder'; + ingestUploadFolder.click(); +}); + ingestUploadFile.addEventListener("change", updateIngestUiState); +ingestUploadFolder.addEventListener("change", updateIngestUiState); ingestScopeMode.addEventListener("change", updateIngestUiState); scopeEditMode.addEventListener("change", updateScopeEditState); @@ -304,7 +335,7 @@ ingestButton.addEventListener("click", async () => { try { let data; - if (ingestUploadFile.files && ingestUploadFile.files[0]) { + if (currentUploadType === 'file' && ingestUploadFile.files && ingestUploadFile.files[0]) { const formData = new FormData(); formData.append("file", ingestUploadFile.files[0]); formData.append("mode", ingestMode.value); @@ -318,9 +349,60 @@ ingestButton.addEventListener("click", async () => { body: formData }); data = await response.json(); - if (!response.ok) { - throw new Error(data.error || `HTTP ${response.status}`); + if (!response.ok) throw new Error(data.error || `HTTP ${response.status}`); + + } else if (currentUploadType === 'folder' && ingestUploadFolder.files && ingestUploadFolder.files.length > 0) { + ingestResult.textContent = "Empaquetando carpeta local (esto puede tardar unos segundos)..."; + + const zip = new JSZip(); + const ig = ignore(); + + // Buscar .gitignore en la raiz + const gitignoreFile = Array.from(ingestUploadFolder.files).find(f => f.webkitRelativePath.match(/^[^\/]+\/\.gitignore$/)); + if (gitignoreFile) { + const content = await gitignoreFile.text(); + ig.add(content); } + + // Reglas hardcodeadas de seguridad + ig.add(['node_modules/', '.git/', '.venv/', 'dist/', 'build/']); + + let addedCount = 0; + for (const file of ingestUploadFolder.files) { + // webkitRelativePath format: "FolderName/path/to/file.ext" + // Le quitamos el primer segmento (FolderName) para validar con ignore correctamente + const relativePath = file.webkitRelativePath.split('/').slice(1).join('/'); + + if (relativePath && !ig.ignores(relativePath)) { + zip.file(relativePath, file); + addedCount++; + } + } + + if (addedCount === 0) { + throw new Error("La carpeta esta vacia o todos sus archivos fueron ignorados (.gitignore, node_modules, etc)."); + } + + ingestResult.textContent = `Subiendo paquete comprimido con ${addedCount} archivos...`; + const zipBlob = await zip.generateAsync({ type: "blob", compression: "STORE" }); + const folderName = ingestUploadFolder.files[0].webkitRelativePath.split('/')[0] || "upload"; + + const formData = new FormData(); + formData.append("file", zipBlob, `${folderName}.zip`); + formData.append("isZipFolder", "true"); + formData.append("mode", ingestMode.value); + formData.append("tags", splitTags(ingestTags.value).join(",")); + if (ingestScopeMode.value === "custom" && ingestSourceId.value.trim()) { + formData.append("sourceId", ingestSourceId.value.trim()); + } + + const response = await fetch("/ingest/upload", { + method: "POST", + body: formData + }); + data = await response.json(); + if (!response.ok) throw new Error(data.error || `HTTP ${response.status}`); + } else { data = await request("/ingest", { sourceId: ingestScopeMode.value === "custom" ? (ingestSourceId.value.trim() || undefined) : undefined, diff --git a/RAG/public/playground/index.html b/RAG/public/playground/index.html index f51fa8a..4d57481 100644 --- a/RAG/public/playground/index.html +++ b/RAG/public/playground/index.html @@ -5,6 +5,8 @@ RAG Playground + +
@@ -52,11 +54,17 @@ -