import { buildDocumentId, buildMetadataHash, hashOrderedPairs, sha256Hex } from "../../shared/utils/ids.js"; import type { SourceType } from "../../shared/types/rag.js"; export type LegacyPointId = string | number; export interface LegacyPointSnapshot { id: LegacyPointId; payload: Record; } export interface LegacyDocumentPlan { documentId: string; documentKey: string; originalHash: string; contentHash: string; mimeType: string; title: string; chunkCount: number; } export interface LegacySourcePlan { sourceId: string; sourceType: SourceType; sourceRef: string; pointIds: LegacyPointId[]; tags: string[]; documents: LegacyDocumentPlan[]; originalManifestHash: string; sourceContentHash: string; metadataHash: string; embeddingProvider: string; embeddingModel: string; embeddingDimensions: number; blockedReasons: string[]; } export interface LegacyMigrationPlan { ok: boolean; sources: LegacySourcePlan[]; blockedReasons: string[]; } function stringValue(value: unknown): string { return typeof value === "string" ? value.trim() : ""; } function stringArray(value: unknown): string[] { return Array.isArray(value) ? value.map(String).filter(Boolean).sort() : []; } function unique(values: string[]): string[] { return [...new Set(values.filter(Boolean))]; } function sortedByUtf8(values: T[], getKey: (value: T) => string): T[] { return [...values].sort((left, right) => Buffer.compare(Buffer.from(getKey(left), "utf8"), Buffer.from(getKey(right), "utf8"))); } export function buildLegacyMigrationPlan(points: LegacyPointSnapshot[]): LegacyMigrationPlan { const groups = new Map(); const globalBlocked: string[] = []; for (const point of points) { const sourceId = stringValue(point.payload.source_id); if (!sourceId) { globalBlocked.push(`Point ${String(point.id)} is missing source_id.`); continue; } const entries = groups.get(sourceId) ?? []; entries.push(point); groups.set(sourceId, entries); } const sources = sortedByUtf8([...groups.entries()], ([sourceId]) => sourceId).map(([sourceId, sourcePoints]) => { const blockedReasons: string[] = []; const sourceRefs = unique(sourcePoints.map((point) => stringValue(point.payload.source_ref))); const sourceTypes = unique(sourcePoints.map((point) => stringValue(point.payload.source_type))); const providers = unique(sourcePoints.map((point) => stringValue(point.payload.embedding_provider))); const models = unique(sourcePoints.map((point) => stringValue(point.payload.embedding_model))); const dimensions = unique(sourcePoints.map((point) => String(Number(point.payload.embedding_dimensions ?? 0))).filter((value) => value !== "0")); const tagSnapshot = unique(sourcePoints.flatMap((point) => stringArray(point.payload.tags))); if (sourceRefs.length !== 1) blockedReasons.push("Source has zero or multiple source_ref values."); if (sourceTypes.length !== 1 || !["file", "folder"].includes(sourceTypes[0])) blockedReasons.push("Source has no unique file/folder source_type."); if (providers.length !== 1) blockedReasons.push("Source has no unique embedding provider."); if (models.length !== 1) blockedReasons.push("Source has no unique embedding model."); if (dimensions.length !== 1) blockedReasons.push("Source has no unique non-zero embedding dimensions."); const documentGroups = new Map(); for (const point of sourcePoints) { const documentId = stringValue(point.payload.document_id); const documentKey = stringValue(point.payload.document_key); const chunkId = stringValue(point.payload.chunk_id); const content = stringValue(point.payload.content); if (!documentId || !documentKey || !chunkId || !content) { blockedReasons.push(`Point ${String(point.id)} has weak document identity or empty content.`); continue; } if (documentId !== buildDocumentId(sourceId, documentKey)) { blockedReasons.push(`Point ${String(point.id)} document_id does not match source_id and document_key.`); } const existing = documentGroups.get(documentId) ?? []; existing.push(point); documentGroups.set(documentId, existing); } const documents: LegacyDocumentPlan[] = sortedByUtf8([...documentGroups.entries()], ([documentId]) => documentId).map(([documentId, documentPoints]) => { const documentKeys = unique(documentPoints.map((point) => stringValue(point.payload.document_key))); const mimeTypes = unique(documentPoints.map((point) => stringValue(point.payload.mime_type) || "text/plain")); const titles = unique(documentPoints.map((point) => stringValue(point.payload.title) || documentKeys[0])); if (documentKeys.length !== 1) blockedReasons.push(`Document ${documentId} has no unique document_key.`); if (mimeTypes.length !== 1) blockedReasons.push(`Document ${documentId} has no unique mime_type.`); const orderedContents = sortedByUtf8(documentPoints, (point) => stringValue(point.payload.chunk_id)) .map((point) => stringValue(point.payload.content)) .join("\n"); const contentHash = sha256Hex(orderedContents); return { documentId, documentKey: documentKeys[0] ?? "", originalHash: contentHash, contentHash, mimeType: mimeTypes[0] ?? "text/plain", title: titles[0] ?? documentKeys[0] ?? documentId, chunkCount: documentPoints.length }; }); const originalManifestHash = hashOrderedPairs(documents.map((document) => [document.documentKey, document.originalHash])) ?? ""; const sourceContentHash = hashOrderedPairs(documents.map((document) => [document.documentKey, document.contentHash])) ?? ""; return { sourceId, sourceType: sourceTypes[0] as SourceType, sourceRef: sourceRefs[0] ?? "", pointIds: sourcePoints.map((point) => point.id), tags: tagSnapshot, documents, originalManifestHash, sourceContentHash, metadataHash: buildMetadataHash(tagSnapshot, { sourceType: sourceTypes[0], sourceRef: sourceRefs[0] }), embeddingProvider: providers[0] ?? "", embeddingModel: models[0] ?? "", embeddingDimensions: Number(dimensions[0] ?? 0), blockedReasons: unique(blockedReasons) }; }); const blockedReasons = [...globalBlocked, ...sources.flatMap((source) => source.blockedReasons.map((reason) => `${source.sourceId}: ${reason}`))]; return { ok: blockedReasons.length === 0, sources, blockedReasons }; }