rag-service/src/modules/catalog/legacy-migration.ts

import { buildDocumentId, buildMetadataHash, hashOrderedPairs, sha256Hex } from "../../shared/utils/ids.js";
import type { SourceType } from "../../shared/types/rag.js";

export type LegacyPointId = string | number;

export interface LegacyPointSnapshot {
  id: LegacyPointId;
  payload: Record<string, unknown>;
}

export interface LegacyDocumentPlan {
  documentId: string;
  documentKey: string;
  originalHash: string;
  contentHash: string;
  mimeType: string;
  title: string;
  chunkCount: number;
}

export interface LegacySourcePlan {
  sourceId: string;
  sourceType: SourceType;
  sourceRef: string;
  pointIds: LegacyPointId[];
  tags: string[];
  documents: LegacyDocumentPlan[];
  originalManifestHash: string;
  sourceContentHash: string;
  metadataHash: string;
  embeddingProvider: string;
  embeddingModel: string;
  embeddingDimensions: number;
  blockedReasons: string[];
}

export interface LegacyMigrationPlan {
  ok: boolean;
  sources: LegacySourcePlan[];
  blockedReasons: string[];
}

function stringValue(value: unknown): string {
  return typeof value === "string" ? value.trim() : "";
}

function stringArray(value: unknown): string[] {
  return Array.isArray(value) ? value.map(String).filter(Boolean).sort() : [];
}

function unique(values: string[]): string[] {
  return [...new Set(values.filter(Boolean))];
}

function sortedByUtf8<T>(values: T[], getKey: (value: T) => string): T[] {
  return [...values].sort((left, right) => Buffer.compare(Buffer.from(getKey(left), "utf8"), Buffer.from(getKey(right), "utf8")));
}

export function buildLegacyMigrationPlan(points: LegacyPointSnapshot[]): LegacyMigrationPlan {
  const groups = new Map<string, LegacyPointSnapshot[]>();
  const globalBlocked: string[] = [];

  for (const point of points) {
    const sourceId = stringValue(point.payload.source_id);
    if (!sourceId) {
      globalBlocked.push(`Point ${String(point.id)} is missing source_id.`);
      continue;
    }
    const entries = groups.get(sourceId) ?? [];
    entries.push(point);
    groups.set(sourceId, entries);
  }

  const sources = sortedByUtf8([...groups.entries()], ([sourceId]) => sourceId).map(([sourceId, sourcePoints]) => {
    const blockedReasons: string[] = [];
    const sourceRefs = unique(sourcePoints.map((point) => stringValue(point.payload.source_ref)));
    const sourceTypes = unique(sourcePoints.map((point) => stringValue(point.payload.source_type)));
    const providers = unique(sourcePoints.map((point) => stringValue(point.payload.embedding_provider)));
    const models = unique(sourcePoints.map((point) => stringValue(point.payload.embedding_model)));
    const dimensions = unique(sourcePoints.map((point) => String(Number(point.payload.embedding_dimensions ?? 0))).filter((value) => value !== "0"));
    const tagSnapshot = unique(sourcePoints.flatMap((point) => stringArray(point.payload.tags)));

    if (sourceRefs.length !== 1) blockedReasons.push("Source has zero or multiple source_ref values.");
    if (sourceTypes.length !== 1 || !["file", "folder"].includes(sourceTypes[0])) blockedReasons.push("Source has no unique file/folder source_type.");
    if (providers.length !== 1) blockedReasons.push("Source has no unique embedding provider.");
    if (models.length !== 1) blockedReasons.push("Source has no unique embedding model.");
    if (dimensions.length !== 1) blockedReasons.push("Source has no unique non-zero embedding dimensions.");

    const documentGroups = new Map<string, LegacyPointSnapshot[]>();
    for (const point of sourcePoints) {
      const documentId = stringValue(point.payload.document_id);
      const documentKey = stringValue(point.payload.document_key);
      const chunkId = stringValue(point.payload.chunk_id);
      const content = stringValue(point.payload.content);
      if (!documentId || !documentKey || !chunkId || !content) {
        blockedReasons.push(`Point ${String(point.id)} has weak document identity or empty content.`);
        continue;
      }
      if (documentId !== buildDocumentId(sourceId, documentKey)) {
        blockedReasons.push(`Point ${String(point.id)} document_id does not match source_id and document_key.`);
      }
      const existing = documentGroups.get(documentId) ?? [];
      existing.push(point);
      documentGroups.set(documentId, existing);
    }

    const documents: LegacyDocumentPlan[] = sortedByUtf8([...documentGroups.entries()], ([documentId]) => documentId).map(([documentId, documentPoints]) => {
      const documentKeys = unique(documentPoints.map((point) => stringValue(point.payload.document_key)));
      const mimeTypes = unique(documentPoints.map((point) => stringValue(point.payload.mime_type) || "text/plain"));
      const titles = unique(documentPoints.map((point) => stringValue(point.payload.title) || documentKeys[0]));
      if (documentKeys.length !== 1) blockedReasons.push(`Document ${documentId} has no unique document_key.`);
      if (mimeTypes.length !== 1) blockedReasons.push(`Document ${documentId} has no unique mime_type.`);
      const orderedContents = sortedByUtf8(documentPoints, (point) => stringValue(point.payload.chunk_id))
        .map((point) => stringValue(point.payload.content))
        .join("\n");
      const contentHash = sha256Hex(orderedContents);
      return {
        documentId,
        documentKey: documentKeys[0] ?? "",
        originalHash: contentHash,
        contentHash,
        mimeType: mimeTypes[0] ?? "text/plain",
        title: titles[0] ?? documentKeys[0] ?? documentId,
        chunkCount: documentPoints.length
      };
    });

    const originalManifestHash = hashOrderedPairs(documents.map((document) => [document.documentKey, document.originalHash])) ?? "";
    const sourceContentHash = hashOrderedPairs(documents.map((document) => [document.documentKey, document.contentHash])) ?? "";

    return {
      sourceId,
      sourceType: sourceTypes[0] as SourceType,
      sourceRef: sourceRefs[0] ?? "",
      pointIds: sourcePoints.map((point) => point.id),
      tags: tagSnapshot,
      documents,
      originalManifestHash,
      sourceContentHash,
      metadataHash: buildMetadataHash(tagSnapshot, { sourceType: sourceTypes[0], sourceRef: sourceRefs[0] }),
      embeddingProvider: providers[0] ?? "",
      embeddingModel: models[0] ?? "",
      embeddingDimensions: Number(dimensions[0] ?? 0),
      blockedReasons: unique(blockedReasons)
    };
  });

  const blockedReasons = [...globalBlocked, ...sources.flatMap((source) => source.blockedReasons.map((reason) => `${source.sourceId}: ${reason}`))];
  return { ok: blockedReasons.length === 0, sources, blockedReasons };
}