rag-service/migrations/001_knowledge_lifecycle.sql

CREATE EXTENSION IF NOT EXISTS pgcrypto;

CREATE TABLE IF NOT EXISTS rag_sources (
  source_id text PRIMARY KEY,
  tenant text NOT NULL DEFAULT 'default',
  source_type text NOT NULL CHECK (source_type IN ('file', 'folder')),
  source_ref text NOT NULL,
  active_version_id uuid NULL,
  needs_reingest boolean NOT NULL DEFAULT false,
  disabled_at timestamptz NULL,
  created_at timestamptz NOT NULL DEFAULT now(),
  updated_at timestamptz NOT NULL DEFAULT now()
);

CREATE TABLE IF NOT EXISTS rag_source_versions (
  version_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  source_id text NOT NULL REFERENCES rag_sources(source_id) ON DELETE RESTRICT,
  version_number bigint NOT NULL CHECK (version_number > 0),
  previous_version_id uuid NULL,
  state text NOT NULL CHECK (state IN (
    'pending', 'indexing', 'review_required', 'ready', 'active',
    'superseded', 'failed', 'rejected', 'purging', 'purged'
  )),
  original_manifest_hash char(64) NOT NULL,
  source_content_hash char(64) NULL,
  processing_fingerprint char(64) NOT NULL,
  metadata_hash char(64) NOT NULL,
  tags text[] NOT NULL DEFAULT '{}',
  activate_requested boolean NOT NULL DEFAULT true,
  base_active_version_id uuid NULL,
  embedding_provider text NOT NULL,
  embedding_model text NOT NULL,
  embedding_dimensions integer NOT NULL CHECK (embedding_dimensions > 0),
  qdrant_collection text NOT NULL,
  expected_document_count integer NOT NULL DEFAULT 0 CHECK (expected_document_count >= 0),
  expected_point_count integer NOT NULL DEFAULT 0 CHECK (expected_point_count >= 0),
  verified_point_count integer NOT NULL DEFAULT 0 CHECK (verified_point_count >= 0),
  error_code text NULL,
  error_detail text NULL,
  created_at timestamptz NOT NULL DEFAULT now(),
  indexing_started_at timestamptz NULL,
  ready_at timestamptz NULL,
  activated_at timestamptz NULL,
  superseded_at timestamptz NULL,
  reviewed_at timestamptz NULL,
  reviewed_by text NULL,
  artifact_state text NOT NULL DEFAULT 'none'
    CHECK (artifact_state IN ('none', 'present', 'retention_deleting', 'retention_deleted')),
  retention_due_at timestamptz NULL,
  purged_at timestamptz NULL,
  UNIQUE (source_id, version_number),
  UNIQUE (version_id, source_id),
  FOREIGN KEY (previous_version_id, source_id)
    REFERENCES rag_source_versions(version_id, source_id)
    DEFERRABLE INITIALLY DEFERRED,
  FOREIGN KEY (base_active_version_id, source_id)
    REFERENCES rag_source_versions(version_id, source_id)
    DEFERRABLE INITIALLY DEFERRED
);

DO $$
BEGIN
  IF NOT EXISTS (
    SELECT 1 FROM pg_constraint WHERE conname = 'rag_sources_active_version_fk'
  ) THEN
    ALTER TABLE rag_sources
      ADD CONSTRAINT rag_sources_active_version_fk
      FOREIGN KEY (active_version_id, source_id)
      REFERENCES rag_source_versions(version_id, source_id)
      DEFERRABLE INITIALLY DEFERRED;
  END IF;
END $$;

CREATE UNIQUE INDEX IF NOT EXISTS rag_one_active_version_per_source
  ON rag_source_versions(source_id)
  WHERE state = 'active';

CREATE UNIQUE INDEX IF NOT EXISTS rag_one_reusable_content_version
  ON rag_source_versions(source_id, source_content_hash, processing_fingerprint, metadata_hash)
  WHERE source_content_hash IS NOT NULL
    AND state NOT IN ('failed', 'rejected', 'purged');

CREATE INDEX IF NOT EXISTS rag_source_versions_source_state_idx
  ON rag_source_versions(source_id, state);

CREATE TABLE IF NOT EXISTS rag_version_documents (
  version_id uuid NOT NULL REFERENCES rag_source_versions(version_id) ON DELETE RESTRICT,
  document_id text NOT NULL,
  document_key text NOT NULL,
  original_hash char(64) NOT NULL,
  original_hash_kind text NOT NULL CHECK (original_hash_kind IN ('bytes', 'legacy-derived')),
  content_hash char(64) NULL,
  mime_type text NOT NULL,
  title text NOT NULL,
  extraction_method text NOT NULL DEFAULT 'native',
  index_state text NOT NULL CHECK (index_state IN ('pending', 'indexing', 'ready', 'failed')),
  chunk_count integer NOT NULL DEFAULT 0 CHECK (chunk_count >= 0),
  artifact_manifest_path text NULL,
  error_detail text NULL,
  created_at timestamptz NOT NULL DEFAULT now(),
  PRIMARY KEY (version_id, document_id),
  UNIQUE (version_id, document_key)
);

CREATE INDEX IF NOT EXISTS rag_version_documents_document_idx
  ON rag_version_documents(document_id);

CREATE TABLE IF NOT EXISTS rag_ingestion_attempts (
  attempt_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  source_id text NOT NULL REFERENCES rag_sources(source_id) ON DELETE RESTRICT,
  version_id uuid NULL REFERENCES rag_source_versions(version_id) ON DELETE SET NULL,
  state text NOT NULL CHECK (state IN ('received', 'processing', 'completed', 'failed')),
  input_locator text NOT NULL,
  input_hash char(64) NULL,
  error_code text NULL,
  error_detail text NULL,
  created_at timestamptz NOT NULL DEFAULT now(),
  completed_at timestamptz NULL
);

CREATE TABLE IF NOT EXISTS rag_system_state (
  singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton),
  maintenance boolean NOT NULL DEFAULT false,
  maintenance_reason text NULL,
  updated_at timestamptz NOT NULL DEFAULT now()
);

INSERT INTO rag_system_state(singleton) VALUES (true)
ON CONFLICT (singleton) DO NOTHING;

CREATE TABLE IF NOT EXISTS rag_legacy_migration_batches (
  batch_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
  state text NOT NULL CHECK (state IN ('planned', 'running', 'verified', 'rolled_back', 'failed')),
  snapshot_ref text NOT NULL,
  report jsonb NOT NULL DEFAULT '{}',
  created_at timestamptz NOT NULL DEFAULT now(),
  completed_at timestamptz NULL
);

CREATE TABLE IF NOT EXISTS rag_legacy_migration_items (
  batch_id uuid NOT NULL REFERENCES rag_legacy_migration_batches(batch_id) ON DELETE RESTRICT,
  source_id text NOT NULL,
  version_id uuid NULL REFERENCES rag_source_versions(version_id) ON DELETE SET NULL,
  expected_point_count integer NOT NULL CHECK (expected_point_count >= 0),
  state text NOT NULL CHECK (state IN ('planned', 'migrated', 'verified', 'rolled_back', 'failed')),
  error_detail text NULL,
  PRIMARY KEY (batch_id, source_id)
);

CREATE OR REPLACE FUNCTION rag_touch_updated_at()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
  NEW.updated_at = now();
  RETURN NEW;
END;
$$;

DROP TRIGGER IF EXISTS rag_sources_touch_updated_at ON rag_sources;
CREATE TRIGGER rag_sources_touch_updated_at
BEFORE UPDATE ON rag_sources
FOR EACH ROW EXECUTE FUNCTION rag_touch_updated_at();

CREATE OR REPLACE FUNCTION rag_validate_active_source_pointer()
RETURNS trigger LANGUAGE plpgsql AS $$
DECLARE
  invalid_pointer_count integer;
  orphan_active_count integer;
BEGIN
  SELECT count(*) INTO invalid_pointer_count
  FROM rag_sources s
  LEFT JOIN rag_source_versions v
    ON v.version_id = s.active_version_id
   AND v.source_id = s.source_id
   AND v.state = 'active'
  WHERE s.active_version_id IS NOT NULL
    AND v.version_id IS NULL;

  IF invalid_pointer_count > 0 THEN
    RAISE EXCEPTION 'rag_sources.active_version_id must point to an active version in the same source';
  END IF;

  SELECT count(*) INTO orphan_active_count
  FROM rag_source_versions v
  JOIN rag_sources s ON s.source_id = v.source_id
  WHERE v.state = 'active'
    AND s.active_version_id IS DISTINCT FROM v.version_id;

  IF orphan_active_count > 0 THEN
    RAISE EXCEPTION 'every active rag_source_versions row must be the source active_version_id';
  END IF;

  RETURN NULL;
END;
$$;

DROP TRIGGER IF EXISTS rag_sources_validate_active_pointer ON rag_sources;
CREATE CONSTRAINT TRIGGER rag_sources_validate_active_pointer
AFTER INSERT OR UPDATE ON rag_sources
DEFERRABLE INITIALLY DEFERRED
FOR EACH ROW EXECUTE FUNCTION rag_validate_active_source_pointer();

DROP TRIGGER IF EXISTS rag_versions_validate_active_pointer ON rag_source_versions;
CREATE CONSTRAINT TRIGGER rag_versions_validate_active_pointer
AFTER INSERT OR UPDATE ON rag_source_versions
DEFERRABLE INITIALLY DEFERRED
FOR EACH ROW EXECUTE FUNCTION rag_validate_active_source_pointer();