open-brainy/tests/integration/index-skips-unvectored.test.ts

/**
 * @module tests/integration/index-skips-unvectored
 * @description THE UNVECTORED-ROW CURE — two integration tests
 * (`tests/lifecycle/biography.test.ts`'s Ch4/5/6 chapter and
 * `tests/integration/clear-persistence.test.ts`'s multi-cycle test) started
 * failing after a canonical-storage change made a vector-less row (the
 * class-J shape: `vector: []`, e.g. the VFS root, a deferred embed not yet
 * landed, or any other legitimately-unvectored canonical record) VISIBLE to
 * the enumeration walk `getNounsWithPagination()` for the first time — before
 * that change such rows were simply invisible to the walk. `hnswIndex.ts`'s
 * `rebuild()` never guarded against that shape: it inserted every row the
 * walk yielded into the live in-memory index, including ones with a length-0
 * vector, because `storage.getVectorIndexData()` derives its {level,
 * connections} answer straight from the noun's OWN record — it returns
 * non-null for ANY existing noun, whether or not that noun was ever actually
 * indexed via `addItem()`. A vector-less node admitted into the graph could
 * become the entry point (or occupy any graph position), and the very next
 * real-vectored `addItem()` then ran a distance calculation against it —
 * `cosineDistance` throws "Vectors must have the same dimensions" the moment
 * one operand is a length-0 array.
 *
 * THE FIX, at two layers (`src/hnsw/hnswIndex.ts`):
 * (1) FILL/REBUILD/LOAD consumers treat `vector.length === 0` as "unvectored —
 *     nothing to index" and skip the row (normal, not an error; one summary
 *     count line, never per-row spam) — `rebuild()`'s loop now checks this
 *     BEFORE ever creating a graph node, so an unvectored row can never
 *     become an index member, entry point, or dimension-setter.
 * (2) THE INDEX ITSELF refuses a length-0 vector in `addItem()` /
 *     `updateItem()` with a typed `EmptyVectorIndexError`, loudly, instead of
 *     ever pinning `dimension = 0` or storing a vector-less node — so no
 *     future fill/rebuild/load path can silently poison the index even if it
 *     forgets law (1).
 *
 * Four legs pinned here:
 * (a) `rebuild()` over a store mixing real-vectored rows and `vector: []`
 *     rows indexes ONLY the vectored ones — size === vectored count,
 *     dimension pinned to the real (non-zero) length.
 * (b) `clear()` then real adds afterward never trip a dimension mismatch —
 *     the exact `clear-persistence.test.ts` regression shape, reproduced
 *     directly against the index/storage seam this module owns.
 * (c) `index.addItem({ id, vector: [] })` throws `EmptyVectorIndexError`
 *     (and `updateItem` does too, for an existing node).
 * (d) crash -> repair: the crashed generation's entities survive, the ledger
 *     recounts honestly, and a fresh real-vectored add afterward never trips
 *     a dimension mismatch against a leftover vector-less phantom.
 */
import { describe, it, expect, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/index.js'
import { NounType } from '../../src/types/graphTypes.js'
import { EmptyVectorIndexError } from '../../src/hnsw/hnswIndex.js'
import { abandonAsCrashed, openBrain as openKillMatrixBrain, uid, vec } from '../helpers/durabilityKillMatrix.js'

const tmpDirs: string[] = []
function mkTmp(): string {
  const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-'))
  tmpDirs.push(d)
  return d
}
afterEach(() => {
  for (const d of tmpDirs.splice(0)) {
    try {
      fs.rmSync(d, { recursive: true, force: true })
    } catch {
      /* best-effort cleanup */
    }
  }
})

/** A filesystem-backed brain with explicit vectors (no embedder needed) and
 * manual persistence — mirrors `durabilityKillMatrix.ts`'s `openBrain` so
 * every write in this module is explicit and provably durable. */
function openBrain(dir: string): any {
  return new Brainy({
    requireSubtype: false,
    storage: { type: 'filesystem', path: dir },
    silent: true,
    persistence: { policy: 'manual' }
  })
}

describe('HNSW index skips unvectored rows', () => {
  it('(a) rebuild() indexes only vectored rows: size === vectored count, dimension pinned to the real length', async () => {
    const dir = mkTmp()
    let brain = openBrain(dir)
    await brain.init()

    // 5 real-vectored rows.
    const vectoredIds: string[] = []
    for (let i = 0; i < 5; i++) {
      const id = uid(`vectored-${i}`)
      await brain.add({ id, data: `real entity ${i}`, type: NounType.Document, vector: vec(i) })
      vectoredIds.push(id)
    }
    // 3 explicit unvectored rows — the class-J "vector: []" shape, a normal,
    // enumerable, countable canonical row that must never reach the index.
    const unvectoredIds: string[] = []
    for (let i = 0; i < 3; i++) {
      const id = uid(`unvectored-${i}`)
      await brain.add({ id, data: `unvectored entity ${i}`, type: NounType.Document, vector: [] })
      unvectoredIds.push(id)
    }
    await brain.flush()

    // The canonical ledger already agrees before any rebuild: nouns.all
    // counts every row (8 + the VFS root); vectors.all counts only the real
    // ones (5) — the VFS root and the 3 explicit unvectored rows are excluded.
    const ledgerBeforeReopen = await brain.storage.getCanonicalCounts()
    expect(ledgerBeforeReopen.vectors.all).toBe(5)
    expect(ledgerBeforeReopen.nouns.all).toBe(9) // 5 vectored + 3 unvectored + 1 VFS root

    await brain.close()

    // Reopen: open()'s index build IS hnswIndex.rebuild() run fresh from
    // storage — this is the exact path that used to admit unvectored rows.
    brain = openBrain(dir)
    await brain.init()

    const status = await brain.getIndexStatus()
    expect(status.hnswIndex.size, 'the rebuilt index must contain ONLY the 5 real-vectored rows').toBe(5)

    // Dimension is pinned to the REAL embedded length (384 via `vec()`), not
    // 0 — adding a wrong-length vector must be refused naming that real
    // dimension, proving no vector-less row ever set it.
    const realDimension = vec(0).length
    let mismatchMessage: string | undefined
    try {
      await brain.index.addItem({ id: uid('dimension-probe'), vector: vec(0).slice(0, realDimension - 1) })
      expect.fail('expected a dimension mismatch error')
    } catch (err) {
      mismatchMessage = (err as Error).message
    }
    expect(mismatchMessage).toContain(`expected ${realDimension}`)

    // Every unvectored row is still a normal, enumerable, readable canonical
    // record — class-J semantics survive the rebuild fix untouched.
    for (const id of unvectoredIds) {
      const entity = await brain.get(id, { includeVectors: true })
      expect(entity, `unvectored entity ${id} must remain readable`).not.toBeNull()
      expect(entity.vector).toEqual([])
    }
    // A correct-dimension add succeeds cleanly against the pinned dimension.
    const freshId = uid('post-reopen-fresh')
    await expect(brain.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(50) })).resolves.toBe(
      freshId
    )

    await brain.close()
  })

  it('(b) clear() then real adds afterward never trip a dimension mismatch (the clear-persistence regression shape)', async () => {
    const dir = mkTmp()
    let brain = openBrain(dir)
    await brain.init() // the VFS root (vector: []) is the store's only row

    await brain.clear()
    await brain.close()

    // Reopen over a store whose only surviving row is the recreated,
    // unvectored VFS root — this is exactly the shape that used to poison
    // the entry point / dimension in `clear-persistence.test.ts`.
    brain = openBrain(dir)
    await brain.init()
    expect((await brain.getIndexStatus()).hnswIndex.size).toBe(0)

    const id1 = uid('after-clear-1')
    await expect(brain.add({ id: id1, data: 'after clear 1', type: NounType.Document, vector: vec(1) })).resolves.toBe(
      id1
    )
    const id2 = uid('after-clear-2')
    await expect(brain.add({ id: id2, data: 'after clear 2', type: NounType.Document, vector: vec(2) })).resolves.toBe(
      id2
    )
    expect((await brain.getIndexStatus()).hnswIndex.size).toBe(2)

    await brain.close()
  })

  it('(c) index.addItem/updateItem refuse a length-0 vector with EmptyVectorIndexError', async () => {
    const dir = mkTmp()
    const brain = openBrain(dir)
    await brain.init()

    await expect(brain.index.addItem({ id: uid('empty-add'), vector: [] })).rejects.toThrow(EmptyVectorIndexError)

    // updateItem on an EXISTING (real-vectored) node must refuse the same way.
    const existingId = uid('existing-for-update')
    await brain.add({ id: existingId, data: 'existing', type: NounType.Document, vector: vec(9) })
    await expect(brain.index.updateItem({ id: existingId, vector: [] })).rejects.toThrow(EmptyVectorIndexError)

    // The index was never disturbed by either refused call.
    expect((await brain.getIndexStatus()).hnswIndex.size).toBe(1)

    await brain.close()
  })

  it('(d) crash -> repair: the crashed generation survives, the ledger recounts honestly, and a fresh add afterward never trips a dimension mismatch', async () => {
    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-crash-'))
    try {
      let brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' })

      // Baseline: real-vectored entities, durably flushed.
      const baselineIds: string[] = []
      for (let i = 0; i < 5; i++) {
        const id = uid(`baseline-${i}`)
        await brain.add({ id, data: `baseline entity ${i}`, type: NounType.Document, vector: vec(i) })
        baselineIds.push(id)
      }
      await brain.flush()

      // Crash window: at-ack writes that are never flushed before the crash.
      const crashedIds: string[] = []
      for (let i = 0; i < 4; i++) {
        const id = uid(`crashed-${i}`)
        await brain.add({ id, data: `crash-window entity ${i}`, type: NounType.Document, vector: vec(100 + i) })
        crashedIds.push(id)
      }
      await abandonAsCrashed(brain)

      // Reopen — logAuthority: 'adopt' replays the at-ack log for the crash window.
      brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' })
      for (const id of [...baselineIds, ...crashedIds]) {
        expect(await brain.get(id), `entity ${id} must survive the crash`).not.toBeNull()
      }

      // Repair — must not disturb any entity, and must recount the ledger honestly.
      const report = await brain.repairIndex()
      expect(report.families.length).toBeGreaterThan(0)
      for (const id of [...baselineIds, ...crashedIds]) {
        expect(await brain.get(id), `entity ${id} must survive repair`).not.toBeNull()
      }

      const ledger = await brain.storage.getCanonicalCounts()
      expect(ledger.suspect).toBe(false)
      expect(ledger.vectors.all).toBe(baselineIds.length + crashedIds.length)

      // Second life: a fresh real-vectored add must never trip a dimension
      // mismatch against a vector-less phantom left in the index — the exact
      // mechanism `clear-persistence.test.ts` and the biography lane hit.
      const secondLifeId = uid('second-life')
      await expect(
        brain.add({ id: secondLifeId, data: 'second life entity', type: NounType.Document, vector: vec(200) })
      ).resolves.toBe(secondLifeId)
      expect(await brain.get(secondLifeId)).not.toBeNull()

      await brain.close()
    } finally {
      fs.rmSync(dir, { recursive: true, force: true })
    }
  })
})