open-brainy/tests/integration/vfs-root-zero-norm.test.ts

/**
 * @module tests/integration/vfs-root-zero-norm
 * @description THE ZERO-NORM ROOT CURE — a production incident traced 150+
 * darkened rows in a downstream engine's index to the VFS root's persisted
 * ALL-ZERO placeholder vector: lawful inside brainy (`cosineDistance`
 * treats a zero-norm operand as MAXIMUM distance, src/utils/distance.ts)
 * but a "false attractor" for an engine serving squared-euclidean distance,
 * which cannot tell a real all-zero vector apart from a legitimate origin
 * point. THE LAW: a zero-norm vector is not a vector — it never crosses an
 * engine boundary.
 *
 * Three legs pinned here:
 * (a) the root persists NO zeros — a brand-new store creates it with
 *     vector `[]` (the "unvectored" shape), absent from the HNSW index, and
 *     the canonical vectored-noun ledger does not count it.
 * (b) a ONE-TIME migration heals an existing (pre-fix) store: an old-shape
 *     root (a REAL all-zero vector, genuinely indexed and ledgered — the
 *     harness reproduces exactly what a pre-fix store looked like on disk)
 *     is rewritten to `[]` on the next `init()`, the ledger is decremented
 *     through the sanctioned path, and a second `init()` is a no-op.
 * (c) THE CANONICAL-WRITE NORMALIZATION (Leg A of the follow-up
 *     zero-norm/unvector-door fix): an entity added with an EXPLICIT
 *     all-zero vector (any dimension) is normalized to the "unvectored"
 *     `[]` shape BEFORE the canonical write, the ledger flag, and the index
 *     ops ever see it — the canonical write still succeeds, loudly, and the
 *     vector-index insert never happens (nothing to index). Supersedes the
 *     original "canonical keeps the zero vector, only the index refuses"
 *     shape: a downstream engine's health-report gate reads the canonical
 *     ledger directly, so leaving a zero-norm vector on the canonical side
 *     re-opened the exact false-attractor risk this whole fix closes.
 * (d) the migrated root never surfaces in `find()` results (it was already
 *     hidden behind `visibility: 'system'` — this pin holds regardless).
 */
import { describe, it, expect, afterEach, vi } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/index.js'
import { NounType } from '../../src/types/graphTypes.js'
import { prodLog } from '../../src/utils/logger.js'

const ROOT_ID = '00000000-0000-0000-0000-000000000000'

process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'

const tmpDirs: string[] = []
function mkTmp(): string {
  const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vfs-root-zero-norm-'))
  tmpDirs.push(d)
  return d
}
afterEach(() => {
  vi.restoreAllMocks()
  for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
})

function openBrain(dir: string): any {
  return new Brainy({
    requireSubtype: false,
    storage: { type: 'filesystem', path: dir },
    silent: true,
    dimensions: 384
  })
}

describe('VFS root zero-norm cure', () => {
  it('(a) a brand-new store persists the root with vector [], absent from the HNSW index, and the canonical ledger counts it unvectored', async () => {
    const dir = mkTmp()
    const brain = openBrain(dir)
    await brain.init()

    const root = await brain.get(ROOT_ID, { includeVectors: true })
    expect(root).not.toBeNull()
    expect(root.vector).toEqual([])

    const status = await brain.getIndexStatus()
    expect(status.hnswIndex.size).toBe(0)

    const ledger = await brain.storage.getCanonicalCounts()
    expect(ledger.vectors.all).toBe(0)

    await brain.close()
  })

  it('(b) an old-shape store (a real all-zero placeholder root) migrates to [] exactly once on init; the ledger is decremented through the sanctioned path; a second init is a no-op', async () => {
    const dir = mkTmp()

    // SESSION 1 — build the store, then hand-rewrite the root to the LEGACY
    // shape: a REAL all-zero 384-dim vector, genuinely inserted into the
    // vector index and genuinely counted by the vectored-noun ledger —
    // reproducing exactly what a pre-fix store's root looked like on disk
    // (the pre-fix add() always indexed + counted it). `index.addItem` is
    // called directly (bypassing AddToVectorIndexOperation's own zero-norm
    // belt, added by this same fix) precisely because the pre-fix code path
    // had no such belt — this harness must match history, not the cure.
    let brain = openBrain(dir)
    await brain.init()
    const oldVector = new Array(384).fill(0)
    await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 })
    await brain.index.addItem({ id: ROOT_ID, vector: oldVector })
    await brain.storage.noteVectorLanded(ROOT_ID)
    await brain.storage.persistCounts()
    await brain.flush()

    const ledgerBeforeMigration = await brain.storage.getCanonicalCounts()
    expect(ledgerBeforeMigration.vectors.all).toBe(1)
    await brain.close()

    // SESSION 2 — reopen: VFS init must detect the legacy shape and migrate.
    // Spy on the sanctioned migration method itself (not console output —
    // `silent: true` monkey-patches `console.log` to a no-op INSIDE init(),
    // which would silently swallow any pre-installed console spy too).
    brain = openBrain(dir)
    const migrateSpy = vi.spyOn(brain, 'unvectorNounForRootMigration')
    await brain.init()

    expect(migrateSpy).toHaveBeenCalledTimes(1)
    expect(migrateSpy).toHaveBeenCalledWith(ROOT_ID)
    await expect(migrateSpy.mock.results[0].value).resolves.toBe(true)

    const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true })
    expect(migratedRoot.vector).toEqual([])

    const ledgerAfterMigration = await brain.storage.getCanonicalCounts()
    expect(ledgerAfterMigration.vectors.all).toBe(0)

    const statusAfterMigration = await brain.getIndexStatus()
    expect(statusAfterMigration.hnswIndex.size).toBe(0)

    await brain.flush()
    await brain.close()

    // SESSION 3 — reopen again: the migration is a permanent no-op, not a
    // one-time flag that silently re-drifts or re-fires. The zero-norm
    // detection at the VFS init site never even calls the migration method
    // again — the root's vector is already `[]`.
    brain = openBrain(dir)
    const migrateSpy2 = vi.spyOn(brain, 'unvectorNounForRootMigration')
    await brain.init()

    expect(migrateSpy2).not.toHaveBeenCalled()

    const rootAfterSecondInit = await brain.get(ROOT_ID, { includeVectors: true })
    expect(rootAfterSecondInit.vector).toEqual([])

    const ledgerAfterSecondInit = await brain.storage.getCanonicalCounts()
    expect(ledgerAfterSecondInit.vectors.all).toBe(0)

    await brain.close()
  })

  it('(c) canonical-write normalization: an entity added with an explicit all-zero vector persists UNVECTORED ([]), loudly, and never reaches the vector index', async () => {
    const dir = mkTmp()
    const brain = openBrain(dir)
    await brain.init()

    const warnSpy = vi.spyOn(prodLog, 'warn')

    const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size
    const ledgerBefore = await brain.storage.getCanonicalCounts()
    const zeroVector = new Array(384).fill(0)
    const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector })

    // The canonical write succeeded — but the zero-norm vector was
    // normalized to the "unvectored" `[]` shape BEFORE it was persisted
    // (Leg A: a zero-norm vector is not a vector — it never crosses an
    // engine boundary, canonical side included).
    const entity = await brain.get(id, { includeVectors: true })
    expect(entity).not.toBeNull()
    expect(entity.vector).toEqual([])

    // Nothing to index — the vector-index size never moved, and the
    // vectored-noun ledger never counted this row.
    const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size
    expect(sizeAfter).toBe(sizeBefore)
    const ledgerAfter = await brain.storage.getCanonicalCounts()
    expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all)

    // The normalization was LOUD and named the entity.
    const loudCall = warnSpy.mock.calls.find(
      (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm')
    )
    expect(loudCall).toBeDefined()

    await brain.close()
  })

  it('(d) find() over a store whose root has been migrated never returns the root (already hidden behind visibility: system — pinned anyway)', async () => {
    const dir = mkTmp()

    // Build an old-shape store (same harness as pin (b)) and let it migrate.
    let brain = openBrain(dir)
    await brain.init()
    const oldVector = new Array(384).fill(0)
    await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 })
    await brain.index.addItem({ id: ROOT_ID, vector: oldVector })
    await brain.storage.noteVectorLanded(ROOT_ID)
    await brain.storage.persistCounts()
    await brain.add({ data: 'a document about technology', type: NounType.Document })
    await brain.flush()
    await brain.close()

    brain = openBrain(dir) // migrates on init()
    await brain.init()

    const results = await brain.find({ query: 'technology', limit: 10 })
    expect(results.some((r: any) => r.id === ROOT_ID)).toBe(false)

    // Even asking explicitly for system-tier entities must never surface the
    // root as a semantic-search HIT (it carries no vector to match against).
    const resultsIncludingSystem = await brain.find({ query: 'technology', limit: 10, includeSystem: true })
    expect(resultsIncludingSystem.some((r: any) => r.id === ROOT_ID)).toBe(false)

    await brain.close()
  })
})