open-brainy/tests/integration/zero-norm-unvector-door.test.ts

/**
 * @module tests/integration/zero-norm-unvector-door
 * @description THE SEAM LAW, GENERALIZED: "a zero-norm vector is not a
 * vector — it never crosses an engine boundary." `tests/integration/
 * vfs-root-zero-norm.test.ts` pins the VFS-root-specific cure; this file
 * pins the follow-up that generalizes it to every write path plus the
 * sanctioned door for shedding a vector on purpose.
 *
 * Four legs pinned here:
 * (A) THE CANONICAL WRITE NORMALIZES ZERO-NORM TO `[]` — `add()` (single and
 *     `transact()`) persists an explicit real all-zero vector as the
 *     "unvectored" `[]` shape, loudly, before the ledger flag/dimension
 *     pin/index ops ever see it. The canonical write still succeeds.
 * (B) THE LEGACY DERIVATION IS ZERO-NORM-AWARE — a lost/corrupted
 *     `counts.json`'s one-time re-derivation walk excludes a persisted
 *     zero-norm row from the vectored-noun scalar, matching the live
 *     ledger's definition of "vectored".
 * (C) THE LEGACY VFS ROOT MIGRATES AT OPEN, BEFORE THE GATE, IN O(1) — a
 *     store whose ONLY vectored row is a legacy all-zero VFS root opens
 *     clean (no `VectorIndexNotReadyError`), via one fixed-path read, never
 *     a listing.
 * (D) THE UNVECTOR DOOR — `update({ id, vector: [] })` (and the same op
 *     inside `transact()`) is the sanctioned, idempotent way to shed a
 *     vector on purpose: ledger decrement exactly once, index removal, no
 *     re-embed, and a pending deferred-embed marker is cleared rather than
 *     left to re-vectorize the row later.
 */
import { describe, it, expect, afterEach, vi } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/index.js'
import { NounType } from '../../src/types/graphTypes.js'
import { prodLog } from '../../src/utils/logger.js'
import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js'
import { BaseStorage } from '../../src/storage/baseStorage.js'

const ROOT_ID = '00000000-0000-0000-0000-000000000000'

process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'

const tmpDirs: string[] = []
function mkTmp(): string {
  const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-zero-norm-unvector-'))
  tmpDirs.push(d)
  return d
}
afterEach(() => {
  vi.restoreAllMocks()
  for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
})

function openBrain(dir: string): any {
  return new Brainy({
    requireSubtype: false,
    storage: { type: 'filesystem', path: dir },
    silent: true,
    dimensions: 384
  })
}

const countsPath = (root: string) => path.join(root, '_system', 'counts.json')

describe('zero-norm canonical write + the sanctioned unvector door', () => {
  it('(A1) add() with an explicit all-zero vector persists [], warns loudly, never indexes, and the ledger is unchanged', async () => {
    const dir = mkTmp()
    const brain = openBrain(dir)
    await brain.init()

    const ledgerBefore = await brain.storage.getCanonicalCounts()
    const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size
    const warnSpy = vi.spyOn(prodLog, 'warn')

    const zeroVector = new Array(384).fill(0)
    const id = await brain.add({ data: 'zero-norm add', type: NounType.Document, vector: zeroVector })

    const entity = await brain.get(id, { includeVectors: true })
    expect(entity).not.toBeNull()
    expect(entity.vector).toEqual([])

    const ledgerAfter = await brain.storage.getCanonicalCounts()
    expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all)

    const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size
    expect(sizeAfter).toBe(sizeBefore)

    const loud = warnSpy.mock.calls.find(
      (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm')
    )
    expect(loud).toBeDefined()

    await brain.close()
  })

  it('(A2) transact() add with an explicit all-zero vector — the same canonical normalization', async () => {
    const dir = mkTmp()
    const brain = openBrain(dir)
    await brain.init()

    const ledgerBefore = await brain.storage.getCanonicalCounts()
    const warnSpy = vi.spyOn(prodLog, 'warn')
    const zeroVector = new Array(384).fill(0)
    const id = 'aaaaaaaa-0000-4000-8000-000000000001'

    await brain.transact([
      { op: 'add', id, type: NounType.Document, data: 'zero-norm transact add', vector: zeroVector }
    ])

    const entity = await brain.get(id, { includeVectors: true })
    expect(entity).not.toBeNull()
    expect(entity.vector).toEqual([])

    const ledgerAfter = await brain.storage.getCanonicalCounts()
    expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all)

    const loud = warnSpy.mock.calls.find(
      (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm')
    )
    expect(loud).toBeDefined()

    await brain.close()
  })

  it('(B) the legacy counts.json derivation excludes a persisted zero-norm row from the vectored-noun scalar', async () => {
    const dir = mkTmp()
    let brain = openBrain(dir)
    await brain.init()

    // The VFS root alone (unvectored — []) — the floor.
    const baseline = (await brain.storage.getCanonicalCounts()).vectors.all

    const realId = await brain.add({ data: 'a real vectored document', type: NounType.Document })

    // Plant the legacy all-zero shape BY HAND: a genuine identity record
    // (via add(), so it has real metadata) whose vector leg is then
    // overwritten directly through the raw storage primitive — bypassing
    // Leg A's canonical-write normalization entirely (brain.storage.saveNoun
    // is not Brainy.add()/update()'s normalized path) — reproducing exactly
    // what a pre-fix store could have persisted on disk.
    const zeroId = await brain.add({ data: 'a legacy zero-norm document', type: NounType.Document })
    const zeroVector = new Array(384).fill(0)
    await brain.storage.saveNoun({ id: zeroId, vector: zeroVector, connections: new Map(), level: 0 })

    await brain.flush()
    await brain.close()

    // Remove counts.json so the next open re-derives from scratch (the
    // one-time legacy/lost-file derivation path — Leg B).
    fs.rmSync(countsPath(dir), { force: true })

    brain = openBrain(dir)
    await brain.init()
    const ledger = await brain.storage.getCanonicalCounts()
    // Only realId counts; zeroId's persisted all-zero vector does not.
    expect(ledger.vectors.all).toBe(baseline + 1)

    await brain.close()
  })

  it('(C) a legacy all-zero VFS root as the ONLY vectored row: open succeeds with no not-ready error, via an O(1) fixed-path read (no entities-tree readdir), and the ledger is 0 after open', async () => {
    const dir = mkTmp()

    // SESSION 1 — build the legacy shape: the root is a REAL all-zero
    // 384-dim vector, genuinely indexed and genuinely ledgered — exactly
    // what a pre-fix store's root looked like on disk (see
    // vfs-root-zero-norm.test.ts pin (b) for the identical harness).
    // `index.addItem` is called directly (bypassing the transactional
    // zero-norm belt) because the pre-fix code path had no such belt — this
    // harness must match history, not the cure. No other entity is added,
    // so the root is the store's ONLY vectored row.
    let brain = openBrain(dir)
    await brain.init()
    const oldVector = new Array(384).fill(0)
    await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 })
    await brain.index.addItem({ id: ROOT_ID, vector: oldVector })
    await brain.storage.noteVectorLanded(ROOT_ID)
    await brain.storage.persistCounts()
    await brain.flush()
    expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(1)
    await brain.close()

    // SESSION 2 — reopen with a FAKE native vector provider that claims
    // `serving: true` at `size()===0` (the exact shape a downstream
    // engine's own health report can legitimately carry — same technique as
    // tests/integration/vector-leg-open-build.test.ts). This is the ONLY
    // codepath where the vector-leg open gate's FAIL-TYPED throw
    // (VectorIndexNotReadyError) can fire; the built-in JS engine alone
    // never reaches it (the size-heuristic branch just rebuilds instead) —
    // so this is the faithful reproduction of the incident Leg C closes.
    const readdirCalls: string[] = []
    const originalReaddir = fs.promises.readdir.bind(fs.promises)
    vi.spyOn(fs.promises, 'readdir').mockImplementation(((...args: any[]) => {
      readdirCalls.push(String(args[0]))
      return (originalReaddir as any)(...args)
    }) as any)

    // Spy at the PROTOTYPE level (BaseStorage.getNoun) — the new brain's
    // storage instance does not exist until init() runs, so an
    // instance-level spy cannot be installed beforehand. Records the
    // readdir-call delta across the FIRST call made with the root id —
    // Leg C's own fixed-path read — proving it needs no directory listing.
    let readdirDeltaDuringRootRead: number | null = null
    const originalGetNoun = BaseStorage.prototype.getNoun
    vi.spyOn(BaseStorage.prototype, 'getNoun').mockImplementation(async function (
      this: unknown,
      id: string
    ) {
      const before = readdirCalls.length
      const result = await originalGetNoun.call(this as BaseStorage, id)
      if (id === ROOT_ID && readdirDeltaDuringRootRead === null) {
        readdirDeltaDuringRootRead = readdirCalls.length - before
      }
      return result
    })

    brain = openBrain(dir)
    brain.use({
      name: 'fake-native-vector-unledgered-coverage',
      activate: async (ctx: any) => {
        ctx.registerProvider('vector', (config: any, distance: any, options: any) => {
          const real = new JsHnswVectorIndex(config, distance, options)
          let rebuilt = false
          const originalRebuild = real.rebuild.bind(real)
          ;(real as any).rebuild = async (...args: any[]) => {
            const r = await originalRebuild(...args)
            rebuilt = true
            return r
          }
          const originalSize = real.size.bind(real)
          ;(real as any).size = () => (rebuilt ? originalSize() : 0)
          ;(real as any).healthReport = () => ({
            provider: 'vector',
            healthy: true,
            serving: true,
            invariants: [],
            checkedAt: Date.now(),
            durationMs: 0,
            generation: 1,
            unledgered: ['vector-coverage']
          })
          return real
        })
        return true
      }
    })

    // Must NOT throw VectorIndexNotReadyError (or anything else) — a
    // near-empty store whose only vectored row is the zero-norm root must
    // never go dark.
    await brain.init()

    const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true })
    expect(migratedRoot.vector).toEqual([])

    const ledgerAfter = await brain.storage.getCanonicalCounts()
    expect(ledgerAfter.vectors.all).toBe(0)

    expect(readdirDeltaDuringRootRead).toBe(0)

    await brain.close()
  })

  describe('the sanctioned unvector door', () => {
    it('(D1) update({ id, vector: [] }) unvectors a real vectored row — canonical [], removed from the index, ledger decremented by exactly 1, no embed call', async () => {
      const dir = mkTmp()
      const brain = openBrain(dir)
      await brain.init()

      const id = await brain.add({ data: 'a real document', type: NounType.Document })
      await brain.flush()

      const ledgerBefore = await brain.storage.getCanonicalCounts()
      const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size

      const embedSpy = vi.spyOn(brain, 'embed')
      await brain.update({ id, vector: [] })
      expect(embedSpy).not.toHaveBeenCalled()

      const entity = await brain.get(id, { includeVectors: true })
      expect(entity.vector).toEqual([])

      const ledgerAfter = await brain.storage.getCanonicalCounts()
      expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1)

      const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size
      expect(sizeAfter).toBe(sizeBefore - 1)

      await brain.close()
    })

    it('(D2) idempotent: a second update({ id, vector: [] }) on an already-unvectored row is a true no-op — no error, no further decrement', async () => {
      const dir = mkTmp()
      const brain = openBrain(dir)
      await brain.init()

      const id = await brain.add({ data: 'a real document', type: NounType.Document })
      await brain.flush()

      await brain.update({ id, vector: [] })
      const ledgerAfterFirst = await brain.storage.getCanonicalCounts()

      await brain.update({ id, vector: [] })
      const ledgerAfterSecond = await brain.storage.getCanonicalCounts()
      expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfterFirst.vectors.all)

      const entity = await brain.get(id, { includeVectors: true })
      expect(entity.vector).toEqual([])

      await brain.close()
    })

    it('(D3) a PENDING deferred-embed row: the unvector door clears the marker; awaitPendingEmbeds() then leaves it unvectored', async () => {
      const dir = mkTmp()
      const brain = openBrain(dir)
      await brain.init()

      // Prevent the background worker from ever actually running — it is
      // fire-and-forget from add(), and a real run would race this test's
      // own assertions (see tests/integration/vector-leg-open-build.test.ts
      // for the same concern). This isolates exactly the marker-clearing
      // behavior under test.
      vi.spyOn(brain as any, 'kickEmbedWorker').mockImplementation(() => {})

      const id = await brain.add({
        data: 'deferred content, never embedded',
        type: NounType.Document,
        deferEmbedding: true
      })
      expect(brain.pendingEmbedCount()).toBe(1)

      const warnSpy = vi.spyOn(prodLog, 'warn')
      await brain.update({ id, vector: [] })

      expect(brain.pendingEmbedCount()).toBe(0)
      const clearedWarn = warnSpy.mock.calls.find(
        (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('pending')
      )
      expect(clearedWarn).toBeDefined()

      // The barrier must not hang and must not re-vectorize the row — the
      // worker (still mocked to a no-op) never runs again.
      await brain.awaitPendingEmbeds()

      const entity = await brain.get(id, { includeVectors: true })
      expect(entity.vector).toEqual([])

      await brain.close()
    })

    it('(D4) update({ vector: [], deferEmbedding: true }) is a typed refusal — the unvector door cannot be paired with a deferred embed', async () => {
      const dir = mkTmp()
      const brain = openBrain(dir)
      await brain.init()

      const id = await brain.add({ data: 'a real document', type: NounType.Document })
      const before = await brain.get(id, { includeVectors: true })

      await expect(
        brain.update({ id, vector: [], deferEmbedding: true })
      ).rejects.toThrow(/unvector door/i)

      // Refused before any write — the row is untouched.
      const after = await brain.get(id, { includeVectors: true })
      expect(after.vector).toEqual(before.vector)

      await brain.close()
    })

    it('(D5) the transact() twin of the unvector door decrements the ledger exactly once, and is idempotent on a second call', async () => {
      const dir = mkTmp()
      const brain = openBrain(dir)
      await brain.init()

      const id = await brain.add({ data: 'a real document for transact unvector', type: NounType.Document })
      await brain.flush()

      const ledgerBefore = await brain.storage.getCanonicalCounts()
      const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size

      await brain.transact([{ op: 'update', id, vector: [] }])

      const entity = await brain.get(id, { includeVectors: true })
      expect(entity.vector).toEqual([])

      const ledgerAfter = await brain.storage.getCanonicalCounts()
      expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1)

      const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size
      expect(sizeAfter).toBe(sizeBefore - 1)

      // Idempotent through transact() too.
      await brain.transact([{ op: 'update', id, vector: [] }])
      const ledgerAfterSecond = await brain.storage.getCanonicalCounts()
      expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfter.vectors.all)

      await brain.close()
    })
  })
})