/** * @module tests/integration/vfs-root-zero-norm * @description THE ZERO-NORM ROOT CURE — a production incident traced 150+ * darkened rows in a downstream engine's index to the VFS root's persisted * ALL-ZERO placeholder vector: lawful inside brainy (`cosineDistance` * treats a zero-norm operand as MAXIMUM distance, src/utils/distance.ts) * but a "false attractor" for an engine serving squared-euclidean distance, * which cannot tell a real all-zero vector apart from a legitimate origin * point. THE LAW: a zero-norm vector is not a vector — it never crosses an * engine boundary. * * Three legs pinned here: * (a) the root persists NO zeros — a brand-new store creates it with * vector `[]` (the "unvectored" shape), absent from the HNSW index, and * the canonical vectored-noun ledger does not count it. * (b) a ONE-TIME migration heals an existing (pre-fix) store: an old-shape * root (a REAL all-zero vector, genuinely indexed and ledgered — the * harness reproduces exactly what a pre-fix store looked like on disk) * is rewritten to `[]` on the next `init()`, the ledger is decremented * through the sanctioned path, and a second `init()` is a no-op. * (c) THE CANONICAL-WRITE NORMALIZATION (Leg A of the follow-up * zero-norm/unvector-door fix): an entity added with an EXPLICIT * all-zero vector (any dimension) is normalized to the "unvectored" * `[]` shape BEFORE the canonical write, the ledger flag, and the index * ops ever see it — the canonical write still succeeds, loudly, and the * vector-index insert never happens (nothing to index). Supersedes the * original "canonical keeps the zero vector, only the index refuses" * shape: a downstream engine's health-report gate reads the canonical * ledger directly, so leaving a zero-norm vector on the canonical side * re-opened the exact false-attractor risk this whole fix closes. * (d) the migrated root never surfaces in `find()` results (it was already * hidden behind `visibility: 'system'` — this pin holds regardless). */ import { describe, it, expect, afterEach, vi } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' import { Brainy } from '../../src/index.js' import { NounType } from '../../src/types/graphTypes.js' import { prodLog } from '../../src/utils/logger.js' const ROOT_ID = '00000000-0000-0000-0000-000000000000' process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' const tmpDirs: string[] = [] function mkTmp(): string { const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vfs-root-zero-norm-')) tmpDirs.push(d) return d } afterEach(() => { vi.restoreAllMocks() for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) }) function openBrain(dir: string): any { return new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) } describe('VFS root zero-norm cure', () => { it('(a) a brand-new store persists the root with vector [], absent from the HNSW index, and the canonical ledger counts it unvectored', async () => { const dir = mkTmp() const brain = openBrain(dir) await brain.init() const root = await brain.get(ROOT_ID, { includeVectors: true }) expect(root).not.toBeNull() expect(root.vector).toEqual([]) const status = await brain.getIndexStatus() expect(status.hnswIndex.size).toBe(0) const ledger = await brain.storage.getCanonicalCounts() expect(ledger.vectors.all).toBe(0) await brain.close() }) it('(b) an old-shape store (a real all-zero placeholder root) migrates to [] exactly once on init; the ledger is decremented through the sanctioned path; a second init is a no-op', async () => { const dir = mkTmp() // SESSION 1 — build the store, then hand-rewrite the root to the LEGACY // shape: a REAL all-zero 384-dim vector, genuinely inserted into the // vector index and genuinely counted by the vectored-noun ledger — // reproducing exactly what a pre-fix store's root looked like on disk // (the pre-fix add() always indexed + counted it). `index.addItem` is // called directly (bypassing AddToVectorIndexOperation's own zero-norm // belt, added by this same fix) precisely because the pre-fix code path // had no such belt — this harness must match history, not the cure. let brain = openBrain(dir) await brain.init() const oldVector = new Array(384).fill(0) await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) await brain.storage.noteVectorLanded(ROOT_ID) await brain.storage.persistCounts() await brain.flush() const ledgerBeforeMigration = await brain.storage.getCanonicalCounts() expect(ledgerBeforeMigration.vectors.all).toBe(1) await brain.close() // SESSION 2 — reopen: VFS init must detect the legacy shape and migrate. // Spy on the sanctioned migration method itself (not console output — // `silent: true` monkey-patches `console.log` to a no-op INSIDE init(), // which would silently swallow any pre-installed console spy too). brain = openBrain(dir) const migrateSpy = vi.spyOn(brain, 'unvectorNounForRootMigration') await brain.init() expect(migrateSpy).toHaveBeenCalledTimes(1) expect(migrateSpy).toHaveBeenCalledWith(ROOT_ID) await expect(migrateSpy.mock.results[0].value).resolves.toBe(true) const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) expect(migratedRoot.vector).toEqual([]) const ledgerAfterMigration = await brain.storage.getCanonicalCounts() expect(ledgerAfterMigration.vectors.all).toBe(0) const statusAfterMigration = await brain.getIndexStatus() expect(statusAfterMigration.hnswIndex.size).toBe(0) await brain.flush() await brain.close() // SESSION 3 — reopen again: the migration is a permanent no-op, not a // one-time flag that silently re-drifts or re-fires. The zero-norm // detection at the VFS init site never even calls the migration method // again — the root's vector is already `[]`. brain = openBrain(dir) const migrateSpy2 = vi.spyOn(brain, 'unvectorNounForRootMigration') await brain.init() expect(migrateSpy2).not.toHaveBeenCalled() const rootAfterSecondInit = await brain.get(ROOT_ID, { includeVectors: true }) expect(rootAfterSecondInit.vector).toEqual([]) const ledgerAfterSecondInit = await brain.storage.getCanonicalCounts() expect(ledgerAfterSecondInit.vectors.all).toBe(0) await brain.close() }) it('(c) canonical-write normalization: an entity added with an explicit all-zero vector persists UNVECTORED ([]), loudly, and never reaches the vector index', async () => { const dir = mkTmp() const brain = openBrain(dir) await brain.init() const warnSpy = vi.spyOn(prodLog, 'warn') const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size const ledgerBefore = await brain.storage.getCanonicalCounts() const zeroVector = new Array(384).fill(0) const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) // The canonical write succeeded — but the zero-norm vector was // normalized to the "unvectored" `[]` shape BEFORE it was persisted // (Leg A: a zero-norm vector is not a vector — it never crosses an // engine boundary, canonical side included). const entity = await brain.get(id, { includeVectors: true }) expect(entity).not.toBeNull() expect(entity.vector).toEqual([]) // Nothing to index — the vector-index size never moved, and the // vectored-noun ledger never counted this row. const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size expect(sizeAfter).toBe(sizeBefore) const ledgerAfter = await brain.storage.getCanonicalCounts() expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) // The normalization was LOUD and named the entity. const loudCall = warnSpy.mock.calls.find( (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') ) expect(loudCall).toBeDefined() await brain.close() }) it('(d) find() over a store whose root has been migrated never returns the root (already hidden behind visibility: system — pinned anyway)', async () => { const dir = mkTmp() // Build an old-shape store (same harness as pin (b)) and let it migrate. let brain = openBrain(dir) await brain.init() const oldVector = new Array(384).fill(0) await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) await brain.storage.noteVectorLanded(ROOT_ID) await brain.storage.persistCounts() await brain.add({ data: 'a document about technology', type: NounType.Document }) await brain.flush() await brain.close() brain = openBrain(dir) // migrates on init() await brain.init() const results = await brain.find({ query: 'technology', limit: 10 }) expect(results.some((r: any) => r.id === ROOT_ID)).toBe(false) // Even asking explicitly for system-tier entities must never surface the // root as a semantic-search HIT (it carries no vector to match against). const resultsIncludingSystem = await brain.find({ query: 'technology', limit: 10, includeSystem: true }) expect(resultsIncludingSystem.some((r: any) => r.id === ROOT_ID)).toBe(false) await brain.close() }) })