/** * HNSW deferred-flush durability tests (Finding 6 — spine-plan Part B, * "blind catch" audit). * * `JsHnswVectorIndex.flush()` used to swallow a per-node * `persistNodeConnections` failure (and a system-record `saveHNSWSystem` * failure) behind `console.error`, then unconditionally clear the dirty set * and report the pre-flush node count as "success". A transient write fault * therefore silently dropped that node's connections from durable storage * forever, with `flush()` having lied about it. These tests pin the fix: * a node (or the system record) that fails to persist stays in the * dirty/retry set, and `flush()` throws {@link HnswFlushError} instead of * returning a count. The immediate-mode first-noun `saveHNSWSystem` swallow * (while `addItem()` still returned the id) is covered too. */ import { describe, it, expect, vi } from 'vitest' import { v4 as uuidv4 } from 'uuid' import { JsHnswVectorIndex, HnswFlushError } from '../../../src/hnsw/hnswIndex.js' import { euclideanDistance } from '../../../src/utils/index.js' import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' // Helper: generate a random vector of given dimension (mirrors lazy-vectors.test.ts). function randomVector(dim: number): number[] { return Array.from({ length: dim }, () => Math.random() * 2 - 1) } describe('HNSW deferred-flush durability (Finding 6)', () => { const dim = 8 it('a clean flush persists every dirty node and clears the dirty set', async () => { const storage = new MemoryStorage() const index = new JsHnswVectorIndex( { M: 4, efConstruction: 50, efSearch: 20 }, euclideanDistance, { useParallelization: false, storage, persistMode: 'deferred' } ) for (let i = 0; i < 5; i++) { await index.addItem({ id: uuidv4(), vector: randomVector(dim) }) } // Sanity: inserting several items in deferred mode does dirty something. expect((index as any).dirtyNodes.size).toBeGreaterThan(0) const flushed = await index.flush() expect(typeof flushed).toBe('number') expect((index as any).dirtyNodes.size).toBe(0) expect((index as any).dirtySystem).toBe(false) }) it('retains a node whose connections failed to persist and surfaces HnswFlushError instead of reporting success', async () => { const storage = new MemoryStorage() const index = new JsHnswVectorIndex( { M: 4, efConstruction: 50, efSearch: 20 }, euclideanDistance, { useParallelization: false, storage, persistMode: 'deferred' } ) const ids: string[] = [] for (let i = 0; i < 5; i++) { const id = uuidv4() ids.push(id) await index.addItem({ id, vector: randomVector(dim) }) } // The very first node is guaranteed to become a neighbor of the second // insert (it is the only existing node in the graph at that point), so it // is deterministically present in the dirty set before any fault. const failId = ids[0] const dirtyBefore = (index as any).dirtyNodes as Set expect(dirtyBefore.has(failId)).toBe(true) const originalSave = storage.saveVectorIndexData.bind(storage) const spy = vi .spyOn(storage, 'saveVectorIndexData') .mockImplementation(async (nounId, hnswData) => { if (nounId === failId) { throw Object.assign(new Error('simulated write fault'), { code: 'EIO' }) } return originalSave(nounId, hnswData) }) await expect(index.flush()).rejects.toBeInstanceOf(HnswFlushError) const dirtyAfter = (index as any).dirtyNodes as Set // The failed node stays dirty for the next retry ... expect(dirtyAfter.has(failId)).toBe(true) // ... and every node that DID persist successfully leaves the dirty set — // the failure of one node must not re-dirty (or fail to clear) the rest. expect(dirtyAfter.size).toBe(1) // Recovery: once the fault clears, the retained node persists and the // flush reports success again (the intended retry path). spy.mockRestore() const flushed = await index.flush() expect(typeof flushed).toBe('number') expect((index as any).dirtyNodes.size).toBe(0) }) it('surfaces a system-record persist failure as HnswFlushError and keeps dirtySystem set for retry', async () => { const storage = new MemoryStorage() const index = new JsHnswVectorIndex( { M: 4, efConstruction: 50, efSearch: 20 }, euclideanDistance, { useParallelization: false, storage, persistMode: 'deferred' } ) // First noun in deferred mode marks dirtySystem (entryPoint/maxLevel). await index.addItem({ id: uuidv4(), vector: randomVector(dim) }) expect((index as any).dirtySystem).toBe(true) const spy = vi .spyOn(storage, 'saveHNSWSystem') .mockRejectedValue( Object.assign(new Error('simulated system write fault'), { code: 'EIO' }) ) let caught: unknown try { await index.flush() } catch (error) { caught = error } expect(caught).toBeInstanceOf(HnswFlushError) expect((caught as HnswFlushError).systemFailed).toBe(true) // The system record must stay dirty — a lost entry point/maxLevel update // must be retried, not silently dropped. expect((index as any).dirtySystem).toBe(true) spy.mockRestore() const flushed = await index.flush() expect(typeof flushed).toBe('number') expect((index as any).dirtySystem).toBe(false) }) it('surfaces the immediate-mode first-noun system persist failure via a rejecting addItem()', async () => { const storage = new MemoryStorage() const index = new JsHnswVectorIndex( { M: 4, efConstruction: 50, efSearch: 20 }, euclideanDistance, { useParallelization: false, storage } // default persistMode: 'immediate' ) vi.spyOn(storage, 'saveHNSWSystem').mockRejectedValue( Object.assign(new Error('simulated system write fault'), { code: 'EIO' }) ) // Previously this swallowed the error (console.error) and addItem() // still resolved with the id — stranding a brand-new index whose root // (entry point) was never actually persisted. Now it must reject. await expect( index.addItem({ id: uuidv4(), vector: randomVector(dim) }) ).rejects.toThrow('simulated system write fault') }) })