/** * @module tests/integration/deferred-embedding * @description MT5 — THE DEFERRED-EMBEDDING CONTRACT (A3 of the service-class * pair, BRAINY-PROD-LATENCY-TRIAD). The production disease: a VFS file write * ran a neural network synchronously while the caller waited (5.6s p50 per * small file). The contract pinned here: * * 1. ACK AT DURABILITY: a deferred write never calls the embedder on the * caller's path — the row is id/metadata-findable immediately, with a * durable pending marker and an honest `pendingEmbeds` gauge. * 2. EVENTUAL VECTOR INDEX: `awaitPendingEmbeds()` is the barrier — after * it, the vector is real, indexed, and the marker is reaped. * 3. STALE-BEATS-ABSENT on deferred updates: the OLD vector keeps serving * until the atomic swap (the flicker law, never a dark window). * 4. CRASH-SAFE: markers survive a session that dies mid-defer; the next * open recovers and lands the vector. A crash DELAYS a vector, never * loses one. * 5. TYPED REFUSALS: deferEmbedding + vector, and deferEmbedding without * data, are caller bugs that refuse with the fix in the message. */ import { describe, it, expect, afterEach, vi } from 'vitest' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Brainy } from '../../src/index.js' import { NounType } from '../../src/types/graphTypes.js' const dirs: string[] = [] const brains: Brainy[] = [] async function memBrain(): Promise { const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) await b.init() brains.push(b) return b } afterEach(async () => { vi.restoreAllMocks() for (const b of brains.splice(0)) await b.close().catch(() => {}) for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) }) describe('MT5 — deferred embedding', () => { it('ACK LAW: add({deferEmbedding}) never embeds on the caller path; row findable immediately; barrier lands the vector and reaps the marker', async () => { const brain = await memBrain() const embedSpy = vi.spyOn(brain, 'embed') const id = await brain.add({ data: 'deferred content', type: NounType.Document, deferEmbedding: true, metadata: { tag: 'deferred' } }) // The caller's path never ran the embedder. expect(embedSpy, 'no embed on the ack path').not.toHaveBeenCalled() // Immediately findable by metadata; vector is the stub; gauge honest. const found = await brain.find({ where: { tag: 'deferred' }, limit: 5 }) expect(found.map((r) => r.id)).toContain(id) expect((await brain.getIndexStatus()).pendingEmbeds).toBeGreaterThanOrEqual(1) // The barrier: vector lands, marker reaped, index carries the row. await brain.awaitPendingEmbeds() expect(embedSpy).toHaveBeenCalled() const after = await brain.get(id, { includeVectors: true }) expect((after!.vector as number[]).length, 'real vector after the barrier').toBeGreaterThan(0) expect(brain.pendingEmbedCount()).toBe(0) expect((await brain.getIndexStatus()).pendingEmbeds).toBe(0) }) it('STALE-BEATS-ABSENT: a deferred update serves the OLD vector until the atomic swap; data reads NEW immediately', async () => { const brain = await memBrain() const id = await brain.add({ data: 'original content', type: NounType.Document, metadata: {} }) const before = await brain.get(id, { includeVectors: true }) const oldVector = [...(before!.vector as number[])] expect(oldVector.length).toBeGreaterThan(0) await brain.update({ id, data: 'completely different content', deferEmbedding: true }) // Data is new IMMEDIATELY; the vector is still the old one (present, // never absent) until the worker swaps it. const mid = await brain.get(id, { includeVectors: true }) expect(mid!.data).toBe('completely different content') expect(mid!.vector as number[], 'old vector keeps serving').toEqual(oldVector) await brain.awaitPendingEmbeds() const after = await brain.get(id, { includeVectors: true }) expect((after!.vector as number[]).length).toBeGreaterThan(0) expect(after!.vector as number[], 'vector swapped after the barrier').not.toEqual(oldVector) }) it('CRASH-SAFE: a session dying mid-defer leaves the durable marker; the next open recovers and lands the vector', async () => { const dir = mkdtempSync(join(tmpdir(), 'brainy-defer-crash-')) dirs.push(dir) // Session 1: the embedder hangs → the worker can never complete; close() // does not wait for it (crash-equivalent for the embed leg). let brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) await brain.init() brains.push(brain) vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) const id = await brain.add({ data: 'survives the crash', type: NounType.Document, deferEmbedding: true, metadata: { k: 1 } }) expect(brain.pendingEmbedCount()).toBe(1) await brain.close() brains.pop() vi.restoreAllMocks() // Session 2: recovery lists the marker and resumes in the background. brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) await brain.init() brains.push(brain) expect(brain.pendingEmbedCount(), 'marker recovered at open').toBe(1) await brain.awaitPendingEmbeds() const after = await brain.get(id, { includeVectors: true }) expect((after!.vector as number[]).length, 'the delayed vector landed').toBeGreaterThan(0) expect(brain.pendingEmbedCount()).toBe(0) }, 120000) it('VFS ACK LAW: writeFile resolves even when the embedder HANGS forever — the ack never depends on a neural net', async () => { const brain = await memBrain() // The strongest form of the pin: an embedder that never resolves. If any // part of the writeFile ack path awaited an embed, this test would hang. // (The background worker legitimately picks the deferred embeds up later // — it may even interleave on the event loop during writeFile's other // awaits — but the CALLER'S promise must never depend on it.) const hang = vi .spyOn(brain, 'embed') .mockImplementation(() => new Promise(() => {})) await brain.vfs.writeFile('/notes/today.md', '# The day\nA deferred capture.') // Acked with the embedder hung: content + metadata fully readable. const content = await brain.vfs.readFile('/notes/today.md') expect(content.toString()).toContain('A deferred capture.') expect(brain.pendingEmbedCount()).toBeGreaterThanOrEqual(1) // Un-hang, abandon the poisoned in-flight run (its embed promise never // resolves — production is covered by the worker's 60s hang guard; the // test takes the white-box shortcut for speed), drain, verify. hang.mockRestore() ;(brain as unknown as { _embedWorkerFlight: Promise | null })._embedWorkerFlight = null await brain.awaitPendingEmbeds() expect(brain.pendingEmbedCount()).toBe(0) }) it('TYPED REFUSALS: defer+vector and defer-without-data both refuse with the fix', async () => { const brain = await memBrain() await expect( brain.add({ data: 'x', vector: new Array(384).fill(0.1), type: NounType.Document, deferEmbedding: true, metadata: {} }) ).rejects.toThrow(/deferEmbedding cannot be combined/) const id = await brain.add({ data: 'y', type: NounType.Document, metadata: {} }) await expect( brain.update({ id, deferEmbedding: true, metadata: { z: 1 } }) ).rejects.toThrow(/requires new 'data'/) }) })