feat(embedding): MT5 — deferred embedding with durable markers; write acks never wait on a neural net

A3 of the service-class pair (BRAINY-PROD-LATENCY-TRIAD): a VFS file write
ran the embedder synchronously while the caller waited — 5.6s p50 / 21.4s
p95 per small file on a production deployment, the dominant stage of every
capture write.

- add()/update() gain deferEmbedding: the write acks at durability (data +
  metadata persisted, a DURABLE pending marker under
  _system/pending_embeds/<id> written BEFORE the commit — orphan-safe
  direction); the single-flight background worker embeds the CURRENT data
  and swaps the vector in ATOMICALLY (ReplaceInVectorIndex — the row is
  never absent from search; a deferred UPDATE keeps serving the OLD vector,
  stale-beats-absent per the flicker law). Typed refusals: defer+vector,
  defer-without-data.
- CRASH-SAFE: markers are recovered at open by a BOUNDED prefix listing
  (never a store walk) and the worker resumes in the background — a crash
  can delay a vector, never lose one. A wedged embedder trips a LOUD 60s
  hang guard and the worker moves on (marker retained for retry).
- The honest gauges: getIndexStatus().pendingEmbeds + pendingEmbedCount();
  awaitPendingEmbeds() is the eventual-vector-index BARRIER for callers
  and tests that need searchability before proceeding.
- VFS adopts it everywhere a write path could wait on the embedder:
  writeFile (both branches) and directory creation. Pinned in the
  strongest form: writeFile resolves while the embedder HANGS FOREVER.

Pins: deferred-embedding 5/5 (ack law · stale-beats-absent · crash
recovery across sessions · VFS hung-embedder ack · typed refusals).
Gates: unit 1928/1928 · integration 765 · conformance 27/27.

tests/integration/deferred-embedding.test.ts

@@ -0,0 +1,171 @@
+/**
+ * @module tests/integration/deferred-embedding
+ * @description MT5 — THE DEFERRED-EMBEDDING CONTRACT (A3 of the service-class
+ * pair, BRAINY-PROD-LATENCY-TRIAD). The production disease: a VFS file write
+ * ran a neural network synchronously while the caller waited (5.6s p50 per
+ * small file). The contract pinned here:
+ *
+ * 1. ACK AT DURABILITY: a deferred write never calls the embedder on the
+ *    caller's path — the row is id/metadata-findable immediately, with a
+ *    durable pending marker and an honest `pendingEmbeds` gauge.
+ * 2. EVENTUAL VECTOR INDEX: `awaitPendingEmbeds()` is the barrier — after
+ *    it, the vector is real, indexed, and the marker is reaped.
+ * 3. STALE-BEATS-ABSENT on deferred updates: the OLD vector keeps serving
+ *    until the atomic swap (the flicker law, never a dark window).
+ * 4. CRASH-SAFE: markers survive a session that dies mid-defer; the next
+ *    open recovers and lands the vector. A crash DELAYS a vector, never
+ *    loses one.
+ * 5. TYPED REFUSALS: deferEmbedding + vector, and deferEmbedding without
+ *    data, are caller bugs that refuse with the fix in the message.
+ */
+import { describe, it, expect, afterEach, vi } from 'vitest'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { Brainy } from '../../src/index.js'
+import { NounType } from '../../src/types/graphTypes.js'
+
+const dirs: string[] = []
+const brains: Brainy[] = []
+
+async function memBrain(): Promise<Brainy> {
+  const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false })
+  await b.init()
+  brains.push(b)
+  return b
+}
+
+afterEach(async () => {
+  vi.restoreAllMocks()
+  for (const b of brains.splice(0)) await b.close().catch(() => {})
+  for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
+})
+
+describe('MT5 — deferred embedding', () => {
+  it('ACK LAW: add({deferEmbedding}) never embeds on the caller path; row findable immediately; barrier lands the vector and reaps the marker', async () => {
+    const brain = await memBrain()
+    const embedSpy = vi.spyOn(brain, 'embed')
+
+    const id = await brain.add({
+      data: 'deferred content',
+      type: NounType.Document,
+      deferEmbedding: true,
+      metadata: { tag: 'deferred' }
+    })
+
+    // The caller's path never ran the embedder.
+    expect(embedSpy, 'no embed on the ack path').not.toHaveBeenCalled()
+
+    // Immediately findable by metadata; vector is the stub; gauge honest.
+    const found = await brain.find({ where: { tag: 'deferred' }, limit: 5 })
+    expect(found.map((r) => r.id)).toContain(id)
+    expect((await brain.getIndexStatus()).pendingEmbeds).toBeGreaterThanOrEqual(1)
+
+    // The barrier: vector lands, marker reaped, index carries the row.
+    await brain.awaitPendingEmbeds()
+    expect(embedSpy).toHaveBeenCalled()
+    const after = await brain.get(id, { includeVectors: true })
+    expect((after!.vector as number[]).length, 'real vector after the barrier').toBeGreaterThan(0)
+    expect(brain.pendingEmbedCount()).toBe(0)
+    expect((await brain.getIndexStatus()).pendingEmbeds).toBe(0)
+  })
+
+  it('STALE-BEATS-ABSENT: a deferred update serves the OLD vector until the atomic swap; data reads NEW immediately', async () => {
+    const brain = await memBrain()
+    const id = await brain.add({ data: 'original content', type: NounType.Document, metadata: {} })
+    const before = await brain.get(id, { includeVectors: true })
+    const oldVector = [...(before!.vector as number[])]
+    expect(oldVector.length).toBeGreaterThan(0)
+
+    await brain.update({ id, data: 'completely different content', deferEmbedding: true })
+
+    // Data is new IMMEDIATELY; the vector is still the old one (present,
+    // never absent) until the worker swaps it.
+    const mid = await brain.get(id, { includeVectors: true })
+    expect(mid!.data).toBe('completely different content')
+    expect(mid!.vector as number[], 'old vector keeps serving').toEqual(oldVector)
+
+    await brain.awaitPendingEmbeds()
+    const after = await brain.get(id, { includeVectors: true })
+    expect((after!.vector as number[]).length).toBeGreaterThan(0)
+    expect(after!.vector as number[], 'vector swapped after the barrier').not.toEqual(oldVector)
+  })
+
+  it('CRASH-SAFE: a session dying mid-defer leaves the durable marker; the next open recovers and lands the vector', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'brainy-defer-crash-'))
+    dirs.push(dir)
+
+    // Session 1: the embedder hangs → the worker can never complete; close()
+    // does not wait for it (crash-equivalent for the embed leg).
+    let brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
+    await brain.init()
+    brains.push(brain)
+    vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
+    const id = await brain.add({
+      data: 'survives the crash',
+      type: NounType.Document,
+      deferEmbedding: true,
+      metadata: { k: 1 }
+    })
+    expect(brain.pendingEmbedCount()).toBe(1)
+    await brain.close()
+    brains.pop()
+    vi.restoreAllMocks()
+
+    // Session 2: recovery lists the marker and resumes in the background.
+    brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
+    await brain.init()
+    brains.push(brain)
+    expect(brain.pendingEmbedCount(), 'marker recovered at open').toBe(1)
+
+    await brain.awaitPendingEmbeds()
+    const after = await brain.get(id, { includeVectors: true })
+    expect((after!.vector as number[]).length, 'the delayed vector landed').toBeGreaterThan(0)
+    expect(brain.pendingEmbedCount()).toBe(0)
+  }, 120000)
+
+  it('VFS ACK LAW: writeFile resolves even when the embedder HANGS forever — the ack never depends on a neural net', async () => {
+    const brain = await memBrain()
+    // The strongest form of the pin: an embedder that never resolves. If any
+    // part of the writeFile ack path awaited an embed, this test would hang.
+    // (The background worker legitimately picks the deferred embeds up later
+    // — it may even interleave on the event loop during writeFile's other
+    // awaits — but the CALLER'S promise must never depend on it.)
+    const hang = vi
+      .spyOn(brain, 'embed')
+      .mockImplementation(() => new Promise<number[]>(() => {}))
+
+    await brain.vfs.writeFile('/notes/today.md', '# The day\nA deferred capture.')
+
+    // Acked with the embedder hung: content + metadata fully readable.
+    const content = await brain.vfs.readFile('/notes/today.md')
+    expect(content.toString()).toContain('A deferred capture.')
+    expect(brain.pendingEmbedCount()).toBeGreaterThanOrEqual(1)
+
+    // Un-hang, abandon the poisoned in-flight run (its embed promise never
+    // resolves — production is covered by the worker's 60s hang guard; the
+    // test takes the white-box shortcut for speed), drain, verify.
+    hang.mockRestore()
+    ;(brain as unknown as { _embedWorkerFlight: Promise<void> | null })._embedWorkerFlight = null
+    await brain.awaitPendingEmbeds()
+    expect(brain.pendingEmbedCount()).toBe(0)
+  })
+
+  it('TYPED REFUSALS: defer+vector and defer-without-data both refuse with the fix', async () => {
+    const brain = await memBrain()
+    await expect(
+      brain.add({
+        data: 'x',
+        vector: new Array(384).fill(0.1),
+        type: NounType.Document,
+        deferEmbedding: true,
+        metadata: {}
+      })
+    ).rejects.toThrow(/deferEmbedding cannot be combined/)
+
+    const id = await brain.add({ data: 'y', type: NounType.Document, metadata: {} })
+    await expect(
+      brain.update({ id, deferEmbedding: true, metadata: { z: 1 } })
+    ).rejects.toThrow(/requires new 'data'/)
+  })
+})