open-brainy/tests/integration/deferred-embedding.test.ts

/**
 * @module tests/integration/deferred-embedding
 * @description MT5 — THE DEFERRED-EMBEDDING CONTRACT (A3 of the service-class
 * pair, BRAINY-PROD-LATENCY-TRIAD). The production disease: a VFS file write
 * ran a neural network synchronously while the caller waited (5.6s p50 per
 * small file). The contract pinned here:
 *
 * 1. ACK AT DURABILITY: a deferred write never calls the embedder on the
 *    caller's path — the row is id/metadata-findable immediately, with a
 *    durable pending marker and an honest `pendingEmbeds` gauge.
 * 2. EVENTUAL VECTOR INDEX: `awaitPendingEmbeds()` is the barrier — after
 *    it, the vector is real, indexed, and the marker is reaped.
 * 3. STALE-BEATS-ABSENT on deferred updates: the OLD vector keeps serving
 *    until the atomic swap (the flicker law, never a dark window).
 * 4. CRASH-SAFE: markers survive a session that dies mid-defer; the next
 *    open recovers and lands the vector. A crash DELAYS a vector, never
 *    loses one.
 * 5. TYPED REFUSALS: deferEmbedding + vector, and deferEmbedding without
 *    data, are caller bugs that refuse with the fix in the message.
 */
import { describe, it, expect, afterEach, vi } from 'vitest'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Brainy } from '../../src/index.js'
import { NounType } from '../../src/types/graphTypes.js'

const dirs: string[] = []
const brains: Brainy[] = []

async function memBrain(): Promise<Brainy> {
  const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false })
  await b.init()
  brains.push(b)
  return b
}

afterEach(async () => {
  vi.restoreAllMocks()
  for (const b of brains.splice(0)) await b.close().catch(() => {})
  for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
})

describe('MT5 — deferred embedding', () => {
  it('ACK LAW: add({deferEmbedding}) never embeds on the caller path; row findable immediately; barrier lands the vector and reaps the marker', async () => {
    const brain = await memBrain()
    const embedSpy = vi.spyOn(brain, 'embed')

    const id = await brain.add({
      data: 'deferred content',
      type: NounType.Document,
      deferEmbedding: true,
      metadata: { tag: 'deferred' }
    })

    // The caller's path never ran the embedder.
    expect(embedSpy, 'no embed on the ack path').not.toHaveBeenCalled()

    // Immediately findable by metadata; vector is the stub; gauge honest.
    const found = await brain.find({ where: { tag: 'deferred' }, limit: 5 })
    expect(found.map((r) => r.id)).toContain(id)
    expect((await brain.getIndexStatus()).pendingEmbeds).toBeGreaterThanOrEqual(1)

    // The barrier: vector lands, marker reaped, index carries the row.
    await brain.awaitPendingEmbeds()
    expect(embedSpy).toHaveBeenCalled()
    const after = await brain.get(id, { includeVectors: true })
    expect((after!.vector as number[]).length, 'real vector after the barrier').toBeGreaterThan(0)
    expect(brain.pendingEmbedCount()).toBe(0)
    expect((await brain.getIndexStatus()).pendingEmbeds).toBe(0)
  })

  it('STALE-BEATS-ABSENT: a deferred update serves the OLD vector until the atomic swap; data reads NEW immediately', async () => {
    const brain = await memBrain()
    const id = await brain.add({ data: 'original content', type: NounType.Document, metadata: {} })
    const before = await brain.get(id, { includeVectors: true })
    const oldVector = [...(before!.vector as number[])]
    expect(oldVector.length).toBeGreaterThan(0)

    await brain.update({ id, data: 'completely different content', deferEmbedding: true })

    // Data is new IMMEDIATELY; the vector is still the old one (present,
    // never absent) until the worker swaps it.
    const mid = await brain.get(id, { includeVectors: true })
    expect(mid!.data).toBe('completely different content')
    expect(mid!.vector as number[], 'old vector keeps serving').toEqual(oldVector)

    await brain.awaitPendingEmbeds()
    const after = await brain.get(id, { includeVectors: true })
    expect((after!.vector as number[]).length).toBeGreaterThan(0)
    expect(after!.vector as number[], 'vector swapped after the barrier').not.toEqual(oldVector)
  })

  it('CRASH-SAFE: a session dying mid-defer leaves the durable marker; the next open recovers and lands the vector', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'brainy-defer-crash-'))
    dirs.push(dir)

    // Session 1: the embedder hangs → the worker can never complete; close()
    // does not wait for it (crash-equivalent for the embed leg).
    let brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
    await brain.init()
    brains.push(brain)
    vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
    const id = await brain.add({
      data: 'survives the crash',
      type: NounType.Document,
      deferEmbedding: true,
      metadata: { k: 1 }
    })
    expect(brain.pendingEmbedCount()).toBe(1)
    await brain.close()
    brains.pop()
    vi.restoreAllMocks()

    // Session 2: recovery lists the marker and resumes in the background.
    brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
    await brain.init()
    brains.push(brain)
    expect(brain.pendingEmbedCount(), 'marker recovered at open').toBe(1)

    await brain.awaitPendingEmbeds()
    const after = await brain.get(id, { includeVectors: true })
    expect((after!.vector as number[]).length, 'the delayed vector landed').toBeGreaterThan(0)
    expect(brain.pendingEmbedCount()).toBe(0)
  }, 120000)

  it('VFS ACK LAW: writeFile resolves even when the embedder HANGS forever — the ack never depends on a neural net', async () => {
    const brain = await memBrain()
    // The strongest form of the pin: an embedder that never resolves. If any
    // part of the writeFile ack path awaited an embed, this test would hang.
    // (The background worker legitimately picks the deferred embeds up later
    // — it may even interleave on the event loop during writeFile's other
    // awaits — but the CALLER'S promise must never depend on it.)
    const hang = vi
      .spyOn(brain, 'embed')
      .mockImplementation(() => new Promise<number[]>(() => {}))

    await brain.vfs.writeFile('/notes/today.md', '# The day\nA deferred capture.')

    // Acked with the embedder hung: content + metadata fully readable.
    const content = await brain.vfs.readFile('/notes/today.md')
    expect(content.toString()).toContain('A deferred capture.')
    expect(brain.pendingEmbedCount()).toBeGreaterThanOrEqual(1)

    // Un-hang, abandon the poisoned in-flight run (its embed promise never
    // resolves — production is covered by the worker's 60s hang guard; the
    // test takes the white-box shortcut for speed), drain, verify.
    hang.mockRestore()
    ;(brain as unknown as { _embedWorkerFlight: Promise<void> | null })._embedWorkerFlight = null
    await brain.awaitPendingEmbeds()
    expect(brain.pendingEmbedCount()).toBe(0)
  })

  it('TYPED REFUSALS: defer+vector and defer-without-data both refuse with the fix', async () => {
    const brain = await memBrain()
    await expect(
      brain.add({
        data: 'x',
        vector: new Array(384).fill(0.1),
        type: NounType.Document,
        deferEmbedding: true,
        metadata: {}
      })
    ).rejects.toThrow(/deferEmbedding cannot be combined/)

    const id = await brain.add({ data: 'y', type: NounType.Document, metadata: {} })
    await expect(
      brain.update({ id, deferEmbedding: true, metadata: { z: 1 } })
    ).rejects.toThrow(/requires new 'data'/)
  })
})