open-brainy/tests/integration/metadata-online-rebuild.test.ts

/**
 * @module tests/integration/metadata-online-rebuild
 * @description THE ONLINE JS METADATA REBUILD (B3 Deliverable 3) pins.
 * `MetadataIndexManager.rebuild()` used to be clear-then-walk — reads went
 * dark for the duration. `repairIndex({ rebuild: ['metadata'] })` now builds
 * a fresh replacement index BESIDE the live one (walk canonical + mirror
 * every live write via `beginShadow`/`endShadow` + a bounded fact-log fold),
 * then atomically swaps the brain's reference — `find()` never observes a
 * half-built index, and a write landing DURING the build is never lost.
 */
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'

import { describe, it, expect, afterEach } from 'vitest'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Brainy } from '../../src/brainy.js'
import { NounType, VerbType } from '../../src/types/graphTypes.js'
import type { MetadataIndexManager } from '../../src/utils/metadataIndex.js'

const dirs: string[] = []
const brains: Brainy<any>[] = []
afterEach(async () => {
  for (const b of brains.splice(0)) await b.close().catch(() => {})
  for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
})

function metadataIndexOf(brain: Brainy<any>): MetadataIndexManager {
  return (brain as unknown as { metadataIndex: MetadataIndexManager }).metadataIndex
}

async function openBrain(): Promise<{ brain: Brainy<any>; dir: string }> {
  const dir = mkdtempSync(join(tmpdir(), 'brainy-online-rebuild-'))
  dirs.push(dir)
  const brain = new Brainy<any>({
    requireSubtype: false,
    storage: { type: 'filesystem', path: dir },
    silent: true,
    persistence: { policy: 'manual' },
    logAuthority: 'adopt'
  })
  await brain.init()
  brains.push(brain)
  return { brain, dir }
}

describe('repairIndex({ rebuild: ["metadata"] }) — the online build-beside rebuild', () => {
  it(
    'a find() polled throughout the rebuild of a 2k-noun store never returns fewer rows than ' +
      'before the build started, and a write landing DURING the build is never lost',
    async () => {
      const { brain, dir } = await openBrain()
      void dir

      const N = 2000
      const ids: string[] = []
      for (let i = 0; i < N; i++) {
        ids.push(
          await brain.add({
            data: `entity ${i}`,
            type: NounType.Person,
            metadata: { status: i % 2 === 0 ? 'active' : 'inactive' }
          })
        )
      }
      for (let i = 0; i < 20; i++) {
        await brain.relate({
          from: ids[i], to: ids[i + 1], type: VerbType.WorksWith, metadata: { tag: 'orig' }
        })
      }
      await brain.flush()

      const baseline = await brain.find({ where: { status: 'active' }, limit: 10000 })
      expect(baseline.length).toBe(N / 2)

      // Kick off the online rebuild WITHOUT awaiting — poll reads and
      // perform a live write concurrently with it.
      const repairPromise = brain.repairIndex({ rebuild: ['metadata'] })

      let minObserved = Infinity
      let polls = 0
      const pollPromise = (async () => {
        // Poll until the rebuild settles — bounded so a slow CI box can't
        // spin forever, generous enough to actually overlap the walk.
        while (polls < 200) {
          const rows = await brain.find({ where: { status: 'active' }, limit: 10000 })
          minObserved = Math.min(minObserved, rows.length)
          polls++
          await new Promise((resolve) => setTimeout(resolve, 1))
        }
      })()

      const newId = await brain.add({
        data: 'added during the rebuild',
        type: NounType.Person,
        metadata: { status: 'active' }
      })
      const newRelId = await brain.relate({
        from: newId, to: ids[0], type: VerbType.WorksWith, metadata: { tag: 'during-build' }
      })

      const [report] = await Promise.all([repairPromise, pollPromise])

      // THE PIN: never fewer rows than the pre-build baseline, at any polled
      // instant — reads served the OLD (fully-populated) manager throughout.
      expect(polls).toBeGreaterThan(0)
      expect(minObserved).toBeGreaterThanOrEqual(baseline.length)

      // The repair report still accounts for the family (same receipt shape
      // regardless of which rebuild mechanism actually ran underneath).
      const metadataFamily = report.families.find((f) => f.family === 'provider:metadata')
      expect(metadataFamily?.checked).toBe(true)
      expect(metadataFamily?.rebuilt).toBe(true)

      // Post-swap correctness: the live write during the build was never
      // lost (the beginShadow mirror + post-walk fold caught it).
      const afterActive = await brain.find({ where: { status: 'active' }, limit: 10000 })
      expect(afterActive.length).toBe(baseline.length + 1)
      expect(afterActive.some((r) => r.id === newId)).toBe(true)

      const index = metadataIndexOf(brain)
      expect(await index.getIds('tag', 'during-build')).toEqual([newRelId])
      expect((await index.getIds('tag', 'orig')).length).toBe(20)

      // The swap stamped the watermark — a reopen adopts, zero rebuild.
      await brain.close()
      brains.length = 0 // already closed above; afterEach must not double-close
      const reopened = new Brainy<any>({
        requireSubtype: false,
        storage: { type: 'filesystem', path: dir },
        silent: true,
        persistence: { policy: 'manual' },
        logAuthority: 'adopt'
      })
      await reopened.init()
      brains.push(reopened)
      const reopenedIndex = metadataIndexOf(reopened)
      expect(reopenedIndex.watermarkVerdict()).toBe('adopt')
      const reopenedActive = await reopened.find({ where: { status: 'active' }, limit: 10000 })
      expect(reopenedActive.length).toBe(afterActive.length)
    },
    60000
  )

  it('repairIndex({ rebuild: ["metadata"] }) on an empty store is a trivial no-op walk', async () => {
    const { brain } = await openBrain()
    const report = await brain.repairIndex({ rebuild: ['metadata'] })
    const metadataFamily = report.families.find((f) => f.family === 'provider:metadata')
    expect(metadataFamily?.checked).toBe(true)
    expect(await brain.getNounCount()).toBe(0)
  })

  it('two consecutive online rebuilds both leave the index correct (idempotent)', async () => {
    const { brain } = await openBrain()
    const a = await brain.add({ data: 'a', type: NounType.Person, metadata: { status: 'active' } })
    await brain.add({ data: 'b', type: NounType.Person, metadata: { status: 'inactive' } })
    await brain.flush()

    await brain.repairIndex({ rebuild: ['metadata'] })
    const first = await brain.find({ where: { status: 'active' } })
    expect(first.map((r) => r.id)).toEqual([a])

    await brain.repairIndex({ rebuild: ['metadata'] })
    const second = await brain.find({ where: { status: 'active' } })
    expect(second.map((r) => r.id)).toEqual([a])
  })
})