/** * @module tests/integration/metadata-online-rebuild * @description THE ONLINE JS METADATA REBUILD (B3 Deliverable 3) pins. * `MetadataIndexManager.rebuild()` used to be clear-then-walk — reads went * dark for the duration. `repairIndex({ rebuild: ['metadata'] })` now builds * a fresh replacement index BESIDE the live one (walk canonical + mirror * every live write via `beginShadow`/`endShadow` + a bounded fact-log fold), * then atomically swaps the brain's reference — `find()` never observes a * half-built index, and a write landing DURING the build is never lost. */ process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' import { describe, it, expect, afterEach } from 'vitest' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Brainy } from '../../src/brainy.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' import type { MetadataIndexManager } from '../../src/utils/metadataIndex.js' const dirs: string[] = [] const brains: Brainy[] = [] afterEach(async () => { for (const b of brains.splice(0)) await b.close().catch(() => {}) for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) }) function metadataIndexOf(brain: Brainy): MetadataIndexManager { return (brain as unknown as { metadataIndex: MetadataIndexManager }).metadataIndex } async function openBrain(): Promise<{ brain: Brainy; dir: string }> { const dir = mkdtempSync(join(tmpdir(), 'brainy-online-rebuild-')) dirs.push(dir) const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, persistence: { policy: 'manual' }, logAuthority: 'adopt' }) await brain.init() brains.push(brain) return { brain, dir } } describe('repairIndex({ rebuild: ["metadata"] }) — the online build-beside rebuild', () => { it( 'a find() polled throughout the rebuild of a 2k-noun store never returns fewer rows than ' + 'before the build started, and a write landing DURING the build is never lost', async () => { const { brain, dir } = await openBrain() void dir const N = 2000 const ids: string[] = [] for (let i = 0; i < N; i++) { ids.push( await brain.add({ data: `entity ${i}`, type: NounType.Person, metadata: { status: i % 2 === 0 ? 'active' : 'inactive' } }) ) } for (let i = 0; i < 20; i++) { await brain.relate({ from: ids[i], to: ids[i + 1], type: VerbType.WorksWith, metadata: { tag: 'orig' } }) } await brain.flush() const baseline = await brain.find({ where: { status: 'active' }, limit: 10000 }) expect(baseline.length).toBe(N / 2) // Kick off the online rebuild WITHOUT awaiting — poll reads and // perform a live write concurrently with it. const repairPromise = brain.repairIndex({ rebuild: ['metadata'] }) let minObserved = Infinity let polls = 0 const pollPromise = (async () => { // Poll until the rebuild settles — bounded so a slow CI box can't // spin forever, generous enough to actually overlap the walk. while (polls < 200) { const rows = await brain.find({ where: { status: 'active' }, limit: 10000 }) minObserved = Math.min(minObserved, rows.length) polls++ await new Promise((resolve) => setTimeout(resolve, 1)) } })() const newId = await brain.add({ data: 'added during the rebuild', type: NounType.Person, metadata: { status: 'active' } }) const newRelId = await brain.relate({ from: newId, to: ids[0], type: VerbType.WorksWith, metadata: { tag: 'during-build' } }) const [report] = await Promise.all([repairPromise, pollPromise]) // THE PIN: never fewer rows than the pre-build baseline, at any polled // instant — reads served the OLD (fully-populated) manager throughout. expect(polls).toBeGreaterThan(0) expect(minObserved).toBeGreaterThanOrEqual(baseline.length) // The repair report still accounts for the family (same receipt shape // regardless of which rebuild mechanism actually ran underneath). const metadataFamily = report.families.find((f) => f.family === 'provider:metadata') expect(metadataFamily?.checked).toBe(true) expect(metadataFamily?.rebuilt).toBe(true) // Post-swap correctness: the live write during the build was never // lost (the beginShadow mirror + post-walk fold caught it). const afterActive = await brain.find({ where: { status: 'active' }, limit: 10000 }) expect(afterActive.length).toBe(baseline.length + 1) expect(afterActive.some((r) => r.id === newId)).toBe(true) const index = metadataIndexOf(brain) expect(await index.getIds('tag', 'during-build')).toEqual([newRelId]) expect((await index.getIds('tag', 'orig')).length).toBe(20) // The swap stamped the watermark — a reopen adopts, zero rebuild. await brain.close() brains.length = 0 // already closed above; afterEach must not double-close const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, persistence: { policy: 'manual' }, logAuthority: 'adopt' }) await reopened.init() brains.push(reopened) const reopenedIndex = metadataIndexOf(reopened) expect(reopenedIndex.watermarkVerdict()).toBe('adopt') const reopenedActive = await reopened.find({ where: { status: 'active' }, limit: 10000 }) expect(reopenedActive.length).toBe(afterActive.length) }, 60000 ) it('repairIndex({ rebuild: ["metadata"] }) on an empty store is a trivial no-op walk', async () => { const { brain } = await openBrain() const report = await brain.repairIndex({ rebuild: ['metadata'] }) const metadataFamily = report.families.find((f) => f.family === 'provider:metadata') expect(metadataFamily?.checked).toBe(true) expect(await brain.getNounCount()).toBe(0) }) it('two consecutive online rebuilds both leave the index correct (idempotent)', async () => { const { brain } = await openBrain() const a = await brain.add({ data: 'a', type: NounType.Person, metadata: { status: 'active' } }) await brain.add({ data: 'b', type: NounType.Person, metadata: { status: 'inactive' } }) await brain.flush() await brain.repairIndex({ rebuild: ['metadata'] }) const first = await brain.find({ where: { status: 'active' } }) expect(first.map((r) => r.id)).toEqual([a]) await brain.repairIndex({ rebuild: ['metadata'] }) const second = await brain.find({ where: { status: 'active' } }) expect(second.map((r) => r.id)).toEqual([a]) }) })