Two gate failures on main, one real and one long-hidden. THE HEALTH-GATE PIN encoded the old law — "narrates once per generation, twice across a generation bump" — which the content-keyed dedupe deliberately replaced. A provider's `generation` bumps on every ledger mutation and every rebuild boundary, so keying narration on it re-printed an unchanged health line on every read that consulted a busy provider, and let a provider that never bumped suppress a line whose reasons had genuinely changed. The pin now asserts BOTH directions: an unchanged verdict stays silent however the counter moves, and a changed verdict is always heard. THE VFS HYBRID-SEARCH SUITE configured its store with `options.basePath`, an alias removed at the 8.0 major that configures nothing. The suite was therefore never using its temp directory — it opened the DEFAULT store, shared with every other run on the machine, and accumulated tens of thousands of rows until it failed on that shared store's graph adjacency instead of on anything it tests. It now passes `storage.path`. The suite drops from 6.5s to 0.3s, which is the measure of how much foreign data it had been opening. Neither failure was caused by the release branch; the first is the branch's own behaviour change meeting its outdated pin, the second predates it.
367 lines
14 KiB
TypeScript
367 lines
14 KiB
TypeScript
/**
|
|
* @module tests/integration/health-gate
|
|
* @description Pins for the health-by-accounting read gate: the read gate stops
|
|
* consulting an unnamed `isReady()` boolean and reads a NAMED, sync, O(1)
|
|
* {@link HealthReport}; no read path may ever start a store walk; the open path
|
|
* brings every provider to serving before it returns; an explicit operator door
|
|
* (`repairIndex({ rebuild: [...] })`) rebuilds a named leg unconditionally.
|
|
*
|
|
* Providers here are white-box test doubles: a `healthReport()` (or, for the
|
|
* interim-path pins, an `isReady()`) function assigned directly onto the LIVE
|
|
* JS provider object, the same pattern `tests/unit/validate-invariants-delegation.test.ts`
|
|
* uses for `validateInvariants`. This exercises brainy's real gate/verify code
|
|
* against a controlled provider self-report — no engine mocks.
|
|
*/
|
|
import { describe, it, expect, afterEach, vi } from 'vitest'
|
|
import { mkdtempSync, rmSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import {
|
|
Brainy,
|
|
NounType,
|
|
VerbType,
|
|
GraphIndexNotReadyError,
|
|
MetadataIndexNotReadyError,
|
|
VectorIndexNotReadyError
|
|
} from '../../src/index.js'
|
|
import type { HealthReport, LedgerInvariantResult } from '../../src/plugin.js'
|
|
import { prodLog } from '../../src/utils/logger.js'
|
|
import { createTestConfig } from '../helpers/test-factory.js'
|
|
|
|
/** The white-box surface these pins drive on a live brain instance. */
|
|
interface BrainInternals {
|
|
storage: {
|
|
getNoun(id: string): Promise<unknown>
|
|
getNounMetadata(id: string): Promise<unknown>
|
|
getNouns(options?: unknown): Promise<unknown>
|
|
getVerbs(options?: unknown): Promise<unknown>
|
|
}
|
|
index: { healthReport?: () => HealthReport; isReady?: () => boolean; rebuild(): Promise<void> }
|
|
metadataIndex: {
|
|
healthReport?: () => HealthReport
|
|
isReady?: () => boolean
|
|
rebuild(): Promise<void>
|
|
validateInvariants?: () => Promise<unknown>
|
|
}
|
|
graphIndex: {
|
|
healthReport?: () => HealthReport
|
|
isReady?: () => boolean
|
|
rebuild(): Promise<void>
|
|
validateInvariants?: () => Promise<unknown>
|
|
}
|
|
rebuildIndexesIfNeeded(force?: boolean): Promise<void>
|
|
}
|
|
|
|
function internalsOf(brain: Brainy): BrainInternals {
|
|
return brain as unknown as BrainInternals
|
|
}
|
|
|
|
function invariant(overrides: Partial<LedgerInvariantResult> = {}): LedgerInvariantResult {
|
|
return {
|
|
name: 'manifest-residency',
|
|
holds: true,
|
|
detail: 'ok',
|
|
heal: 'none',
|
|
source: 'ledger',
|
|
...overrides
|
|
}
|
|
}
|
|
|
|
function healthReport(overrides: Partial<HealthReport> = {}): HealthReport {
|
|
return {
|
|
provider: 'vector',
|
|
healthy: true,
|
|
serving: true,
|
|
invariants: [],
|
|
checkedAt: Date.now(),
|
|
durationMs: 1,
|
|
generation: 1,
|
|
unledgered: [],
|
|
...overrides
|
|
}
|
|
}
|
|
|
|
const brains: Brainy[] = []
|
|
const dirs: string[] = []
|
|
afterEach(async () => {
|
|
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
|
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
|
vi.restoreAllMocks()
|
|
})
|
|
|
|
describe('health gate (a) — not-serving refuses loudly, ZERO canonical reads during the refusal', () => {
|
|
it('metadata not-serving: find() throws MetadataIndexNotReadyError naming the failing invariant', async () => {
|
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
|
await brain.init()
|
|
brains.push(brain)
|
|
await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
await brain.flush()
|
|
|
|
const internals = internalsOf(brain)
|
|
internals.metadataIndex.healthReport = () =>
|
|
healthReport({
|
|
provider: 'metadata',
|
|
serving: false,
|
|
healthy: false,
|
|
invariants: [invariant({ name: 'posted-count-floor', holds: false, heal: 'rebuild', detail: 'posted 2 < canonical 5' })]
|
|
})
|
|
|
|
const getNounSpy = vi.spyOn(internals.storage, 'getNoun')
|
|
const getNounMetadataSpy = vi.spyOn(internals.storage, 'getNounMetadata')
|
|
const getNounsSpy = vi.spyOn(internals.storage, 'getNouns')
|
|
|
|
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError)
|
|
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/posted-count-floor/)
|
|
|
|
expect(getNounSpy).not.toHaveBeenCalled()
|
|
expect(getNounMetadataSpy).not.toHaveBeenCalled()
|
|
expect(getNounsSpy).not.toHaveBeenCalled()
|
|
|
|
delete internals.metadataIndex.healthReport
|
|
})
|
|
|
|
it('graph not-serving: related() throws GraphIndexNotReadyError naming the failing invariant, no canonical reads', async () => {
|
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
|
await brain.init()
|
|
brains.push(brain)
|
|
const a = await brain.add({ data: 'a', type: NounType.Person })
|
|
const b = await brain.add({ data: 'b', type: NounType.Person })
|
|
await brain.relate({ from: a, to: b, type: VerbType.Knows })
|
|
await brain.flush()
|
|
|
|
const internals = internalsOf(brain)
|
|
internals.graphIndex.healthReport = () =>
|
|
healthReport({
|
|
provider: 'graph',
|
|
serving: false,
|
|
healthy: false,
|
|
invariants: [invariant({ name: 'adjacency-residency', holds: false, heal: 'rebuild', detail: 'edges not loaded' })]
|
|
})
|
|
|
|
const getNounSpy = vi.spyOn(internals.storage, 'getNoun')
|
|
const getVerbsSpy = vi.spyOn(internals.storage, 'getVerbs')
|
|
|
|
await expect(brain.related({ from: a })).rejects.toBeInstanceOf(GraphIndexNotReadyError)
|
|
await expect(brain.related({ from: a })).rejects.toThrow(/adjacency-residency/)
|
|
|
|
expect(getNounSpy).not.toHaveBeenCalled()
|
|
expect(getVerbsSpy).not.toHaveBeenCalled()
|
|
|
|
delete internals.graphIndex.healthReport
|
|
})
|
|
})
|
|
|
|
describe('health gate (b) — unledgered is unknown: never blocks a serving provider', () => {
|
|
it('serving:true with an unledgered family and no failing invariant serves normally; at most one narration', async () => {
|
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
|
await brain.init()
|
|
brains.push(brain)
|
|
await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
await brain.flush()
|
|
|
|
const internals = internalsOf(brain)
|
|
internals.metadataIndex.healthReport = () =>
|
|
healthReport({
|
|
provider: 'metadata',
|
|
serving: true,
|
|
healthy: true,
|
|
invariants: [],
|
|
unledgered: ['canonical-verb-coverage']
|
|
})
|
|
|
|
const warnSpy = vi.spyOn(prodLog, 'warn')
|
|
|
|
const r1 = await brain.find({ where: { team: 'atlas' } })
|
|
const r2 = await brain.find({ where: { team: 'atlas' } })
|
|
expect(r1.length).toBe(1)
|
|
expect(r2.length).toBe(1)
|
|
|
|
const narrations = warnSpy.mock.calls.filter(
|
|
([msg]) => typeof msg === 'string' && msg.includes('canonical-verb-coverage')
|
|
)
|
|
expect(narrations.length).toBe(1) // one narration at most across both reads (same generation)
|
|
|
|
delete internals.metadataIndex.healthReport
|
|
})
|
|
})
|
|
|
|
describe('health gate (c) — degraded-but-serving narrates once per generation', () => {
|
|
// PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the
|
|
// degraded report lives on the family the read actually consults.
|
|
it('a heal:"repair" failure serves; narrates once per DISTINCT VERDICT, not once per generation bump', async () => {
|
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
|
await brain.init()
|
|
brains.push(brain)
|
|
await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
await brain.flush()
|
|
|
|
const internals = internalsOf(brain)
|
|
let generation = 1
|
|
let detail = 'counter drift'
|
|
internals.metadataIndex.healthReport = () =>
|
|
healthReport({
|
|
provider: 'vector',
|
|
serving: true,
|
|
healthy: false,
|
|
invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail })],
|
|
generation
|
|
})
|
|
|
|
const warnSpy = vi.spyOn(prodLog, 'warn')
|
|
const countNarrations = () =>
|
|
warnSpy.mock.calls.filter(([msg]) => typeof msg === 'string' && msg.includes('stale-vector-counter')).length
|
|
|
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
|
expect(countNarrations()).toBe(1) // same verdict both times — one narration
|
|
|
|
// THE DEDUPE KEY IS THE VERDICT, NOT THE COUNTER. A provider's `generation`
|
|
// bumps on every ledger mutation and every rebuild boundary, so keying the
|
|
// narration on it re-printed an UNCHANGED health line on every read that
|
|
// consulted a busy provider — and, in the other direction, let a provider
|
|
// that never bumped suppress a line whose reasons had genuinely changed.
|
|
// An unchanged verdict is silent however the counter moves:
|
|
generation = 2
|
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
|
expect(countNarrations()).toBe(1) // generation bumped, verdict identical — still silent
|
|
|
|
// ...and a CHANGED verdict is always heard, bump or no bump:
|
|
detail = 'counter drift widened to 12 rows'
|
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
|
expect(countNarrations()).toBe(2) // the reasons changed — a new narration
|
|
|
|
delete internals.metadataIndex.healthReport
|
|
})
|
|
})
|
|
|
|
describe('health gate (d) — interim isReady()-only path (no healthReport) is unchanged', () => {
|
|
it('isReady() === true serves; isReady() === false refuses via the typed NotReady error', async () => {
|
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
|
await brain.init()
|
|
brains.push(brain)
|
|
await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
await brain.flush()
|
|
|
|
const internals = internalsOf(brain)
|
|
internals.metadataIndex.isReady = () => true
|
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
|
|
|
internals.metadataIndex.isReady = () => false
|
|
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError)
|
|
|
|
delete internals.metadataIndex.isReady
|
|
})
|
|
})
|
|
|
|
describe('health gate (e) — open builds; the first read never does', () => {
|
|
it('disableAutoRebuild:true on a populated store: open narrates + builds; the first find() triggers zero rebuilds', async () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'brainy-healthgate-open-'))
|
|
dirs.push(dir)
|
|
|
|
const writer = new Brainy({
|
|
storage: { type: 'filesystem', path: dir },
|
|
requireSubtype: false,
|
|
silent: true,
|
|
disableAutoRebuild: true
|
|
})
|
|
await writer.init()
|
|
brains.push(writer)
|
|
await writer.add({ data: 'row one', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
await writer.flush()
|
|
await brains.pop()!.close()
|
|
|
|
const warnSpy = vi.spyOn(prodLog, 'warn')
|
|
const reader = new Brainy({
|
|
storage: { type: 'filesystem', path: dir },
|
|
requireSubtype: false,
|
|
silent: true,
|
|
disableAutoRebuild: true
|
|
})
|
|
const internals = internalsOf(reader)
|
|
const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded')
|
|
|
|
await reader.init()
|
|
brains.push(reader)
|
|
|
|
expect(rebuildSpy).toHaveBeenCalledTimes(1) // open() built it, exactly once
|
|
expect(
|
|
warnSpy.mock.calls.some(
|
|
([msg]) => typeof msg === 'string' && msg.includes('open() is building')
|
|
)
|
|
).toBe(true)
|
|
|
|
rebuildSpy.mockClear()
|
|
const rows = await reader.find({ where: { team: 'atlas' } })
|
|
expect(rebuildSpy).toHaveBeenCalledTimes(0) // the read never builds
|
|
expect(rows.length).toBe(1)
|
|
}, 30000)
|
|
})
|
|
|
|
describe('health gate (f) — the ceremony door: explicit rebuild bypasses invariant consultation', () => {
|
|
it("repairIndex({ rebuild: ['graph'] }) rebuilds unconditionally without consulting validateInvariants", async () => {
|
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
|
await brain.init()
|
|
brains.push(brain)
|
|
await brain.add({ data: 'x', type: NounType.Concept })
|
|
await brain.flush()
|
|
|
|
const internals = internalsOf(brain)
|
|
let validateCalls = 0
|
|
internals.graphIndex.validateInvariants = async () => {
|
|
validateCalls++
|
|
return healthReport({ provider: 'graph' })
|
|
}
|
|
const rebuildSpy = vi.spyOn(internals.graphIndex, 'rebuild')
|
|
|
|
const report = await brain.repairIndex({ rebuild: ['graph'] })
|
|
|
|
expect(rebuildSpy).toHaveBeenCalledTimes(1)
|
|
expect(validateCalls).toBe(0) // the door never consults validateInvariants to decide
|
|
|
|
const graphFamily = report.families.find((f) => f.family === 'provider:graph')
|
|
expect(graphFamily?.rebuilt).toBe(true)
|
|
expect(graphFamily?.checked).toBe(true)
|
|
expect(graphFamily?.reason).toBe('explicit rebuild requested')
|
|
|
|
delete internals.graphIndex.validateInvariants
|
|
})
|
|
|
|
it('bare repairIndex() on a healthy provider calls no rebuild()', async () => {
|
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
|
await brain.init()
|
|
brains.push(brain)
|
|
await brain.add({ data: 'x', type: NounType.Concept })
|
|
await brain.flush()
|
|
|
|
const internals = internalsOf(brain)
|
|
internals.graphIndex.validateInvariants = async () => healthReport({ provider: 'graph', healthy: true, serving: true })
|
|
const rebuildSpy = vi.spyOn(internals.graphIndex, 'rebuild')
|
|
|
|
await brain.repairIndex()
|
|
|
|
expect(rebuildSpy).not.toHaveBeenCalled()
|
|
|
|
delete internals.graphIndex.validateInvariants
|
|
})
|
|
})
|
|
|
|
describe('health gate (g) — a throwing healthReport() is a contract violation, never read as healthy', () => {
|
|
// PER-FAMILY LAW (10.4.1): the throwing report sits on the family the read consults.
|
|
it('healthReport() that throws refuses loudly with the typed NotReady error naming the throw', async () => {
|
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
|
await brain.init()
|
|
brains.push(brain)
|
|
await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } })
|
|
await brain.flush()
|
|
|
|
const internals = internalsOf(brain)
|
|
internals.metadataIndex.healthReport = () => {
|
|
throw new Error('accelerator: mmap window busy')
|
|
}
|
|
|
|
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError)
|
|
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/mmap window busy/)
|
|
|
|
delete internals.metadataIndex.healthReport
|
|
})
|
|
})
|