Merge branch 'release/8.10.0'

This commit is contained in:
David Snelling 2026-07-23 10:50:39 -07:00
commit d918c060f4
21 changed files with 1283 additions and 346 deletions

View file

@ -0,0 +1,310 @@
/**
* @module tests/unit/brainy/warm
* @description Coverage for `brain.warm()` / `warmOnOpen` / the provider
* `warm?()` contract (the cold-restart readiness fix): first operations after
* a cold restart should run at steady-state cost instead of paying
* demand-load latency on the critical path.
*
* Uses a real, in-process fake plugin provider implementing the actual
* `VectorIndexProvider` contract from `src/plugin.ts` the real seam a
* native provider (e.g. a disk-native accelerator) plugs into. The real
* built-in JS metadata index and graph adjacency index run against real
* (filesystem or in-memory) storage with pre-existing data, so their
* hydration paths are exercised for real, not mocked.
*/
import { describe, it, expect, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../../src/brainy.js'
import { NounType, VerbType } from '../../../src/types/graphTypes.js'
import type { VectorIndexProvider } from '../../../src/plugin.js'
import type { VectorDocument, Vector } from '../../../src/coreTypes.js'
import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js'
import { GraphAdjacencyIndex } from '../../../src/graph/graphAdjacencyIndex.js'
const tmpDirs: string[] = []
function mkTmp(): string {
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-warm-'))
tmpDirs.push(d)
return d
}
afterEach(() => {
for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
})
// Brainy's ValidationConfig fixes vectors at exactly 384 dimensions
// (src/utils/paramValidation.ts) — match it so `add()` doesn't reject test data.
const DIM = 384
const V = (seed = 1): number[] => Array.from({ length: DIM }, (_, i) => Math.sin(seed + i))
/**
* A real (not mocked) VectorIndexProvider implementation, backed by a plain
* Map, that optionally implements `warm()` the exact seam
* `AddToVectorIndexOperation` / `brain.warm()` call through.
*/
class FakeVectorProvider implements VectorIndexProvider {
readonly name = 'fake-vector-provider'
readonly items = new Map<string, Vector>()
warmCalls = 0
searchCalls: Array<{ k?: number }> = []
// Only present on the instance when the constructor is told to — mirrors a
// real provider that may or may not implement the optional hook. Assigned
// in the constructor BODY (not as a field initializer): under native ES
// class fields, field initializers run before constructor-body statements
// — including the parameter-property assignment — so referencing a
// parameter property from a field initializer would see it as still
// `undefined`.
warm?: () => Promise<void>
constructor(hasWarm: boolean) {
if (hasWarm) {
this.warm = async (): Promise<void> => {
this.warmCalls++
}
}
}
async addItem(item: VectorDocument): Promise<string> {
this.items.set(item.id, item.vector)
return item.id
}
async removeItem(id: string): Promise<boolean> {
return this.items.delete(id)
}
async search(_queryVector: Vector, k?: number): Promise<Array<[string, number]>> {
this.searchCalls.push({ k })
return [...this.items.keys()].slice(0, k ?? 10).map((id) => [id, 0])
}
size(): number {
return this.items.size
}
clear(): void {
this.items.clear()
}
async rebuild(): Promise<void> {}
async flush(): Promise<number> {
return 0
}
getPersistMode(): 'immediate' | 'deferred' {
return 'deferred'
}
}
/** Registers `provider` under the `'vector'` plugin key, before `init()`. */
function useFakeVectorProvider(brain: Brainy<any>, provider: FakeVectorProvider): void {
brain.use({
name: 'fake-vector-provider',
activate: async (ctx: any) => {
ctx.registerProvider('vector', () => provider)
return true
}
})
}
describe('brain.warm()', () => {
it('(a) calls the vector provider\'s warm() when present and reports "warmed"', async () => {
const brain = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
silent: true
})
const provider = new FakeVectorProvider(true)
useFakeVectorProvider(brain, provider)
await brain.init()
await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) })
const report = await brain.warm()
expect(provider.warmCalls).toBe(1)
expect(provider.searchCalls.length).toBe(0) // warm() ran — no probe fallback
expect(report.vector.outcome).toBe('warmed')
await brain.close()
})
it('(b) falls back to a probe search() when warm() is absent and reports "probed", never "warmed"', async () => {
const brain = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
silent: true
})
const provider = new FakeVectorProvider(false)
useFakeVectorProvider(brain, provider)
await brain.init()
await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) })
await brain.add({ data: 'b', type: NounType.Thing, vector: V(2) })
const report = await brain.warm()
expect(provider.warmCalls).toBe(0) // no warm() on this provider
expect(provider.searchCalls.length).toBe(1) // the probe ran
expect(provider.searchCalls[0].k).toBe(Math.min(100, provider.size())) // k = min(100, size)
expect(report.vector.outcome).toBe('probed')
expect(report.vector.outcome).not.toBe('warmed') // never conflated
await brain.close()
})
it('vector: reports "unavailable" when there is nothing to probe (empty index, unknown dimension)', async () => {
const brain = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
silent: true
})
// A disk-native provider MAY honestly report 0 resident entries while
// durable data exists on disk (the same posture documented on
// `VectorIndexProvider.isReady` — "an mmap/disk-native index may
// legitimately report 0 resident entries"). warm()'s probe fallback
// reads `size()`, so this is the real trigger for "nothing to probe":
// never a k=0 search, an honest skip.
const provider = new FakeVectorProvider(false)
provider.size = () => 0
useFakeVectorProvider(brain, provider)
await brain.init() // the VFS root bootstrap write sets `dimensions`, but size() still reports 0
const report = await brain.warm()
expect(provider.searchCalls.length).toBe(0) // nothing probed
expect(report.vector.outcome).toBe('unavailable')
await brain.close()
})
it('(c) metadata + graph hydration paths actually execute against filesystem storage with pre-existing data', async () => {
const dir = mkTmp()
// Build a brain with real data (including a field NOT in the metadata
// index's own common-fields warm subset — 'wave' — so hydrateAll()'s
// FULL hydration is distinguishable from init()'s partial warmCache()),
// and real graph edges, then close it (persisting everything).
const seed = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true
})
await seed.init()
const ids: string[] = []
for (let i = 0; i < 6; i++) {
ids.push(
await seed.add({
data: `entity ${i}`,
type: NounType.Thing,
metadata: { wave: i % 3 },
vector: V(i + 1)
})
)
}
for (let i = 0; i + 1 < ids.length; i++) {
await seed.relate({ from: ids[i], to: ids[i + 1], type: VerbType.RelatedTo })
}
await seed.close()
// Cold-reopen a FRESH instance and spy on the real hydration seams before
// calling warm(), so we assert they actually ran (not just that the
// report claims they did).
const metaHydrateCalls: number[] = []
const loadedFields: string[] = []
const graphInitCalls: number[] = []
const origHydrateAll = MetadataIndexManager.prototype.hydrateAll
const origLoadSparseIndex = (MetadataIndexManager.prototype as any).loadSparseIndex
const origGraphInit = GraphAdjacencyIndex.prototype.init
MetadataIndexManager.prototype.hydrateAll = async function (...args: any[]) {
metaHydrateCalls.push(1)
return origHydrateAll.apply(this, args as any)
}
;(MetadataIndexManager.prototype as any).loadSparseIndex = async function (
field: string,
...args: any[]
) {
loadedFields.push(field)
return origLoadSparseIndex.apply(this, [field, ...args] as any)
}
GraphAdjacencyIndex.prototype.init = async function (...args: any[]) {
graphInitCalls.push(1)
return origGraphInit.apply(this, args as any)
}
try {
const brain = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true
})
await brain.init()
const report = await brain.warm()
expect(metaHydrateCalls.length).toBe(1) // hydrateAll() actually ran
expect(loadedFields).toContain('wave') // a NON-common field was loaded — full hydration, not the heuristic subset
expect(report.metadata.outcome).toBe('warmed')
expect(graphInitCalls.length).toBeGreaterThanOrEqual(1) // graph's full-load seam ran (once at brain init, once via warm())
expect(report.graph.outcome).toBe('warmed')
// Correctness survives: the hydrated data still answers queries.
const byWhere = await brain.find({ type: NounType.Thing, where: { wave: 1 } })
expect(byWhere.length).toBe(2) // waves 1,4 of 0..5
await brain.close()
} finally {
MetadataIndexManager.prototype.hydrateAll = origHydrateAll
;(MetadataIndexManager.prototype as any).loadSparseIndex = origLoadSparseIndex
GraphAdjacencyIndex.prototype.init = origGraphInit
}
})
it('(d) warmOnOpen: true runs warm() during init() — observable via the fake provider', async () => {
const brain = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
warmOnOpen: true,
silent: true
})
const provider = new FakeVectorProvider(true)
useFakeVectorProvider(brain, provider)
// No explicit brain.warm() call — warmOnOpen must have run it as part of init().
await brain.init()
expect(provider.warmCalls).toBe(1)
await brain.close()
})
it('warmOnOpen defaults to false — init() does NOT run warm() unless opted in', async () => {
const brain = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
silent: true
})
const provider = new FakeVectorProvider(true)
useFakeVectorProvider(brain, provider)
await brain.init()
expect(provider.warmCalls).toBe(0)
await brain.close()
})
it('(e) WarmReport shape: outcome literal + durationMs per surface, plus totalDurationMs', async () => {
const brain = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
silent: true
})
const provider = new FakeVectorProvider(true)
useFakeVectorProvider(brain, provider)
await brain.init()
await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) })
const report = await brain.warm()
for (const surface of ['vector', 'metadata', 'graph'] as const) {
expect(['warmed', 'probed', 'unavailable']).toContain(report[surface].outcome)
expect(typeof report[surface].durationMs).toBe('number')
expect(report[surface].durationMs).toBeGreaterThanOrEqual(0)
}
expect(typeof report.totalDurationMs).toBe('number')
expect(report.totalDurationMs).toBeGreaterThanOrEqual(0)
await brain.close()
})
})

View file

@ -0,0 +1,149 @@
/**
* @module tests/unit/transaction/budget-commit-completed-work
* @description Regression coverage for the "commit-completed-work" transaction
* budget semantics: the budget gates STARTING the next operation; it never
* converts already-completed work into a rollback. Concretely:
*
* - A single-op transaction can never time out post-hoc its one operation
* either runs (and the transaction commits, however long it took) or it
* never gets to start (which cannot happen: there is no operation before it
* to have overrun the budget).
* - A multi-op transaction whose op `i` overruns the budget stops op `i+1`
* from starting: everything applied so far rolls back atomically and a
* `TransactionTimeoutError` is thrown the mid-batch zero-loss guarantee is
* unchanged.
* - `transactTimeoutBudget`'s scaling floor (`max(floorMs, opCount * 2000)`)
* is overridable per call, the seam `BrainyConfig.transactionBudgetFloorMs`
* feeds at the brainy.ts read sites.
*
* Uses a real class implementing the `Operation` interface (not an anonymous
* object literal, not a mock of Transaction internals) so the exercised path
* is exactly what a real caller's operation looks like.
*/
import { describe, it, expect } from 'vitest'
import { Transaction, transactTimeoutBudget } from '../../../src/transaction/Transaction.js'
import type { Operation, RollbackAction } from '../../../src/transaction/types.js'
import { TransactionTimeoutError } from '../../../src/transaction/errors.js'
const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms))
/**
* A real `Operation` implementation that sleeps for `delayMs` before applying
* a write to `log`, and returns a rollback action that removes it again. Used
* to deterministically make one operation "slow" relative to a transaction's
* budget without touching any Transaction internals.
*/
class SlowOperation implements Operation {
readonly name: string
executed = false
rolledBack = false
constructor(
private readonly log: string[],
label: string,
private readonly delayMs: number
) {
this.name = label
}
async execute(): Promise<RollbackAction | undefined> {
this.executed = true
if (this.delayMs > 0) await sleep(this.delayMs)
this.log.push(this.name)
return async () => {
this.rolledBack = true
const idx = this.log.indexOf(this.name)
if (idx >= 0) this.log.splice(idx, 1)
}
}
}
describe('Transaction budget — commit-completed-work semantics', () => {
it('(a) single-op transaction whose op overruns the budget COMMITS — no rollback, no throw', async () => {
const log: string[] = []
const op = new SlowOperation(log, 'slow-single-op', 40)
// Budget (5ms) is far smaller than the op's 40ms — under the OLD
// (post-completion-check) semantics this would have thrown and rolled
// back after the op finished. Under the new semantics there is no
// operation after it to gate, so it commits.
const tx = new Transaction({ timeout: 5 })
tx.addOperation(op)
await expect(tx.execute()).resolves.toBeUndefined()
expect(tx.getState()).toBe('committed')
expect(op.executed).toBe(true)
expect(op.rolledBack).toBe(false)
expect(log).toEqual(['slow-single-op'])
})
it('(b) two-op transaction: op0 overruns the budget → op1 never starts, op0 rolls back, throws TransactionTimeoutError', async () => {
const log: string[] = []
const op0 = new SlowOperation(log, 'op0-overruns', 40)
const op1 = new SlowOperation(log, 'op1-never-starts', 0)
const tx = new Transaction({ timeout: 5 })
tx.addOperation(op0)
tx.addOperation(op1)
const error = await tx.execute().catch((e) => e)
expect(error).toBeInstanceOf(TransactionTimeoutError)
expect(op0.executed).toBe(true)
expect(op0.rolledBack).toBe(true)
expect(op1.executed).toBe(false)
expect(op1.rolledBack).toBe(false)
expect(log).toEqual([]) // op0's write was undone; op1 never wrote
expect(tx.getState()).toBe('rolled_back')
})
it('a single-op transaction never checks the budget before its only operation starts', async () => {
// Budget of 0ms: under the old "check before every op including 0" code
// this could theoretically trip before op 0 even started (if any
// measurable time elapsed between startTime capture and the check). The
// documented contract is stronger: operation 0 ALWAYS gets to start.
const log: string[] = []
const op = new SlowOperation(log, 'only-op', 5)
const tx = new Transaction({ timeout: 0 })
tx.addOperation(op)
await expect(tx.execute()).resolves.toBeUndefined()
expect(tx.getState()).toBe('committed')
expect(op.executed).toBe(true)
})
it('(c) transactTimeoutBudget: an explicit floorMs overrides the 30s default floor', () => {
// Below the per-op scaling term, a raised floor wins.
expect(transactTimeoutBudget(1, undefined, 5_000)).toBe(5_000) // 1 op × 2000 = 2000 < 5000 floor
expect(transactTimeoutBudget(1)).toBe(30_000) // unchanged default when floorMs omitted
// Above the floor, opCount * 2000 still wins over a smaller floor.
expect(transactTimeoutBudget(10, undefined, 5_000)).toBe(20_000) // 10 × 2000 = 20000 > 5000 floor
// A full `override` always wins, regardless of floorMs.
expect(transactTimeoutBudget(10, 999, 5_000)).toBe(999)
})
it('(c) a configured floor changes real Transaction behavior end-to-end via TransactionManager-style options', async () => {
// Simulates how brainy.ts wires `this.config.transactionBudgetFloorMs`
// into `transactTimeoutBudget()`: opCount 0 isolates the floor term
// (0 × 2000 = 0), so a lowered floor makes a normally-generous budget
// fail-fast for a real 2-op Transaction.
const lowFloorBudget = transactTimeoutBudget(0, undefined, 10) // 10ms floor
expect(lowFloorBudget).toBe(10)
const log: string[] = []
const op0 = new SlowOperation(log, 'op0', 30)
const op1 = new SlowOperation(log, 'op1-gated-by-lowered-floor', 0)
const tx = new Transaction({ timeout: lowFloorBudget })
tx.addOperation(op0)
tx.addOperation(op1)
const error = await tx.execute().catch((e) => e)
expect(error).toBeInstanceOf(TransactionTimeoutError)
expect(op1.executed).toBe(false)
})
})

View file

@ -0,0 +1,176 @@
/**
* @module tests/unit/transaction/vectorIndexOperations-rename
* @description Coverage for the backend-neutral vector-index transaction
* operations (formerly `AddToHNSWOperation` / `RemoveFromHNSWOperation`).
* These op-name strings surface directly in consumer-visible transaction
* journals and timings a fossil "HNSW" name misdirects an operator running
* a native (non-HNSW) vector provider. Verifies:
* 1. rollback wiring stayed byte-identical under the rename,
* 2. the emitted `name` stamps the active backend `'hnsw-js'` for
* Brainy's own JS index, a provider's own required `name` when it
* self-identifies, and the tolerant-loud `'unknown-provider'` literal
* (plus one console.warn) when a runtime instance lacks `name`
* altogether (an older native provider compiled against the previous,
* optional `providerId` contract) never a silently-wrong guess.
*/
import { describe, it, expect, vi, afterEach } from 'vitest'
import {
AddToVectorIndexOperation,
RemoveFromVectorIndexOperation,
BatchAddToVectorIndexOperation
} from '../../../src/transaction/operations/IndexOperations.js'
import type { VectorIndexProvider } from '../../../src/plugin.js'
import type { VectorDocument, Vector } from '../../../src/coreTypes.js'
import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js'
/**
* A minimal, real (not mocked) VectorIndexProvider implementation backed by
* a plain Map exercises the exact contract the operations call through,
* without any Brainy internals.
*/
class FakeVectorProvider implements VectorIndexProvider {
readonly items = new Map<string, Vector>()
constructor(readonly name: string) {}
async addItem(item: VectorDocument): Promise<string> {
this.items.set(item.id, item.vector)
return item.id
}
async removeItem(id: string): Promise<boolean> {
return this.items.delete(id)
}
async getItem(id: string): Promise<VectorDocument | undefined> {
const vector = this.items.get(id)
return vector ? { id, vector } : undefined
}
async search(): Promise<Array<[string, number]>> {
return []
}
size(): number {
return this.items.size
}
clear(): void {
this.items.clear()
}
async rebuild(): Promise<void> {}
async flush(): Promise<number> {
return 0
}
getPersistMode(): 'immediate' | 'deferred' {
return 'immediate'
}
}
describe('Vector index transaction operations — backend-neutral rename', () => {
describe('rollback wiring (byte-identical behavior under the rename)', () => {
it('AddToVectorIndexOperation rollback removes a newly-added item', async () => {
const provider = new FakeVectorProvider('fake-provider')
const op = new AddToVectorIndexOperation(provider, 'id-1', [1, 2, 3])
const rollback = await op.execute()
expect(provider.items.has('id-1')).toBe(true)
await rollback()
expect(provider.items.has('id-1')).toBe(false)
})
it('AddToVectorIndexOperation rollback is a no-op when the item pre-existed (update semantics)', async () => {
const provider = new FakeVectorProvider('fake-provider')
await provider.addItem({ id: 'id-1', vector: [9, 9, 9] })
const op = new AddToVectorIndexOperation(provider, 'id-1', [1, 2, 3])
const rollback = await op.execute()
expect(provider.items.get('id-1')).toEqual([1, 2, 3])
await rollback()
// Pre-existing item is NOT removed by rollback — it stays (the update
// itself is not undone by this op; that's RemoveFromVectorIndexOperation's job).
expect(provider.items.has('id-1')).toBe(true)
})
it('RemoveFromVectorIndexOperation rollback re-adds the item with its original vector', async () => {
const provider = new FakeVectorProvider('fake-provider')
await provider.addItem({ id: 'id-1', vector: [4, 5, 6] })
const op = new RemoveFromVectorIndexOperation(provider, 'id-1', [4, 5, 6])
const rollback = await op.execute()
expect(provider.items.has('id-1')).toBe(false)
await rollback()
expect(provider.items.get('id-1')).toEqual([4, 5, 6])
})
it('BatchAddToVectorIndexOperation rolls back every item in reverse order on undo', async () => {
const provider = new FakeVectorProvider('fake-provider')
const op = new BatchAddToVectorIndexOperation(provider, [
{ id: 'a', vector: [1] },
{ id: 'b', vector: [2] },
{ id: 'c', vector: [3] }
])
const rollback = await op.execute()
expect([...provider.items.keys()].sort()).toEqual(['a', 'b', 'c'])
await rollback()
expect(provider.items.size).toBe(0)
})
})
describe('backend stamping in the emitted op name', () => {
afterEach(() => {
vi.restoreAllMocks()
})
it('stamps the real JS HNSW index as hnsw-js (self-identifying name)', () => {
const index = new JsHnswVectorIndex()
const addOp = new AddToVectorIndexOperation(index, 'id-1', [1, 2, 3])
const removeOp = new RemoveFromVectorIndexOperation(index, 'id-1', [1, 2, 3])
expect(addOp.name).toBe('AddToVectorIndex(hnsw-js)')
expect(removeOp.name).toBe('RemoveFromVectorIndex(hnsw-js)')
})
it('stamps a self-identifying provider\'s own name, never "HNSW"', () => {
const provider = new FakeVectorProvider('acme-diskann')
const addOp = new AddToVectorIndexOperation(provider, 'id-1', [1, 2, 3])
const removeOp = new RemoveFromVectorIndexOperation(provider, 'id-1', [1, 2, 3])
const batchOp = new BatchAddToVectorIndexOperation(provider, [{ id: 'a', vector: [1] }])
expect(addOp.name).toBe('AddToVectorIndex(acme-diskann)')
expect(removeOp.name).toBe('RemoveFromVectorIndex(acme-diskann)')
expect(batchOp.name).toBe('BatchAddToVectorIndex(acme-diskann)')
expect(addOp.name).not.toContain('HNSW')
expect(removeOp.name).not.toContain('HNSW')
})
it('tolerant-loud: stamps "unknown-provider" and warns exactly once when a runtime provider lacks the required `name` (an older native provider compiled against the previous optional `providerId` contract)', () => {
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
// Simulate a native provider compiled before `name` was required — the
// TypeScript contract requires `name`, but `as unknown as` mirrors what
// actually reaches this code at runtime from an un-rebuilt native addon.
const legacyProvider = {
addItem: async (item: VectorDocument) => item.id,
removeItem: async () => true,
search: async () => [],
size: () => 0,
clear: () => {},
rebuild: async () => {},
flush: async () => 0,
getPersistMode: () => 'immediate' as const
} as unknown as VectorIndexProvider
const addOp = new AddToVectorIndexOperation(legacyProvider, 'id-1', [1, 2, 3])
const removeOp = new RemoveFromVectorIndexOperation(legacyProvider, 'id-1', [1, 2, 3])
expect(addOp.name).toBe('AddToVectorIndex(unknown-provider)')
expect(removeOp.name).toBe('RemoveFromVectorIndex(unknown-provider)')
// Never silently falls back to the JS engine's name for a provider it
// knows nothing about — that's the exact fossil-naming bug being fixed.
expect(addOp.name).not.toContain('hnsw-js')
// Loud, not silent — but exactly once per provider instance, not once
// per op stamped against it.
expect(warnSpy).toHaveBeenCalledTimes(1)
expect(warnSpy.mock.calls[0][0]).toContain('name')
})
})
})