soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

Merge branch 'release/8.10.0'

Browse source
This commit is contained in:
David Snelling 2026-07-23 10:50:39 -07:00
commit d918c060f4
21 changed files with 1283 additions and 346 deletions
Download patch file Download diff file Expand all files Collapse all files

205
src/brainy.ts
View file

@ -63,7 +63,9 @@ import type {
PathOptions,
MetadataIndexProvider,
OpaqueIdSet,
AtGenerationVectors
AtGenerationVectors,
VectorIndexProvider,
GraphIndexProvider
} from './plugin.js'
import type {
BrainyPlugin,
@ -88,12 +90,12 @@ import { findCallerLocation } from './utils/callerLocation.js'
import {
SaveNounMetadataOperation,
SaveNounOperation,
AddToHNSWOperation,
AddToVectorIndexOperation,
AddToMetadataIndexOperation,
SaveVerbMetadataOperation,
SaveVerbOperation,
AddToGraphIndexOperation,
RemoveFromHNSWOperation,
RemoveFromVectorIndexOperation,
RemoveFromMetadataIndexOperation,
RemoveFromGraphIndexOperation,
UpdateNounMetadataOperation,
@ -266,6 +268,7 @@ type ResolvedBrainyConfig = Required<
| 'retention'
| 'eagerEmbeddings'
| 'migrationWaitTimeoutMs'
| 'transactionBudgetFloorMs'
>
> &
Pick<
@ -278,6 +281,7 @@ type ResolvedBrainyConfig = Required<
| 'retention'
| 'eagerEmbeddings'
| 'migrationWaitTimeoutMs'
| 'transactionBudgetFloorMs'
>
/**
@ -388,6 +392,38 @@ class InsertPreconditionExistsSignal extends Error {
*/
export type IndexFamily = 'vector' | 'metadata' | 'graph'
/**
* @description Honest per-surface outcome for {@link Brainy.warm}. Literal
* meanings — never conflate the first two:
* - `'warmed'` — the provider's own `warm?()` hook ran (vector/graph), or the
* surface's full-hydration seam loaded EVERY shard/field/segment from
* storage (metadata; graph's fallback path). The surface is genuinely at
* steady-state cost for the next operation.
* - `'probed'` — no `warm?()` hook was available, so a best-effort read
* (e.g. one `search()` call) faulted in *some* backing storage as a side
* effect. Real work happened, but it is NOT the same guarantee as
* `'warmed'` — never reported as `'warmed'`.
* - `'unavailable'` — nothing ran: no hook, no hydration seam, and (for the
* vector probe fallback) nothing to probe (an empty index or unknown
* vector dimension). The surface is unchanged by this `warm()` call.
*/
export type WarmOutcome = 'warmed' | 'probed' | 'unavailable'
/**
* @description Result of {@link Brainy.warm}: one {@link WarmOutcome} +
* elapsed time per index surface, plus the total wall-clock time for the
* whole call. `durationMs` is measured around exactly the work described by
* that surface's `outcome` (e.g. the vector entry's `durationMs` times the
* provider `warm()` call OR the probe `search()` call — whichever ran).
*/
export interface WarmReport {
vector: { outcome: WarmOutcome; durationMs: number }
metadata: { outcome: WarmOutcome; durationMs: number }
graph: { outcome: WarmOutcome; durationMs: number }
/** Total wall-clock time for the whole `warm()` call (all three surfaces). */
totalDurationMs: number
}
/**
* How long a failed aggregation-backfill walk suppresses fresh walk attempts.
* Within the window, queries rethrow the recorded failure instantly (loud,
@ -1382,6 +1418,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
})
}
// Eager index warm (operator opt-in, `warmOnOpen: true`). Runs AFTER
// every step above — index construction, crash recovery, migrations,
// VFS bootstrap — never in place of any of it, so `warm()` always
// operates on a fully-initialized brain. Blocking BY DESIGN: the
// operator traded a longer startup for a first-request that runs at
// steady-state cost instead of paying demand-load latency on the
// critical path. See the `warmOnOpen` JSDoc in brainy.types.ts.
if (this.config.warmOnOpen) {
await this.warm()
}
// Resolve ready Promise - consumers awaiting brain.ready will now proceed
if (this._readyResolve) {
this._readyResolve()
@ -1780,7 +1827,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
await this.generationStore.runWithoutGeneration(() =>
this.transactionManager.executeTransaction(run, {
timeout: transactTimeoutBudget(
(touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0)
(touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0),
undefined,
this.config.transactionBudgetFloorMs
)
})
)
@ -1797,7 +1846,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
execute: () =>
this.transactionManager.executeTransaction(run, {
timeout: transactTimeoutBudget(
(touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0)
(touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0),
undefined,
this.config.transactionBudgetFloorMs
)
})
})
@ -2109,7 +2160,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Operation 3: Add to HNSW index (after entity saved)
tx.addOperation(
new AddToHNSWOperation(this.index, id, vector)
new AddToVectorIndexOperation(this.index, id, vector)
)
// Operation 4: Add to metadata index
@ -3098,10 +3149,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Operation 3-4: Update HNSW index (remove and re-add if reindexing needed)
if (needsReindexing) {
tx.addOperation(
new RemoveFromHNSWOperation(this.index, params.id, existing.vector)
new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector)
)
tx.addOperation(
new AddToHNSWOperation(this.index, params.id, vector)
new AddToVectorIndexOperation(this.index, params.id, vector)
)
}
@ -3217,7 +3268,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Operation 1: Remove from vector index
if (noun) {
tx.addOperation(
new RemoveFromHNSWOperation(this.index, id, noun.vector)
new RemoveFromVectorIndexOperation(this.index, id, noun.vector)
)
}
@ -7025,7 +7076,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Add delete operations to transaction
if (noun) {
tx.addOperation(
new RemoveFromHNSWOperation(this.index, id, noun.vector)
new RemoveFromVectorIndexOperation(this.index, id, noun.vector)
)
}
@ -7888,7 +7939,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Budget scales with the batch (or the caller's explicit
// timeoutMs): a flat 30s cap silently limited honest bulk work
// to ~15 ops on network disks (~2s/op measured in the field).
{ timeout: transactTimeoutBudget(plan.operations.length, options?.timeoutMs) }
{
timeout: transactTimeoutBudget(
plan.operations.length,
options?.timeoutMs,
this.config.transactionBudgetFloorMs
)
}
)
}
}))
@ -9311,7 +9368,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
plan.operations.push(
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew),
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
new AddToHNSWOperation(this.index, id, vector),
new AddToVectorIndexOperation(this.index, id, vector),
new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing)
)
plan.touchedNouns.push(id)
@ -9479,8 +9536,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
)
if (needsReindexing) {
plan.operations.push(
new RemoveFromHNSWOperation(this.index, params.id, existing.vector),
new AddToHNSWOperation(this.index, params.id, vector)
new RemoveFromVectorIndexOperation(this.index, params.id, existing.vector),
new AddToVectorIndexOperation(this.index, params.id, vector)
)
}
plan.operations.push(
@ -9565,7 +9622,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
if (noun) {
plan.operations.push(new RemoveFromHNSWOperation(this.index, id, noun.vector))
plan.operations.push(new RemoveFromVectorIndexOperation(this.index, id, noun.vector))
}
if (metadata) {
plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata))
@ -14280,6 +14337,118 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return this.embedder(textToEmbed)
}
/**
* Eagerly load/fault-in the vector index, metadata index, and graph
* adjacency so the FIRST real operation after this call runs at
* steady-state cost — no page-cache-miss / demand-load latency on the
* critical path. Complements {@link warmupEmbeddings} (which warms the
* embedding engine, not the storage indexes).
*
* Sequence per surface:
* - **Vector**: calls the provider's own `warm?()` when the active
* `'vector'` provider implements it (`'warmed'`). Otherwise runs a
* best-effort probe — one `search()` with a deterministic unit vector of
* the index's known dimension, `k = min(100, size())` — which faults in
* *some* backing storage as a side effect but is reported honestly as
* `'probed'`, never `'warmed'`. An empty index or unknown dimension has
* nothing to probe (`'unavailable'`).
* - **Metadata**: full hydration — every persisted field's sparse index is
* loaded from storage (`MetadataIndexManager.hydrateAll()`), not just the
* heuristic common-fields subset `init()` warms.
* - **Graph**: calls the provider's own `warm?()` when the active graph
* provider implements it; otherwise re-runs its existing eager cold-load
* `init()` seam (idempotent — the JS adjacency index's `init()` already
* loads the full LSM manifests + SSTables, so re-invoking it here is a
* genuine full hydration, not a stat call).
*
* A single surface's `'unavailable'`/probe outcome never fails the whole
* call — `warm()` is a best-effort readiness step, not a correctness gate;
* queries still demand-load normally regardless of what `warm()` achieved.
*
* @returns A {@link WarmReport}: honest per-surface outcome + timing.
* @example
* ```typescript
* const brain = new Brainy({ storage: { path: '/data' } })
* await brain.init()
* const report = await brain.warm() // blocking, explicit control over timing
* console.log(report.vector.outcome, report.vector.durationMs)
*
* // Or opt into the same work automatically during init():
* const eager = new Brainy({ storage: { path: '/data' }, warmOnOpen: true })
* await eager.init() // warm() already ran before this resolves
* ```
*/
async warm(): Promise<WarmReport> {
await this.ensureInitialized({ needs: ['vector', 'metadata', 'graph'] })
const totalStart = Date.now()
// --- Vector ---------------------------------------------------------
const vectorStart = Date.now()
let vectorOutcome: WarmOutcome
const vectorProvider = this.index as VectorIndexProvider & { warm?: () => Promise<void> }
if (typeof vectorProvider.warm === 'function') {
await vectorProvider.warm()
vectorOutcome = 'warmed'
} else {
const size = this.index.size()
const dimension = this.dimensions
if (size > 0 && dimension && dimension > 0) {
// A deterministic unit vector (L2 norm 1) — same probe every call,
// no reliance on stored data shape.
const probeVector = new Array(dimension).fill(1 / Math.sqrt(dimension))
await this.index.search(probeVector, Math.min(100, size))
vectorOutcome = 'probed'
} else {
// Nothing to probe: an empty index, or the vector dimension is not
// yet known (no entity has ever been added).
vectorOutcome = 'unavailable'
}
}
const vectorDurationMs = Date.now() - vectorStart
// --- Metadata --------------------------------------------------------
const metadataStart = Date.now()
let metadataOutcome: WarmOutcome
const metadataWithHydrate = this.metadataIndex as unknown as { hydrateAll?: () => Promise<void> }
if (typeof metadataWithHydrate.hydrateAll === 'function') {
await metadataWithHydrate.hydrateAll()
metadataOutcome = 'warmed'
} else {
// No hydration seam on this metadata provider — nothing to run.
metadataOutcome = 'unavailable'
}
const metadataDurationMs = Date.now() - metadataStart
// --- Graph -------------------------------------------------------------
const graphStart = Date.now()
let graphOutcome: WarmOutcome
const graphProvider = this.graphIndex as GraphIndexProvider & {
warm?: () => Promise<void>
init?: () => Promise<void>
}
if (typeof graphProvider.warm === 'function') {
await graphProvider.warm()
graphOutcome = 'warmed'
} else if (typeof graphProvider.init === 'function') {
// Existing eager cold-load seam (readiness contract): idempotent, and
// for the JS adjacency index it's already a genuine full load of the
// LSM manifests + SSTables — a real hydration, not a stat call.
await graphProvider.init()
graphOutcome = 'warmed'
} else {
graphOutcome = 'unavailable'
}
const graphDurationMs = Date.now() - graphStart
return {
vector: { outcome: vectorOutcome, durationMs: vectorDurationMs },
metadata: { outcome: metadataOutcome, durationMs: metadataDurationMs },
graph: { outcome: graphOutcome, durationMs: graphDurationMs },
totalDurationMs: Date.now() - totalStart
}
}
/**
* Explicitly warm up the embedding engine
*
@ -14798,6 +14967,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Migration LOCK wait budget — left undefined when omitted so
// awaitMigrationLock() applies its 30 s default at the read site.
migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined,
// Apply-phase transaction budget floor — left undefined when omitted so
// transactTimeoutBudget() applies its 30 s default at the read site.
transactionBudgetFloorMs: config?.transactionBudgetFloorMs ?? undefined,
// Pre-upgrade backup — default-on; opt out with `migrationBackup: false`.
migrationBackup: config?.migrationBackup ?? true,
// Vector index configuration (8.0) — algorithm-neutral surface with
@ -14813,6 +14985,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// is the active one on a non-reader writer outside unit tests). Explicit
// true/false always wins. See the eagerEmbeddings JSDoc in brainy.types.ts.
eagerEmbeddings: config?.eagerEmbeddings ?? undefined,
// Eager index warm at init() — default off (lazy demand-load), the
// pre-8.9 behavior. See the warmOnOpen JSDoc in brainy.types.ts.
warmOnOpen: config?.warmOnOpen ?? false,
// Plugin configuration - undefined = auto-detect
plugins: config?.plugins ?? undefined,
// Integration Hub - undefined/false = disabled