feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate
A production restart storm measured 90,017ms for a single brain init vs 1,117ms quiet (~80x contention multiplier), traced to performInit() eagerly awaiting the process-global WASM embedding engine before the VFS root even existed. Every writer's open() queued on the one throttled model compile (90-140s on throttled CPUs). - VirtualFileSystem.doInitializeRoot() no longer embeds '/'. The root is system-tier plumbing nothing ever searches; when the default WASM engine is active it now gets an explicit all-zero placeholder vector (cosineDistance returns max distance for a zero vector, so it never ranks ahead of real content). deferEmbedding was considered and rejected: its landing path kicks the embed worker synchronously right after commit, which would still force the cold compile within milliseconds — just off the awaited path, not avoided. A registered native 'embeddings' provider (no cold-start cost, possibly a different dimension) still embeds the root for real, via the new Brainy.usesDefaultWasmEmbedder() seam. - performInit()'s eager-embedding step now only STARTS the WASM engine warm in the background instead of awaiting it inline. embed()/embeddingManager already serialize concurrent callers on one shared init promise, so the first real embed() converges correctly either way; a failed warm narrates loudly instead of surfacing as a silent latency spike or an unhandled rejection. eagerEmbeddings: false still means no warm at all. - FileSystemStorage.init() batches its ~8 independent bootstrap mkdirs (each creates its own full subtree via recursive:true, so none depend on the others existing) into one Promise.all. The restore-completion step and initializeCounts() stay strictly sequential — they have real order dependencies on rootDir and systemDir respectively. - performInit() now times five phases (storage init / generation-store open+fold / index init+gate / VFS bootstrap / embedding-warm-started) and logs one warning with the per-phase breakdown when total open exceeds 2000ms; silent otherwise.
This commit is contained in:
parent
f8f64780b1
commit
96624f408c
5 changed files with 446 additions and 53 deletions
146
src/brainy.ts
146
src/brainy.ts
|
|
@ -742,6 +742,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
private _pendingEmbedIds = new Set<string>()
|
||||
private _embedWorkerFlight: Promise<void> | null = null
|
||||
|
||||
// OPEN-PATH FIX: the background embedding-engine warm kicked off (never
|
||||
// awaited) by `performInit()` when `eagerEmbeddings` resolves true. Stored
|
||||
// for observability only — `embed()`/`embeddingManager.embed()` already
|
||||
// await the engine's OWN singleton init promise internally, so nothing
|
||||
// needs to explicitly await this field for correctness. Never rejects on
|
||||
// its own: a `.catch` narrates the failure and swallows it so a failed
|
||||
// warm never surfaces as an unhandled rejection.
|
||||
private _embeddingWarmPromise: Promise<void> | null = null
|
||||
|
||||
/** The stored log-authority switch, read once at open (default: tree). */
|
||||
private _logAuthority: LogAuthorityRecord = { authority: 'tree' }
|
||||
// A failed walk latches its error: retries within the cooldown rethrow it
|
||||
|
|
@ -1079,6 +1088,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging
|
||||
}
|
||||
|
||||
// OPEN-PATH NARRATION: lightweight phase timing across the five
|
||||
// named stretches of init — storage init / generation-store open+fold /
|
||||
// index init+gate / VFS bootstrap / embedding-warm-started. Each
|
||||
// `markPhase()` call records elapsed ms SINCE THE PREVIOUS checkpoint,
|
||||
// so the buckets always sum to the pre-integration/warmOnOpen total.
|
||||
// Silent under 2s; one `prodLog.warn` line naming every phase's ms
|
||||
// above it, so the operator's next restart storm names its own slow
|
||||
// phase instead of re-deriving it from a stack of raw timestamps.
|
||||
const initStart = Date.now()
|
||||
let lastPhaseCheckpoint = initStart
|
||||
const phaseTimingsMs: Record<string, number> = {}
|
||||
const markPhase = (name: string): void => {
|
||||
const now = Date.now()
|
||||
phaseTimingsMs[name] = now - lastPhaseCheckpoint
|
||||
lastPhaseCheckpoint = now
|
||||
}
|
||||
|
||||
try {
|
||||
// Auto-detect and activate plugins BEFORE storage setup
|
||||
// so plugin-provided storage factories (e.g., filesystem override from cor) are available
|
||||
|
|
@ -1150,6 +1176,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
// PHASE 1 of 5 — "storage init": plugin/legacy-layout bootstrap,
|
||||
// storage adapter construction+init, the OS-limit check, and the
|
||||
// writer-lock claim, all folded into one bucket (everything above this
|
||||
// line since performInit started).
|
||||
markPhase('storage-init')
|
||||
|
||||
// 8.0 generational MVCC: open the record layer BEFORE any index is
|
||||
// created or loaded. Crash recovery may rewrite canonical entity files
|
||||
// (restoring before-images of an uncommitted transaction), and every
|
||||
|
|
@ -1224,6 +1256,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
await this.createMigrationBackupIfNeeded()
|
||||
}
|
||||
|
||||
// PHASE 2 of 5 — "generation-store open+fold": GenerationStore
|
||||
// construction+open (crash-recovery replay/rollback fold), the
|
||||
// derived-family registration, the fact-scan seam, the entity-tree
|
||||
// stamp check, the brain-format handshake, and the pre-upgrade backup.
|
||||
markPhase('generation-store-open-fold')
|
||||
|
||||
// Provider: embeddings (reassign embedder if plugin provides one)
|
||||
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
|
||||
if (embeddingProvider) {
|
||||
|
|
@ -1461,6 +1499,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Check for pending data migrations
|
||||
await this.checkMigrations()
|
||||
|
||||
// PHASE 3 of 5 — "index init+gate": provider wiring (embeddings,
|
||||
// cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph
|
||||
// index construction, the eager cold-load, id-resolver + connections-
|
||||
// codec wiring, crash-recovery index rebuild, the replay-gap check,
|
||||
// legacy VFS blob adoption, blob-history backfill, and the
|
||||
// rebuildIndexesIfNeeded() gate + migration check.
|
||||
markPhase('index-init-gate')
|
||||
|
||||
// Register shutdown hooks for graceful count flushing (once globally)
|
||||
if (!Brainy.shutdownHooksRegisteredGlobally) {
|
||||
this.registerShutdownHooks()
|
||||
|
|
@ -1612,15 +1658,40 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
// Eager embedding initialization.
|
||||
// PHASE 4 of 5 — "VFS bootstrap": shutdown-hook registration, blob
|
||||
// storage init, the provider-summary log, flipping `initialized`,
|
||||
// the migration-lock wait, VFS construction+init, flipping generation
|
||||
// stamping active, the log-authority adopt/oracle check, and
|
||||
// pending-embed crash recovery.
|
||||
markPhase('vfs-bootstrap')
|
||||
|
||||
// Eager embedding initialization — BACKGROUND WARM (open-path fix).
|
||||
//
|
||||
// Adaptive default (8.0): the WASM embedding engine eagerly initializes
|
||||
// Adaptive default (8.0): the WASM embedding engine eagerly WARMS
|
||||
// during init() WHENEVER it is the active embedder — i.e. no native
|
||||
// 'embeddings' provider has taken over — and the instance is a writer
|
||||
// (not reader-mode) outside of unit tests. The WASM module (≈93MB with
|
||||
// the embedded model) takes 90-140s to compile on throttled CPUs; paying
|
||||
// that during boot rather than on the first embed()-driven call is the
|
||||
// right default for the overwhelmingly common single-process server.
|
||||
// the embedded model) takes 90-140s to compile on throttled CPUs.
|
||||
//
|
||||
// Historically this AWAITED `embeddingManager.init()` INLINE, so every
|
||||
// writer's open() blocked on the compile — N concurrent opens all
|
||||
// queued on the ONE process-global singleton (an ~80x contention
|
||||
// multiplier measured in a production restart storm: 90,017ms busy vs
|
||||
// 1,117ms quiet). The engine only needs to be ready before the FIRST
|
||||
// REAL embed() call, not before init() returns, so this now only
|
||||
// STARTS the warm and moves on — init() never waits for it.
|
||||
//
|
||||
// No double-await needed for correctness: `this.embed()` (~line 15420)
|
||||
// delegates to `this.embedder`, which for the default engine is
|
||||
// `embeddingManager.getEmbeddingFunction()` → `embeddingManager.embed()`
|
||||
// (src/embeddings/EmbeddingManager.ts). That method calls `await
|
||||
// this.init()` FIRST, and `init()` itself serializes every concurrent
|
||||
// caller onto ONE shared `globalInitPromise` — so the first real
|
||||
// embed() automatically waits for whichever finishes first: this
|
||||
// background warm (if still running) or a fresh init() (if the warm
|
||||
// hasn't reached this code yet, e.g. `eagerEmbeddings: false`).
|
||||
// Verified by reading both call sites; `_embeddingWarmPromise` below
|
||||
// is stored for observability only, never re-awaited by embed().
|
||||
//
|
||||
// Skipped automatically when:
|
||||
// - a native 'embeddings' provider is registered (it owns embeddings;
|
||||
|
|
@ -1628,8 +1699,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// - reader-mode (readers don't embed — they query existing vectors),
|
||||
// - unit-test mode (tests must stay fast and use the mock embedder).
|
||||
//
|
||||
// `eagerEmbeddings: false` is the explicit override to force lazy init
|
||||
// (first-embed) even when this instance is the active embedder.
|
||||
// `eagerEmbeddings: false` keeps meaning "no warm at all" — fully lazy,
|
||||
// the first embed() call pays the full cost inline, same as before.
|
||||
const isUnitTestMode = isDeterministicEmbedMode()
|
||||
const eager = this.config.eagerEmbeddings ?? true
|
||||
if (
|
||||
|
|
@ -1638,9 +1709,45 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this.config.mode !== 'reader' &&
|
||||
!isUnitTestMode
|
||||
) {
|
||||
console.log('Eager embedding initialization enabled...')
|
||||
await embeddingManager.init()
|
||||
console.log('Embedding engine ready')
|
||||
const warmStart = Date.now()
|
||||
console.log('Background embedding-engine warm started (init() does not wait for it)...')
|
||||
this._embeddingWarmPromise = embeddingManager
|
||||
.init()
|
||||
.then(() => {
|
||||
prodLog.info(
|
||||
`[Brainy] background embedding-engine warm complete in ${Date.now() - warmStart}ms`
|
||||
)
|
||||
})
|
||||
.catch((err) => {
|
||||
// Loud, never silent: a warm that fails to compile must be
|
||||
// heard NOW, not discovered as a mystery latency spike on
|
||||
// whichever request happens to trigger the first real embed().
|
||||
// That first embed() call still retries init() itself (the
|
||||
// singleton promise contract above) and surfaces its own typed
|
||||
// error to its caller — this is the immediate, background echo.
|
||||
prodLog.warn(
|
||||
`[Brainy] background embedding-engine warm FAILED: ` +
|
||||
`${(err as Error).message} — the first embed() call will retry ` +
|
||||
`initialization and surface the error there`
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
// PHASE 5 of 5 — "embedding-warm-started": just the synchronous cost
|
||||
// of kicking off the background warm above (the warm's own compile
|
||||
// time is NOT included — that's the whole point of backgrounding it).
|
||||
markPhase('embedding-warm-started')
|
||||
{
|
||||
const totalOpenMs = Date.now() - initStart
|
||||
if (totalOpenMs > 2000) {
|
||||
const phaseList = Object.entries(phaseTimingsMs)
|
||||
.map(([name, ms]) => `${name}=${ms}ms`)
|
||||
.join(', ')
|
||||
prodLog.warn(
|
||||
`[Brainy] slow open: ${totalOpenMs}ms total (${phaseList}) — see the ` +
|
||||
`phase breakdown above to find which one to investigate first`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Integration Hub initialization
|
||||
|
|
@ -15695,6 +15802,25 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return embeddingManager.isInitialized()
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the process-global WASM embedding engine (all-MiniLM-L6-v2,
|
||||
* fixed 384-dim output, ≈93MB with the bundled model, 90-140s cold compile
|
||||
* on throttled CPUs) is this instance's active embedder — `false` when a
|
||||
* plugin has replaced it via the `'embeddings'` provider key. A native
|
||||
* provider has no such cold-start cost and may use a different output
|
||||
* dimension, so it is never worth avoiding.
|
||||
*
|
||||
* Used by init-path bootstrap writes (the VFS root — see
|
||||
* `VirtualFileSystem.doInitializeRoot()`) to decide whether embedding a
|
||||
* value during `init()` risks paying the WASM engine's cold compile.
|
||||
*
|
||||
* @returns true when the default WASM engine is active (no native
|
||||
* `'embeddings'` provider registered).
|
||||
*/
|
||||
usesDefaultWasmEmbedder(): boolean {
|
||||
return !this.pluginRegistry.hasProvider('embeddings')
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup embedder
|
||||
*/
|
||||
|
|
|
|||
Reference in a new issue