feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate
A production restart storm measured 90,017ms for a single brain init vs 1,117ms quiet (~80x contention multiplier), traced to performInit() eagerly awaiting the process-global WASM embedding engine before the VFS root even existed. Every writer's open() queued on the one throttled model compile (90-140s on throttled CPUs). - VirtualFileSystem.doInitializeRoot() no longer embeds '/'. The root is system-tier plumbing nothing ever searches; when the default WASM engine is active it now gets an explicit all-zero placeholder vector (cosineDistance returns max distance for a zero vector, so it never ranks ahead of real content). deferEmbedding was considered and rejected: its landing path kicks the embed worker synchronously right after commit, which would still force the cold compile within milliseconds — just off the awaited path, not avoided. A registered native 'embeddings' provider (no cold-start cost, possibly a different dimension) still embeds the root for real, via the new Brainy.usesDefaultWasmEmbedder() seam. - performInit()'s eager-embedding step now only STARTS the WASM engine warm in the background instead of awaiting it inline. embed()/embeddingManager already serialize concurrent callers on one shared init promise, so the first real embed() converges correctly either way; a failed warm narrates loudly instead of surfacing as a silent latency spike or an unhandled rejection. eagerEmbeddings: false still means no warm at all. - FileSystemStorage.init() batches its ~8 independent bootstrap mkdirs (each creates its own full subtree via recursive:true, so none depend on the others existing) into one Promise.all. The restore-completion step and initializeCounts() stay strictly sequential — they have real order dependencies on rootDir and systemDir respectively. - performInit() now times five phases (storage init / generation-store open+fold / index init+gate / VFS bootstrap / embedding-warm-started) and logs one warning with the per-phase breakdown when total open exceeds 2000ms; silent otherwise.
This commit is contained in:
parent
f8f64780b1
commit
96624f408c
5 changed files with 446 additions and 53 deletions
146
src/brainy.ts
146
src/brainy.ts
|
|
@ -742,6 +742,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
private _pendingEmbedIds = new Set<string>()
|
||||
private _embedWorkerFlight: Promise<void> | null = null
|
||||
|
||||
// OPEN-PATH FIX: the background embedding-engine warm kicked off (never
|
||||
// awaited) by `performInit()` when `eagerEmbeddings` resolves true. Stored
|
||||
// for observability only — `embed()`/`embeddingManager.embed()` already
|
||||
// await the engine's OWN singleton init promise internally, so nothing
|
||||
// needs to explicitly await this field for correctness. Never rejects on
|
||||
// its own: a `.catch` narrates the failure and swallows it so a failed
|
||||
// warm never surfaces as an unhandled rejection.
|
||||
private _embeddingWarmPromise: Promise<void> | null = null
|
||||
|
||||
/** The stored log-authority switch, read once at open (default: tree). */
|
||||
private _logAuthority: LogAuthorityRecord = { authority: 'tree' }
|
||||
// A failed walk latches its error: retries within the cooldown rethrow it
|
||||
|
|
@ -1079,6 +1088,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging
|
||||
}
|
||||
|
||||
// OPEN-PATH NARRATION: lightweight phase timing across the five
|
||||
// named stretches of init — storage init / generation-store open+fold /
|
||||
// index init+gate / VFS bootstrap / embedding-warm-started. Each
|
||||
// `markPhase()` call records elapsed ms SINCE THE PREVIOUS checkpoint,
|
||||
// so the buckets always sum to the pre-integration/warmOnOpen total.
|
||||
// Silent under 2s; one `prodLog.warn` line naming every phase's ms
|
||||
// above it, so the operator's next restart storm names its own slow
|
||||
// phase instead of re-deriving it from a stack of raw timestamps.
|
||||
const initStart = Date.now()
|
||||
let lastPhaseCheckpoint = initStart
|
||||
const phaseTimingsMs: Record<string, number> = {}
|
||||
const markPhase = (name: string): void => {
|
||||
const now = Date.now()
|
||||
phaseTimingsMs[name] = now - lastPhaseCheckpoint
|
||||
lastPhaseCheckpoint = now
|
||||
}
|
||||
|
||||
try {
|
||||
// Auto-detect and activate plugins BEFORE storage setup
|
||||
// so plugin-provided storage factories (e.g., filesystem override from cor) are available
|
||||
|
|
@ -1150,6 +1176,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
// PHASE 1 of 5 — "storage init": plugin/legacy-layout bootstrap,
|
||||
// storage adapter construction+init, the OS-limit check, and the
|
||||
// writer-lock claim, all folded into one bucket (everything above this
|
||||
// line since performInit started).
|
||||
markPhase('storage-init')
|
||||
|
||||
// 8.0 generational MVCC: open the record layer BEFORE any index is
|
||||
// created or loaded. Crash recovery may rewrite canonical entity files
|
||||
// (restoring before-images of an uncommitted transaction), and every
|
||||
|
|
@ -1224,6 +1256,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
await this.createMigrationBackupIfNeeded()
|
||||
}
|
||||
|
||||
// PHASE 2 of 5 — "generation-store open+fold": GenerationStore
|
||||
// construction+open (crash-recovery replay/rollback fold), the
|
||||
// derived-family registration, the fact-scan seam, the entity-tree
|
||||
// stamp check, the brain-format handshake, and the pre-upgrade backup.
|
||||
markPhase('generation-store-open-fold')
|
||||
|
||||
// Provider: embeddings (reassign embedder if plugin provides one)
|
||||
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
|
||||
if (embeddingProvider) {
|
||||
|
|
@ -1461,6 +1499,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Check for pending data migrations
|
||||
await this.checkMigrations()
|
||||
|
||||
// PHASE 3 of 5 — "index init+gate": provider wiring (embeddings,
|
||||
// cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph
|
||||
// index construction, the eager cold-load, id-resolver + connections-
|
||||
// codec wiring, crash-recovery index rebuild, the replay-gap check,
|
||||
// legacy VFS blob adoption, blob-history backfill, and the
|
||||
// rebuildIndexesIfNeeded() gate + migration check.
|
||||
markPhase('index-init-gate')
|
||||
|
||||
// Register shutdown hooks for graceful count flushing (once globally)
|
||||
if (!Brainy.shutdownHooksRegisteredGlobally) {
|
||||
this.registerShutdownHooks()
|
||||
|
|
@ -1612,15 +1658,40 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
// Eager embedding initialization.
|
||||
// PHASE 4 of 5 — "VFS bootstrap": shutdown-hook registration, blob
|
||||
// storage init, the provider-summary log, flipping `initialized`,
|
||||
// the migration-lock wait, VFS construction+init, flipping generation
|
||||
// stamping active, the log-authority adopt/oracle check, and
|
||||
// pending-embed crash recovery.
|
||||
markPhase('vfs-bootstrap')
|
||||
|
||||
// Eager embedding initialization — BACKGROUND WARM (open-path fix).
|
||||
//
|
||||
// Adaptive default (8.0): the WASM embedding engine eagerly initializes
|
||||
// Adaptive default (8.0): the WASM embedding engine eagerly WARMS
|
||||
// during init() WHENEVER it is the active embedder — i.e. no native
|
||||
// 'embeddings' provider has taken over — and the instance is a writer
|
||||
// (not reader-mode) outside of unit tests. The WASM module (≈93MB with
|
||||
// the embedded model) takes 90-140s to compile on throttled CPUs; paying
|
||||
// that during boot rather than on the first embed()-driven call is the
|
||||
// right default for the overwhelmingly common single-process server.
|
||||
// the embedded model) takes 90-140s to compile on throttled CPUs.
|
||||
//
|
||||
// Historically this AWAITED `embeddingManager.init()` INLINE, so every
|
||||
// writer's open() blocked on the compile — N concurrent opens all
|
||||
// queued on the ONE process-global singleton (an ~80x contention
|
||||
// multiplier measured in a production restart storm: 90,017ms busy vs
|
||||
// 1,117ms quiet). The engine only needs to be ready before the FIRST
|
||||
// REAL embed() call, not before init() returns, so this now only
|
||||
// STARTS the warm and moves on — init() never waits for it.
|
||||
//
|
||||
// No double-await needed for correctness: `this.embed()` (~line 15420)
|
||||
// delegates to `this.embedder`, which for the default engine is
|
||||
// `embeddingManager.getEmbeddingFunction()` → `embeddingManager.embed()`
|
||||
// (src/embeddings/EmbeddingManager.ts). That method calls `await
|
||||
// this.init()` FIRST, and `init()` itself serializes every concurrent
|
||||
// caller onto ONE shared `globalInitPromise` — so the first real
|
||||
// embed() automatically waits for whichever finishes first: this
|
||||
// background warm (if still running) or a fresh init() (if the warm
|
||||
// hasn't reached this code yet, e.g. `eagerEmbeddings: false`).
|
||||
// Verified by reading both call sites; `_embeddingWarmPromise` below
|
||||
// is stored for observability only, never re-awaited by embed().
|
||||
//
|
||||
// Skipped automatically when:
|
||||
// - a native 'embeddings' provider is registered (it owns embeddings;
|
||||
|
|
@ -1628,8 +1699,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// - reader-mode (readers don't embed — they query existing vectors),
|
||||
// - unit-test mode (tests must stay fast and use the mock embedder).
|
||||
//
|
||||
// `eagerEmbeddings: false` is the explicit override to force lazy init
|
||||
// (first-embed) even when this instance is the active embedder.
|
||||
// `eagerEmbeddings: false` keeps meaning "no warm at all" — fully lazy,
|
||||
// the first embed() call pays the full cost inline, same as before.
|
||||
const isUnitTestMode = isDeterministicEmbedMode()
|
||||
const eager = this.config.eagerEmbeddings ?? true
|
||||
if (
|
||||
|
|
@ -1638,9 +1709,45 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this.config.mode !== 'reader' &&
|
||||
!isUnitTestMode
|
||||
) {
|
||||
console.log('Eager embedding initialization enabled...')
|
||||
await embeddingManager.init()
|
||||
console.log('Embedding engine ready')
|
||||
const warmStart = Date.now()
|
||||
console.log('Background embedding-engine warm started (init() does not wait for it)...')
|
||||
this._embeddingWarmPromise = embeddingManager
|
||||
.init()
|
||||
.then(() => {
|
||||
prodLog.info(
|
||||
`[Brainy] background embedding-engine warm complete in ${Date.now() - warmStart}ms`
|
||||
)
|
||||
})
|
||||
.catch((err) => {
|
||||
// Loud, never silent: a warm that fails to compile must be
|
||||
// heard NOW, not discovered as a mystery latency spike on
|
||||
// whichever request happens to trigger the first real embed().
|
||||
// That first embed() call still retries init() itself (the
|
||||
// singleton promise contract above) and surfaces its own typed
|
||||
// error to its caller — this is the immediate, background echo.
|
||||
prodLog.warn(
|
||||
`[Brainy] background embedding-engine warm FAILED: ` +
|
||||
`${(err as Error).message} — the first embed() call will retry ` +
|
||||
`initialization and surface the error there`
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
// PHASE 5 of 5 — "embedding-warm-started": just the synchronous cost
|
||||
// of kicking off the background warm above (the warm's own compile
|
||||
// time is NOT included — that's the whole point of backgrounding it).
|
||||
markPhase('embedding-warm-started')
|
||||
{
|
||||
const totalOpenMs = Date.now() - initStart
|
||||
if (totalOpenMs > 2000) {
|
||||
const phaseList = Object.entries(phaseTimingsMs)
|
||||
.map(([name, ms]) => `${name}=${ms}ms`)
|
||||
.join(', ')
|
||||
prodLog.warn(
|
||||
`[Brainy] slow open: ${totalOpenMs}ms total (${phaseList}) — see the ` +
|
||||
`phase breakdown above to find which one to investigate first`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Integration Hub initialization
|
||||
|
|
@ -15695,6 +15802,25 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return embeddingManager.isInitialized()
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the process-global WASM embedding engine (all-MiniLM-L6-v2,
|
||||
* fixed 384-dim output, ≈93MB with the bundled model, 90-140s cold compile
|
||||
* on throttled CPUs) is this instance's active embedder — `false` when a
|
||||
* plugin has replaced it via the `'embeddings'` provider key. A native
|
||||
* provider has no such cold-start cost and may use a different output
|
||||
* dimension, so it is never worth avoiding.
|
||||
*
|
||||
* Used by init-path bootstrap writes (the VFS root — see
|
||||
* `VirtualFileSystem.doInitializeRoot()`) to decide whether embedding a
|
||||
* value during `init()` risks paying the WASM engine's cold compile.
|
||||
*
|
||||
* @returns true when the default WASM engine is active (no native
|
||||
* `'embeddings'` provider registered).
|
||||
*/
|
||||
usesDefaultWasmEmbedder(): boolean {
|
||||
return !this.pluginRegistry.hasProvider('embeddings')
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup embedder
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -239,38 +239,54 @@ export class FileSystemStorage extends BaseStorage {
|
|||
|
||||
// Finish any restore interrupted by a crash (resume the staged swap, or
|
||||
// discard an uncommitted staging area) BEFORE counts/derived state load,
|
||||
// so the rest of startup sees the completed store.
|
||||
// so the rest of startup sees the completed store. ORDER-DEPENDENT:
|
||||
// `swapStagedRestoreIn()` reads `fs.readdir(rootDir)` and then
|
||||
// removes/renames rootDir's own TOP-LEVEL entries to place the staged
|
||||
// copy — racing that against the directory-creation batch below (which
|
||||
// also touches rootDir's children) could see a half-created directory
|
||||
// mid-swap or a mkdir racing a concurrent rm/rename on the same path.
|
||||
// Stays strictly sequential, never folded into the OPEN-PATH batch.
|
||||
await this.completeInterruptedRestore()
|
||||
|
||||
// OPEN-PATH FIX: the remaining bootstrap directories are mutually
|
||||
// independent — each is its own subtree under rootDir, and
|
||||
// `fs.mkdir(dir, { recursive: true })` creates every intermediate
|
||||
// segment of ITS OWN path in one call, so it never depends on any
|
||||
// sibling here existing first. Nothing between here and
|
||||
// `initializeCounts()` reads any of them, so batching collapses what
|
||||
// was up to 8 sequential mkdir round-trips (each a real syscall+await)
|
||||
// into one wave — this is what serialized an N-writer restart storm on
|
||||
// filesystem I/O it never structurally needed. `initializeCounts()`
|
||||
// right after DOES depend on `systemDir` (which the batch creates), so
|
||||
// it stays outside, awaited only once every directory has landed.
|
||||
await Promise.all([
|
||||
// Create the nouns directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.nounsDir)
|
||||
|
||||
this.ensureDirectoryExists(this.nounsDir),
|
||||
// Create the verbs directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.verbsDir)
|
||||
|
||||
this.ensureDirectoryExists(this.verbsDir),
|
||||
// Create the metadata directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.metadataDir)
|
||||
|
||||
this.ensureDirectoryExists(this.metadataDir),
|
||||
// Create the noun metadata directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.nounMetadataDir)
|
||||
|
||||
this.ensureDirectoryExists(this.nounMetadataDir),
|
||||
// Create the verb metadata directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.verbMetadataDir)
|
||||
|
||||
this.ensureDirectoryExists(this.verbMetadataDir),
|
||||
// Create both directories for backward compatibility
|
||||
await this.ensureDirectoryExists(this.systemDir)
|
||||
// Only create legacy directory if it exists (don't create new legacy dirs)
|
||||
this.ensureDirectoryExists(this.systemDir),
|
||||
// Only create legacy directory if it exists (don't create new legacy
|
||||
// dirs) — a read-then-maybe-write, but on its own subtree, so it's
|
||||
// still independent of every other entry in this batch.
|
||||
(async () => {
|
||||
if (await this.directoryExists(this.indexDir)) {
|
||||
await this.ensureDirectoryExists(this.indexDir)
|
||||
}
|
||||
|
||||
})(),
|
||||
// Create the locks directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.lockDir)
|
||||
|
||||
this.ensureDirectoryExists(this.lockDir),
|
||||
// Create the binary blobs directory if it doesn't exist
|
||||
await this.ensureDirectoryExists(this.blobsDir)
|
||||
this.ensureDirectoryExists(this.blobsDir)
|
||||
])
|
||||
|
||||
// Initialize count management
|
||||
// Initialize count management — depends on systemDir, created above.
|
||||
this.countsFilePath = path.join(this.systemDir, 'counts.json')
|
||||
await this.initializeCounts()
|
||||
|
||||
|
|
|
|||
|
|
@ -1983,25 +1983,32 @@ export interface BrainyConfig {
|
|||
reservedQueryMemory?: number // Memory reserved for queries in bytes (e.g., 1073741824 = 1GB)
|
||||
|
||||
/**
|
||||
* Controls when the WASM embedding engine is initialized.
|
||||
* Controls whether `init()` starts a BACKGROUND warm of the WASM embedding
|
||||
* engine.
|
||||
*
|
||||
* **Adaptive default (8.0):** when omitted, the engine eagerly initializes
|
||||
* during `init()` whenever the WASM embedder is the *active* one — i.e. no
|
||||
* native `'embeddings'` provider is registered — and this instance is a
|
||||
* writer (not `mode: 'reader'`) running outside unit tests. The WASM module
|
||||
* (≈93MB with the embedded model) takes 90-140s to compile on throttled
|
||||
* CPUs, so paying that during boot rather than on the first `embed()`-driven
|
||||
* call is the right default for a single-process server.
|
||||
* **Adaptive default (8.0, background since the open-path fix):** when
|
||||
* omitted, `init()` STARTS a background warm of the engine whenever the
|
||||
* WASM embedder is the *active* one — i.e. no native `'embeddings'`
|
||||
* provider is registered — and this instance is a writer (not
|
||||
* `mode: 'reader'`) running outside unit tests. The WASM module (≈93MB with
|
||||
* the embedded model) takes 90-140s to compile on throttled CPUs — but
|
||||
* `init()` never awaits that compile. It only starts it, so N concurrent
|
||||
* opens no longer serialize on the one process-global engine singleton.
|
||||
* The first `embed()` call then waits for whichever finishes first: the
|
||||
* background warm (if still running) or its own fresh init (if the warm
|
||||
* never started, e.g. `eagerEmbeddings: false`) — both paths converge on
|
||||
* the SAME shared promise inside the engine singleton, so the vector is
|
||||
* always correct; only the timing of who pays the wait differs.
|
||||
*
|
||||
* The adaptive path skips itself automatically when a native embeddings
|
||||
* provider owns embeddings, in reader-mode (readers query existing vectors
|
||||
* and never embed), and in unit-test mode (kept fast via the mock embedder).
|
||||
*
|
||||
* - `true` — force eager init during `init()` (the adaptive default already
|
||||
* does this for the active-embedder writer case; set it explicitly to be
|
||||
* unambiguous).
|
||||
* - `false` — explicit override to force lazy init (first `embed()` call)
|
||||
* even when this instance is the active embedder.
|
||||
* - `true` — force the background warm to start during `init()` (the
|
||||
* adaptive default already does this for the active-embedder writer
|
||||
* case; set it explicitly to be unambiguous).
|
||||
* - `false` — no warm at all. Fully lazy: the first `embed()` call pays the
|
||||
* full cold-compile cost inline, on whichever request triggers it.
|
||||
*/
|
||||
eagerEmbeddings?: boolean
|
||||
|
||||
|
|
|
|||
|
|
@ -89,6 +89,16 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
// Uses deterministic UUID format for storage compatibility
|
||||
private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000'
|
||||
|
||||
// OPEN-PATH FIX: the dimension of the placeholder vector given to the VFS
|
||||
// root when it is first created (see `doInitializeRoot`). Mirrors the
|
||||
// built-in WASM embedding engine's fixed, hardcoded output size
|
||||
// (all-MiniLM-L6-v2 — see src/embeddings/candle-wasm/src/lib.rs
|
||||
// HIDDEN_SIZE and src/embeddings/EmbeddingManager.ts). Deliberately NOT
|
||||
// derived from `brain.dimensions` — this constant only applies to the
|
||||
// default-embedder branch, where the true output dimension is this fixed
|
||||
// value by construction, never a moving target.
|
||||
private static readonly VFS_ROOT_VECTOR_DIMENSIONS = 384
|
||||
|
||||
/**
|
||||
* Construct a VFS bound to a Brainy instance.
|
||||
*
|
||||
|
|
@ -260,6 +270,40 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
try {
|
||||
console.log('VFS: Creating root directory (fixed ID: 00000000-0000-0000-0000-000000000000)')
|
||||
|
||||
// OPEN-PATH FIX: the VFS root is Brainy's own system-tier plumbing — it
|
||||
// is hidden from find()/getNounCount()/stats() by default and nothing
|
||||
// ever runs a semantic search against it — so it needs no REAL
|
||||
// embedding. Historically this add() always called embed('/'), which
|
||||
// meant every writer's FIRST-EVER open forced the process-global WASM
|
||||
// engine to cold-compile its model (measured 90-140s on throttled
|
||||
// CPUs) before the brain could even finish init(). This branch only
|
||||
// runs once per store (the root already exists — with a real vector —
|
||||
// in every previously-opened production store; reopening never re-adds
|
||||
// it), so the fix applies only to brand-new stores.
|
||||
//
|
||||
// Chosen fix: an explicit all-zero vector, not `deferEmbedding: true`.
|
||||
// `deferEmbedding` looked attractive (ack now, embed later) but its
|
||||
// landing path (`kickEmbedWorker()`, called synchronously right after
|
||||
// commit — see brainy.ts add()/update()) would still force the WASM
|
||||
// engine to cold-compile within milliseconds of open, just off the
|
||||
// awaited path instead of never paying it at all — worse than the
|
||||
// explicit-vector path, which never touches the engine for this row.
|
||||
// An all-zero vector is safe: `cosineDistance` (src/utils/distance.ts)
|
||||
// explicitly returns the MAXIMUM distance whenever either operand's
|
||||
// norm is zero, so the root can never rank ahead of real content in a
|
||||
// similarity search, and HNSW indexes it like any other vector.
|
||||
//
|
||||
// Only the default WASM engine gets this treatment — its output
|
||||
// dimension (384) is fixed and hardcoded, so the placeholder can never
|
||||
// mis-pin `brain.dimensions` for it. A plugin-registered native
|
||||
// 'embeddings' provider has no cold-compile cost AND may use a
|
||||
// different dimension, so it keeps embedding the root for real (same
|
||||
// as before this fix) rather than risk pinning the wrong dimension
|
||||
// ahead of the caller's own first real embed.
|
||||
const rootVector = this.brain.usesDefaultWasmEmbedder()
|
||||
? new Array(VirtualFileSystem.VFS_ROOT_VECTOR_DIMENSIONS).fill(0)
|
||||
: undefined
|
||||
|
||||
await this.brain.add({
|
||||
id: rootId, // Fixed ID - storage ensures uniqueness
|
||||
data: '/',
|
||||
|
|
@ -271,7 +315,8 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
// public AddParams.visibility union ('public' | 'internal') — this is the single
|
||||
// sanctioned internal setter, hence the cast.
|
||||
visibility: 'system' as 'public' | 'internal',
|
||||
metadata: this.getRootMetadata()
|
||||
metadata: this.getRootMetadata(),
|
||||
...(rootVector ? { vector: rootVector } : {})
|
||||
})
|
||||
|
||||
return rootId
|
||||
|
|
|
|||
199
tests/unit/brainy/open-path.test.ts
Normal file
199
tests/unit/brainy/open-path.test.ts
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
/**
|
||||
* OPEN-PATH tests: init() must never gate on the embedding model, the VFS
|
||||
* root bootstrap must never touch the embedding engine, and a slow open
|
||||
* must narrate its phases.
|
||||
*
|
||||
* Background: a production restart storm measured 90,017ms for a single
|
||||
* brain init vs 1,117ms quiet — an ~80x contention multiplier — traced to
|
||||
* every writer's init() eagerly awaiting the process-global WASM embedding
|
||||
* engine before the VFS root even existed. See src/brainy.ts performInit()
|
||||
* and src/vfs/VirtualFileSystem.ts doInitializeRoot().
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi } from 'vitest'
|
||||
import { Brainy } from '../../../src/brainy'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage'
|
||||
import { embeddingManager } from '../../../src/embeddings/EmbeddingManager'
|
||||
import { createTestConfig } from '../../helpers/test-factory'
|
||||
|
||||
/**
|
||||
* The four signals `isDeterministicEmbedMode()` checks (see
|
||||
* src/embeddings/deterministicEmbedMode.ts). The global unit-test setup
|
||||
* (tests/setup-unit.ts) sets some of these for the whole file/run in some
|
||||
* vitest configurations; other configurations leave them unset and run the
|
||||
* real WASM engine instead. The background-warm tests below need the
|
||||
* "not unit-test mode" branch of performInit() to actually execute, so they
|
||||
* save/clear/restore all four explicitly — deterministic regardless of
|
||||
* which config invoked this file, never relying on ambient state.
|
||||
*/
|
||||
function withRealEmbedderBranch<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const savedEnvDeterministic = process.env.BRAINY_DETERMINISTIC_EMBEDDINGS
|
||||
const savedEnvUnitTest = process.env.BRAINY_UNIT_TEST
|
||||
const g = globalThis as Record<string, unknown>
|
||||
const savedGlobalDeterministic = g.__BRAINY_DETERMINISTIC_EMBED__
|
||||
const savedGlobalUnitTest = g.__BRAINY_UNIT_TEST__
|
||||
|
||||
delete process.env.BRAINY_DETERMINISTIC_EMBEDDINGS
|
||||
delete process.env.BRAINY_UNIT_TEST
|
||||
delete g.__BRAINY_DETERMINISTIC_EMBED__
|
||||
delete g.__BRAINY_UNIT_TEST__
|
||||
|
||||
const restore = () => {
|
||||
if (savedEnvDeterministic !== undefined) process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = savedEnvDeterministic
|
||||
if (savedEnvUnitTest !== undefined) process.env.BRAINY_UNIT_TEST = savedEnvUnitTest
|
||||
if (savedGlobalDeterministic !== undefined) g.__BRAINY_DETERMINISTIC_EMBED__ = savedGlobalDeterministic
|
||||
if (savedGlobalUnitTest !== undefined) g.__BRAINY_UNIT_TEST__ = savedGlobalUnitTest
|
||||
}
|
||||
|
||||
return fn().finally(restore)
|
||||
}
|
||||
|
||||
/**
|
||||
* A MemoryStorage whose init() takes an artificially long time — a
|
||||
* controllable fake seam (not a wall-clock race) that reliably pushes
|
||||
* performInit()'s "storage-init" phase (and therefore the total open time)
|
||||
* past the 2000ms narration threshold, without touching the filesystem or
|
||||
* relying on real contention.
|
||||
*/
|
||||
class SlowMemoryStorage extends MemoryStorage {
|
||||
override async init(): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, 2200))
|
||||
await super.init()
|
||||
}
|
||||
}
|
||||
|
||||
describe('OPEN-PATH: init() never gates on the embedding model', () => {
|
||||
it('bootstrapping a fresh store never calls the embedding engine (VFS root add is engine-untouched)', async () => {
|
||||
const embedSpy = vi.spyOn(embeddingManager, 'embed')
|
||||
const brain = new Brainy(createTestConfig())
|
||||
try {
|
||||
await brain.init()
|
||||
|
||||
// The VFS root's add() must never have reached the embedding engine —
|
||||
// it carries an explicit placeholder vector instead (see
|
||||
// VirtualFileSystem.doInitializeRoot()).
|
||||
expect(embedSpy).not.toHaveBeenCalled()
|
||||
|
||||
// Sanity: the VFS is genuinely usable afterwards.
|
||||
const files = await brain.vfs.readdir('/')
|
||||
expect(files).toEqual([])
|
||||
} finally {
|
||||
await brain.close()
|
||||
embedSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('starts the embedding-engine warm in the BACKGROUND — init() resolves before the warm does', async () => {
|
||||
await withRealEmbedderBranch(async () => {
|
||||
const events: string[] = []
|
||||
let releaseWarm!: () => void
|
||||
const warmGate = new Promise<void>((resolve) => {
|
||||
releaseWarm = resolve
|
||||
})
|
||||
|
||||
const initSpy = vi.spyOn(embeddingManager, 'init').mockImplementation(async () => {
|
||||
events.push('warm-start')
|
||||
await warmGate
|
||||
events.push('warm-resolve')
|
||||
})
|
||||
|
||||
const brain = new Brainy(createTestConfig())
|
||||
try {
|
||||
await brain.init()
|
||||
events.push('init-resolved')
|
||||
|
||||
// init() started the warm but returned WITHOUT waiting for it.
|
||||
expect(initSpy).toHaveBeenCalledTimes(1)
|
||||
expect(events).toEqual(['warm-start', 'init-resolved'])
|
||||
|
||||
// Now let the fake warm finish and confirm it lands strictly after.
|
||||
releaseWarm()
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
expect(events).toEqual(['warm-start', 'init-resolved', 'warm-resolve'])
|
||||
} finally {
|
||||
await brain.close()
|
||||
initSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('narrates a background warm FAILURE loudly instead of losing it silently', async () => {
|
||||
await withRealEmbedderBranch(async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const initSpy = vi
|
||||
.spyOn(embeddingManager, 'init')
|
||||
.mockRejectedValue(new Error('simulated cold-compile failure'))
|
||||
|
||||
const brain = new Brainy(createTestConfig())
|
||||
try {
|
||||
// init() itself must still resolve — a failed background warm is
|
||||
// never fatal to open().
|
||||
await expect(brain.init()).resolves.toBeUndefined()
|
||||
|
||||
// Give the background .catch() a microtask/macrotask to run.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||
|
||||
const failureLine = warnSpy.mock.calls
|
||||
.map((args) => args.map(String).join(' '))
|
||||
.find((line) => line.includes('background embedding-engine warm FAILED'))
|
||||
expect(failureLine).toBeDefined()
|
||||
expect(failureLine).toContain('simulated cold-compile failure')
|
||||
} finally {
|
||||
await brain.close()
|
||||
initSpy.mockRestore()
|
||||
warnSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('eagerEmbeddings: false starts no warm at all', async () => {
|
||||
await withRealEmbedderBranch(async () => {
|
||||
const initSpy = vi.spyOn(embeddingManager, 'init')
|
||||
const brain = new Brainy({ ...createTestConfig(), eagerEmbeddings: false })
|
||||
try {
|
||||
await brain.init()
|
||||
expect(initSpy).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
await brain.close()
|
||||
initSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('narrates a slow open with a per-phase ms breakdown once total time exceeds 2000ms', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const brain = new Brainy({ ...createTestConfig(), storage: new SlowMemoryStorage() })
|
||||
try {
|
||||
await brain.init()
|
||||
|
||||
const slowOpenLine = warnSpy.mock.calls
|
||||
.map((args) => args.map(String).join(' '))
|
||||
.find((line) => line.includes('[Brainy] slow open:'))
|
||||
|
||||
expect(slowOpenLine).toBeDefined()
|
||||
expect(slowOpenLine).toContain('storage-init=')
|
||||
expect(slowOpenLine).toContain('generation-store-open-fold=')
|
||||
expect(slowOpenLine).toContain('index-init-gate=')
|
||||
expect(slowOpenLine).toContain('vfs-bootstrap=')
|
||||
expect(slowOpenLine).toContain('embedding-warm-started=')
|
||||
} finally {
|
||||
await brain.close()
|
||||
warnSpy.mockRestore()
|
||||
}
|
||||
}, 20000)
|
||||
|
||||
it('stays silent about phase timing when open is fast (under 2000ms)', async () => {
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const brain = new Brainy(createTestConfig())
|
||||
try {
|
||||
await brain.init()
|
||||
const slowOpenLine = warnSpy.mock.calls
|
||||
.map((args) => args.map(String).join(' '))
|
||||
.find((line) => line.includes('[Brainy] slow open:'))
|
||||
expect(slowOpenLine).toBeUndefined()
|
||||
} finally {
|
||||
await brain.close()
|
||||
warnSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue