From f8f64780b11a305084348b9b0a92d4a33f0e43c1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 24 Aug 2026 12:45:51 -0700 Subject: [PATCH] =?UTF-8?q?feat(health):=20the=20gate=20reads=20the=20name?= =?UTF-8?q?d=20report=20=E2=80=94=20reads=20refuse=20loudly,=20never=20reb?= =?UTF-8?q?uild;=20open=20serves=20before=20it=20returns;=20the=20ceremony?= =?UTF-8?q?=20door?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read gate stops consulting the unnamed isReady() boolean: every provider may expose healthReport() (sync, O(1), composed from exact ledgers — HealthReport with a monotonic generation, per-invariant source ledger|deep|unledgered, missing {count, sample}), and one readiness authority (assessProviderHealth) derives the verdict. Unledgered families are UNKNOWN — never healthy, never broken; a report that throws is a loud not-ready, never a shrug. Reads at the four index choke points refuse with the typed NotReady errors, narrated once per (provider, generation) — a read NEVER starts a store walk: - the first-read lazy build retires (open builds instead, regardless of size — the ≥10k deferral and the "lazy loading on first query" branch go; disableAutoRebuild is re-meant honestly in its docs); - the verify*Live read-path rebuild triggers retire (refuse-or-serve); - the read-time consistency probe that could launch a dark rebuild from an ordinary find() retires; - repairIndex({ rebuild: ['metadata'|'graph'|'vector'] | 'all' }) is the one explicit door: rebuilds the named leg unconditionally and reports rebuilt per family; bare repairIndex() stays report-driven. test(lifecycle): the biography lane — a store's whole life, refereed tests/lifecycle/: an independent shadow model referees every read after every chapter (founding, a working day, clean restart, crash, repair, second life). Chapters 1-3 green. Chapters 4-6 assert the true contract and are marked .fails as a release-blocking finding (the kill-matrix convention): after a crash + adopt reopen the metadata index computes its 'catchup' watermark verdict and nothing consumes it — find() serves the pre-crash index while canonical and counts recover. The catchup wiring is the cure; a passing .fails will force the marker's removal. The lane runs in the integration gate (config + coverage guard). --- docs/api/README.md | 23 + src/brainy.ts | 792 ++++++++---------- src/index.ts | 4 + src/plugin.ts | 102 +++ src/types/brainy.types.ts | 14 +- src/utils/indexReadiness.ts | 117 +++ tests/configs/vitest.integration.config.ts | 3 + .../cold-graph-connected-8.0.test.ts | 129 ++- tests/integration/health-gate.test.ts | 352 ++++++++ tests/lifecycle/README.md | 16 + tests/lifecycle/biography.test.ts | 429 ++++++++++ tests/lifecycle/biographyHarness.ts | 389 +++++++++ tests/unit/brainy/lazy-notready-honor.test.ts | 94 ++- .../brainy/metadata-provider-contract.test.ts | 59 +- tests/unit/brainy/migration-deference.test.ts | 41 +- tests/unit/metadata-cold-read-guard.test.ts | 40 +- tests/unit/test-suite-coverage-guard.test.ts | 3 + tests/unit/utils/indexReadiness.test.ts | 153 ++++ tests/unit/vector-cold-read-guard.test.ts | 52 +- 19 files changed, 2160 insertions(+), 652 deletions(-) create mode 100644 tests/integration/health-gate.test.ts create mode 100644 tests/lifecycle/README.md create mode 100644 tests/lifecycle/biography.test.ts create mode 100644 tests/lifecycle/biographyHarness.ts create mode 100644 tests/unit/utils/indexReadiness.test.ts diff --git a/docs/api/README.md b/docs/api/README.md index 4ca84364..fb9cc920 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -1831,6 +1831,29 @@ const semanticOnly = await brain.getStats({ excludeVFS: true }) --- +### `repairIndex(options?)` → `Promise` + +The ceremony door for index repair. Bare `repairIndex()` is report-driven: it +prunes orphaned containers, recomputes count rollups, reconciles VFS +containment, and rebuilds only a derived-index family whose own health check +asks for it. Pass `options.rebuild` to force one or more families to rebuild +UNCONDITIONALLY — no health check is consulted — when an operator has +independent reason to reconcile a family regardless of what it self-reports. + +```typescript +// Report-driven: only heals what actually needs it +const report = await brain.repairIndex() +console.log(report.healedTotal, report.families) + +// Explicit: force the graph adjacency to rebuild from canonical, unconditionally +await brain.repairIndex({ rebuild: ['graph'] }) + +// Explicit: force all three derived indexes to rebuild +await brain.repairIndex({ rebuild: 'all' }) +``` + +--- + ## Lifecycle ### Initialization diff --git a/src/brainy.ts b/src/brainy.ts index db332e18..c8912e26 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -197,7 +197,7 @@ import { import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' -import { assessIndexReadiness } from './utils/indexReadiness.js' +import { assessIndexReadiness, assessProviderHealth } from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { @@ -634,8 +634,6 @@ export class Brainy implements BrainyInterface { /** One-shot guard so the degraded-reads warning fires once per degraded window * (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */ private _degradedReadWarned = false - /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */ - private _metadataConsistencyProbed = false /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */ private _graphAdjacencyVerified = false /** Re-entrancy guard: a verify (rebuild → reads) is in flight. */ @@ -804,11 +802,20 @@ export class Brainy implements BrainyInterface { // applies only to instances that were never closed. private closed = false - // Lazy rebuild state (Production-scale lazy loading) - // Prevents race conditions when multiple queries trigger rebuild simultaneously - private lazyRebuildInProgress = false + // Index-build-at-open state. `lazyRebuildCompleted` predates the health-gate + // law (it named a first-QUERY lazy rebuild) and stays for `getIndexStatus()` + // API compatibility, but its truth changed: a needed rebuild now runs + // unconditionally at open() (see `rebuildIndexesIfNeeded`), never deferred to + // a read, so this simply flips true once that open-time step has run. + // `lazyRebuildInProgress` / `lazyRebuildPromise` (the first-query rebuild's + // concurrency guard) are retired with the lazy-build path they served — + // `ensureIndexesLoaded()` is a read-time CHECK now, never a build. private lazyRebuildCompleted = false - private lazyRebuildPromise: Promise | null = null + + // Read-gate narration dedup: a degraded-but-serving or not-ready health + // report narrates via prodLog.warn ONCE per (provider, report.generation) — + // never once per read. Keyed on the provider instance itself. + private _lastNarratedHealthGeneration = new Map() constructor(config?: BrainyConfig) { // The reserved-field write policy died with the field-addressing law: @@ -1444,8 +1451,12 @@ export class Brainy implements BrainyInterface { }).backfillBlobHistoryRefCountsIfNeeded() } - // Rebuild indexes if needed for existing data + // Rebuild indexes if needed for existing data. Runs to completion before + // init() returns — there is no more first-query lazy path, so the flag + // below (kept for getIndexStatus() API compatibility) simply flips true + // once this open-time step has run. await this.rebuildIndexesIfNeeded() + this.lazyRebuildCompleted = true // Check for pending data migrations await this.checkMigrations() @@ -3976,8 +3987,9 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. Fast path after the latch is one boolean. - await this.ensureIndexesLoaded() + // provider for 15 minutes. A CHECK only — it never builds; throws a typed + // NotReady error if a provider's health report says it isn't serving. + this.ensureIndexesLoaded() const entityInt = this.graphEntityInt(uuid) if (entityInt === undefined) return [] const neighborInts = await this.graphIndex.getNeighbors(entityInt, options) @@ -4005,72 +4017,59 @@ export class Brainy implements BrainyInterface { /** * @description Verify that the graph adjacency is actually LIVE before a graph read trusts * its result. A native graph index can load its relationship COUNT (manifest) on a cold open - * of a LARGE brain (≥10k nouns, which skips the eager index rebuild) but NOT its - * source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even though - * edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would serve - * that `[]` as if it were truth. + * but NOT its source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even + * though edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would + * serve that `[]` as if it were truth. * - * Two detection strategies, in order of honesty: - * - **Preferred (8.0 contract):** the provider exposes a sync `isReady()` that is true ONLY - * when the edges are loaded. `false` → hydrate the id-mapper (a native int adjacency - * resolves endpoints through it), rebuild from storage, and re-check `isReady()`; if it is - * still `false`, throw {@link GraphIndexNotReadyError} rather than returning `[]`. - * - **Fallback (providers without `isReady()`):** a GLOBAL known-edge sample (a real + * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the dark-rebuild + * failure mode this contract retires (open() alone owns building; see + * {@link rebuildIndexesIfNeeded}). Two detection strategies, in order of honesty: + * - **Preferred:** {@link assessProviderHealth} — the provider's named `healthReport()` when + * exposed, else its sync `isReady()`. Not serving → THROW {@link GraphIndexNotReadyError} + * naming the reasons, immediately — no rebuild attempt. + * - **Fallback (providers with neither signal):** a READ-ONLY GLOBAL known-edge sample (a real * persisted verb's `sourceId`, which by definition HAS an outgoing edge) — NOT any queried * anchor, because brainy cannot cheaply tell "adjacency unloaded" from "this node is - * genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, the - * adjacency did not load: rebuild and re-probe; if even that fails, throw. + * genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, THROW — + * the probe refuses loudly; it does not self-heal. * - * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing - * to verify), or `'rebuilt'` when a cold-unloaded adjacency was just healed from storage — - * in which case callers that observed an empty result must RE-RUN their collection. - * @throws {GraphIndexNotReadyError} when the index claims edges but cannot serve a known - * persisted edge (or stays not-ready) even after a rebuild. + * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing to + * verify). + * @throws {GraphIndexNotReadyError} when the index is not serving, or claims edges but cannot + * serve a known persisted edge. */ - private async verifyGraphAdjacencyLive(): Promise<'live' | 'rebuilt'> { + private async verifyGraphAdjacencyLive(): Promise<'live'> { if (this._graphAdjacencyVerified) return 'live' // Coordinated migration LOCK (#18): while the graph provider owns a locked - // rebuild-from-canonical, brainy must NOT fire its own graphIndex.rebuild() - // on a read — that would race the provider's in-place rebuild. The data-plane - // lock (awaitMigrationLock in ensureInitialized) already makes callers wait, - // so this is normally unreachable mid-migration; the guard is defensive. It + // rebuild-from-canonical, brainy must NOT judge it here — the provider owns + // its index until it verifies-and-swaps. The data-plane lock + // (awaitMigrationLock in ensureInitialized) already makes callers wait, so + // this is normally unreachable mid-migration; the guard is defensive. It // deliberately does NOT set `_graphAdjacencyVerified`, so the real verify runs // once the migration clears. if (this.providerIsMigrating(this.graphIndex)) return 'live' - // Re-entrancy: rebuild() can trigger reads (neighbors/related) that call back into this - // guard. While a verify is in flight, short-circuit so we cannot recurse into rebuild(). + // Re-entrancy: a fallback probe below calls getNeighbors(), which does not + // re-enter this guard, but the short-circuit is kept defensively cheap. if (this._graphAdjacencyVerifying) return 'live' this._graphAdjacencyVerifying = true try { - const gi = this.graphIndex as GraphAdjacencyIndex & { isReady?: () => boolean } - - // ── Strategy 1: honest isReady() signal (cortex >= 2.7.8 / 3.0) ────────── - if (typeof gi.isReady === 'function') { - if (gi.isReady()) { + // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── + const assessment = assessProviderHealth(this.graphIndex) + if (assessment.via === 'health-report' || assessment.via === 'is-ready') { + if (assessment.readiness === 'ready') { this._graphAdjacencyVerified = true return 'live' } - // Not ready: the edges did not load on open. Hydrate the id-mapper, then rebuild. - if (!this.config.silent) { - console.warn( - `[Brainy] Graph adjacency reports not-ready (isReady() === false) — the persisted ` + - `adjacency did not load on open. Rebuilding from storage…` - ) - } - await this.hydrateIdMapperForGraphRebuild() - await this.graphIndex.rebuild() - if (gi.isReady()) { - this._graphAdjacencyVerified = true - return 'rebuilt' - } throw new GraphIndexNotReadyError( - `Graph adjacency index reports not-ready even after a rebuild — the persisted ` + - `adjacency could not be loaded. find({ connected }), neighbors() and related() ` + - `cannot be served reliably for this brain.` + `Graph adjacency index is not serving (via ${assessment.via}): ` + + `${assessment.reasons.join('; ') || 'not ready'}. find({ connected }), neighbors() and ` + + `related() refuse rather than serve an empty result — rebuild via ` + + `repairIndex({ rebuild: ['graph'] }) or reopen the brain.` ) } - // ── Strategy 2: known-edge-sample probe (providers without isReady()) ──── + // ── Strategy 2: known-edge-sample probe (providers with neither signal) ─ + // READ-ONLY — refuses loudly on failure; never calls rebuild(). const claimed = await this.graphIndex.size() if (!claimed || claimed <= 0) return 'live' // no edges claimed — nothing to verify @@ -4086,10 +4085,9 @@ export class Brainy implements BrainyInterface { // the sample is not one of this brain's own edges — e.g. a shared on-disk store reused // across instances surfaces a foreign verb whose UUID this brain's resident mapper never // interned. We cannot prove a cold-unloaded adjacency from such a sample, so treat it as - // INCONCLUSIVE: mark verified and return 'live' rather than rebuilding/throwing. (The honest - // cold-load signal for native providers is isReady(), checked above; the JS baseline keeps - // its mapper resident, so its OWN edges always resolve — the targeted 7.x failure mode, - // "mapper loaded but adjacency empty", still resolves the source and is detected below.) + // INCONCLUSIVE: mark verified and return 'live' rather than throwing. (The honest cold-load + // signal for native providers is Strategy 1, checked above; the JS baseline keeps its mapper + // resident, so its OWN edges always resolve.) const sourceInt = this.graphEntityInt(verb.sourceId) if (sourceInt === undefined) { this._graphAdjacencyVerified = true @@ -4097,37 +4095,23 @@ export class Brainy implements BrainyInterface { } // Ask the adjacency for ONE neighbor of the (mapped) known-edge source. - const probeKnownSource = async (): Promise => - (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0 - - if (await probeKnownSource()) { + const hasNeighbor = (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0 + if (hasNeighbor) { this._graphAdjacencyVerified = true return 'live' // adjacency is live — the common case } // INCONSISTENT: the index reports edges but a KNOWN-mapped persisted edge's source has none → - // the adjacency did not load on open. Hydrate the mapper and rebuild from storage. - if (!this.config.silent) { - console.warn( - `[Brainy] Graph adjacency reports ${claimed} relationship(s) but a persisted edge ` + - `resolves to none — the persisted adjacency did not load on open. Rebuilding from storage…` - ) - } - await this.hydrateIdMapperForGraphRebuild() - await this.graphIndex.rebuild() - - if (await probeKnownSource()) { - this._graphAdjacencyVerified = true - return 'rebuilt' - } + // the adjacency did not load. Refuse loudly — never rebuild from a read. throw new GraphIndexNotReadyError( - `Graph adjacency index reports ${claimed} relationship(s) but returns no edges even ` + - `after a rebuild — the persisted adjacency could not be loaded. find({ connected }), ` + - `neighbors() and related() cannot be served reliably for this brain.` + `Graph adjacency index reports ${claimed} relationship(s) but a persisted edge's source ` + + `resolves to none — the persisted adjacency did not load. find({ connected }), ` + + `neighbors() and related() refuse rather than serve an empty result — rebuild via ` + + `repairIndex({ rebuild: ['graph'] }) or reopen the brain.` ) } catch (err) { if (err instanceof GraphIndexNotReadyError) throw err - // A transient probe/rebuild failure must not break the actual query NOR be + // A transient probe failure must not break the actual query NOR be // masked as "no data". Allow a re-check on the next graph read and fall through. this._graphAdjacencyVerified = false if (!this.config.silent) { @@ -4144,27 +4128,51 @@ export class Brainy implements BrainyInterface { * On a cold open a native metadata provider can report data yet not serve its * `where` postings, so `find({ where })` silently returns `[]` — the exact * failure a downstream deployment reported (cold reads blanking filtered pages - * after every restart). This one-shot guard, run on the first FILTERED `find()`, - * closes that: it takes a KNOWN persisted entity + one of its plain field values - * and asks the index to resolve it. If the index returns the known id the field - * postings are live (the common case, and the ONLY cost on a warm brain — one - * O(1) probe). If it does not, the postings did not load: brainy rebuilds the - * index from the canonical records and re-probes; if it STILL cannot serve the - * known value it throws a loud {@link MetadataIndexNotReadyError} rather than - * let a silent `[]` stand. Inconclusive cases (empty store, no plain field to - * probe, a shared store surfacing a foreign entity) are treated as live — never - * a false rebuild. A migrating provider is skipped (it owns its locked rebuild). - * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. + * after every restart). + * + * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the + * dark-rebuild failure mode this contract retires (open() alone owns + * building; see {@link rebuildIndexesIfNeeded}). Two detection strategies: + * - **Preferred:** {@link assessProviderHealth} — the provider's named + * `healthReport()` when exposed, else its sync `isReady()`. Not serving → + * THROW {@link MetadataIndexNotReadyError} naming the reasons, immediately. + * - **Fallback (providers with neither signal):** a READ-ONLY known-value + * probe, run on the first FILTERED `find()`: take a KNOWN persisted entity + * + one of its plain field values and ask the index to resolve it. If the + * index does not return the known id, THROW — the probe refuses loudly; + * it does not self-heal. Inconclusive cases (empty store, no plain field + * to probe, a shared store surfacing a foreign entity) are treated as + * live — never a false throw. A migrating provider is skipped (it owns + * its locked rebuild). + * @returns `'live'` when the index serves. */ - private async verifyMetadataLive(): Promise<'live' | 'rebuilt'> { + private async verifyMetadataLive(): Promise<'live'> { if (this._metadataVerified) return 'live' // Migration LOCK (#18): a migrating provider owns its in-place rebuild — do // not race it. Defensive; the data-plane lock already gates callers upstream. if (this.providerIsMigrating(this.metadataIndex)) return 'live' - // Re-entrancy: rebuild() can trigger reads that call back into this guard. + // Re-entrancy: the fallback probe below calls filterIdsBelted(), which + // re-enters ensureIndexesLoaded() (a cheap CHECK) but not this guard. if (this._metadataVerifying) return 'live' this._metadataVerifying = true try { + // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── + const assessment = assessProviderHealth(this.metadataIndex) + if (assessment.via === 'health-report' || assessment.via === 'is-ready') { + if (assessment.readiness === 'ready') { + this._metadataVerified = true + return 'live' + } + throw new MetadataIndexNotReadyError( + `Metadata field index is not serving (via ${assessment.via}): ` + + `${assessment.reasons.join('; ') || 'not ready'}. find({ where }) and other filtered ` + + `reads refuse rather than serve an empty result — rebuild via ` + + `repairIndex({ rebuild: ['metadata'] }) or reopen the brain.` + ) + } + + // ── Strategy 2: known-value probe (providers with neither signal) ────── + // READ-ONLY — refuses loudly on failure; never calls rebuild(). // A KNOWN persisted entity + one plain field to probe. Sample a few so a // system-only entity (e.g. the VFS root) doesn't make every open inconclusive. const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) @@ -4186,7 +4194,7 @@ export class Brainy implements BrainyInterface { return ids.includes(p.id) } catch { // FIELD_NOT_INDEXED for a field a persisted entity actually holds is - // itself the cold/broken signal — treat as not-serving (→ rebuild). + // itself the cold/broken signal — treat as not-serving. return false } } @@ -4196,26 +4204,15 @@ export class Brainy implements BrainyInterface { return 'live' // field postings are live — the common case } - if (!this.config.silent) { - console.warn( - `[Brainy] Metadata field index returns no match for a known persisted value of ` + - `'${p.field}' — the field postings did not load on open. Rebuilding from storage…` - ) - } - await this.metadataIndex.rebuild() - - if (await probeServes()) { - this._metadataVerified = true - return 'rebuilt' - } throw new MetadataIndexNotReadyError( - `Metadata field index cannot serve a known persisted value of '${p.field}' even after ` + - `a rebuild — find({ where }) and other filtered reads cannot be served reliably for ` + - `this brain (a silent empty result would misrepresent existing data).` + `Metadata field index cannot serve a known persisted value of '${p.field}' — the field ` + + `postings did not load. find({ where }) and other filtered reads refuse rather than ` + + `serve an empty result — rebuild via repairIndex({ rebuild: ['metadata'] }) or reopen ` + + `the brain.` ) } catch (err) { if (err instanceof MetadataIndexNotReadyError) throw err - // A transient probe/rebuild failure must not break the query NOR mask as + // A transient probe failure must not break the query NOR mask as // "no data". Allow a re-check on the next filtered read and fall through. this._metadataVerified = false if (!this.config.silent) { @@ -4261,57 +4258,52 @@ export class Brainy implements BrainyInterface { * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded * its serving structure (the mmap/DiskANN graph) — so a pure semantic * `find({ query })` silently returns `[]`. A pure semantic query has - * `hasFilterCriteria === false`, so the metadata guard never fires; this guard - * closes that gap. Run one-shot on the first vector/proximity search: - * - **Preferred (honest signal):** the provider exposes `isReady()`. `false` - * → rebuild from storage, re-check; if still `false`, throw - * {@link VectorIndexNotReadyError} rather than serving `[]`. - * - **Fallback (no `isReady()`):** a KNOWN persisted vector (sampled + - * hydrated) is searched against the index; if it does not self-match, the - * serving structure did not load — rebuild + re-probe, else throw. + * `hasFilterCriteria === false`, so the metadata guard never fires; this + * guard closes that gap. Run one-shot on the first vector/proximity search. + * + * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the + * dark-rebuild failure mode this contract retires (open() alone owns + * building; see {@link rebuildIndexesIfNeeded}). Two detection strategies: + * - **Preferred:** {@link assessProviderHealth} — the provider's named + * `healthReport()` when exposed, else its sync `isReady()`. Not serving → + * THROW {@link VectorIndexNotReadyError} naming the reasons, immediately. + * - **Fallback (providers with neither signal):** a READ-ONLY KNOWN + * persisted vector (sampled + hydrated) is searched against the index; if + * it does not self-match, THROW — the probe refuses loudly; it does not + * self-heal. * Inconclusive cases (empty store, no probeable vector, `size()===0` — where - * the JS baseline's cold load is `ensureIndexesLoaded`'s job) are treated as - * live: never a false rebuild. A migrating provider is skipped (it owns its - * locked rebuild). - * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. + * the JS baseline is built at open) are treated as live: never a false + * throw. A migrating provider is skipped (it owns its locked rebuild). + * @returns `'live'` when the index serves. */ - private async verifyVectorLive(): Promise<'live' | 'rebuilt'> { + private async verifyVectorLive(): Promise<'live'> { if (this._vectorVerified) return 'live' // Migration LOCK (#18): a migrating provider owns its in-place rebuild. if (this.providerIsMigrating(this.index)) return 'live' - // Re-entrancy: rebuild() can trigger reads that call back into this guard. + // Re-entrancy: the fallback probe below calls index.search(), which does + // not re-enter this guard, but the short-circuit is kept defensively cheap. if (this._vectorVerifying) return 'live' this._vectorVerifying = true try { - // ── Strategy 1: honest isReady() signal (native provider) ────────────── - const readiness = assessIndexReadiness(this.index) - if (readiness !== 'unknown') { - if (readiness === 'ready') { + // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── + const assessment = assessProviderHealth(this.index) + if (assessment.via === 'health-report' || assessment.via === 'is-ready') { + if (assessment.readiness === 'ready') { this._vectorVerified = true return 'live' } - // Not ready: the serving structure did not load on open. Rebuild. - if (!this.config.silent) { - console.warn( - `[Brainy] Vector index reports not-ready (isReady() === false) — the persisted ` + - `vector index did not load on open. Rebuilding from storage…` - ) - } - await this.index.rebuild() - if (assessIndexReadiness(this.index) === 'ready') { - this._vectorVerified = true - return 'rebuilt' - } throw new VectorIndexNotReadyError( - `Vector index reports not-ready even after a rebuild — semantic find({ query }) and ` + - `proximity search cannot be served reliably for this brain (a silent empty result ` + - `would misrepresent existing data).` + `Vector index is not serving (via ${assessment.via}): ` + + `${assessment.reasons.join('; ') || 'not ready'}. Semantic find({ query }) and ` + + `proximity search refuse rather than serve an empty result — rebuild via ` + + `repairIndex({ rebuild: ['vector'] }) or reopen the brain.` ) } - // ── Strategy 2: known-vector probe (providers without isReady()) ─────── + // ── Strategy 2: known-vector probe (providers with neither signal) ───── + // READ-ONLY — refuses loudly on failure; never calls rebuild(). const claimed = this.index.size() - if (!claimed || claimed <= 0) return 'live' // JS cold path is ensureIndexesLoaded's job + if (!claimed || claimed <= 0) return 'live' // JS cold path is built at open const probe = await this.pickVectorProbe() if (!probe) { @@ -4321,44 +4313,30 @@ export class Brainy implements BrainyInterface { } const p = probe - const probeServes = async (): Promise => { - // The failure mode we guard is the SILENT EMPTY result: a cold index that - // loaded its COUNT but not its serving structure returns `[]` for a - // known-present vector, while a warm index returns at least one hit. We - // check for a NON-EMPTY result, NOT an exact self-match — HNSW is - // approximate and `get()` may return a re-hydrated/normalized vector, so - // demanding the exact self as top-1 would false-positive on a perfectly - // healthy index (and wrongly rebuild → throw). - const hits = await this.index.search(p.vector, 1) - return hits.length > 0 - } + // The failure mode we guard is the SILENT EMPTY result: a cold index that + // loaded its COUNT but not its serving structure returns `[]` for a + // known-present vector, while a warm index returns at least one hit. We + // check for a NON-EMPTY result, NOT an exact self-match — HNSW is + // approximate and `get()` may return a re-hydrated/normalized vector, so + // demanding the exact self as top-1 would false-positive on a perfectly + // healthy index (and wrongly throw). + const hits = await this.index.search(p.vector, 1) void p.id // probe keyed on the vector; id retained for diagnostics only - if (await probeServes()) { + if (hits.length > 0) { this._vectorVerified = true return 'live' // serving structure is live — the common case } - if (!this.config.silent) { - console.warn( - `[Brainy] Vector index reports ${claimed} vector(s) but a known persisted vector ` + - `returns no results — the serving structure did not load on open. Rebuilding…` - ) - } - await this.index.rebuild() - - if (await probeServes()) { - this._vectorVerified = true - return 'rebuilt' - } throw new VectorIndexNotReadyError( `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` + - `results even after a rebuild — semantic find({ query }) cannot be served reliably ` + - `for this brain (a silent empty result would misrepresent existing data).` + `results — the serving structure did not load. Semantic find({ query }) refuses rather ` + + `than serve an empty result — rebuild via repairIndex({ rebuild: ['vector'] }) or ` + + `reopen the brain.` ) } catch (err) { if (err instanceof VectorIndexNotReadyError) throw err - // A transient probe/rebuild failure must not break the query NOR mask as + // A transient probe failure must not break the query NOR mask as // "no data". Allow a re-check on the next vector read and fall through. this._vectorVerified = false if (!this.config.silent) { @@ -6559,14 +6537,10 @@ export class Brainy implements BrainyInterface { // loader and cold-read probes below already defer to a migrating provider. await this.ensureInitialized({ needs: [] }) - // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) - // This is a production-safe, concurrency-controlled lazy load - await this.ensureIndexesLoaded() - - // One-shot cold-open self-heal: an O(1) probe of the metadata index (when the - // provider offers one) repairs an already-poisoned index on first read — the - // metadata counterpart of the graph cold-load guard. No-op for the JS index. - await this.ensureMetadataConsistencyProbed() + // READ-SURFACE READINESS GATE (see filterIdsBelted): a CHECK only — it + // never builds. open() already brought every provider to serving before + // init() returned; this throws a typed NotReady error if one isn't. + this.ensureIndexesLoaded() // Loudly flag a degraded derived index (failed init rebuild, or an // adopt-forward degraded commit) so a partial result is never mistaken for @@ -11851,10 +11825,10 @@ export class Brainy implements BrainyInterface { } /** - * Get index loading status (Diagnostic for lazy loading) + * Get index loading status (diagnostic) * - * Returns detailed information about index population and lazy loading state. - * Useful for debugging empty query results or performance troubleshooting. + * Returns detailed information about index population state. Useful for + * debugging empty query results or performance troubleshooting. * * @example * ```typescript @@ -11863,7 +11837,7 @@ export class Brainy implements BrainyInterface { * console.log(`Metadata Index: ${status.metadataIndex.entries} entries`) * console.log(`Graph Index: ${status.graphIndex.relationships} relationships`) * console.log(`Pending embeds: ${status.projections.semantic.pendingEmbeds}`) - * console.log(`Lazy rebuild completed: ${status.lazyRebuildCompleted}`) + * console.log(`Index build completed at open: ${status.lazyRebuildCompleted}`) * ``` */ @@ -11882,8 +11856,9 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. Fast path after the latch is one boolean. - await this.ensureIndexesLoaded() + // provider for 15 minutes. A CHECK only — it never builds; throws a typed + // NotReady error if a provider's health report says it isn't serving. + this.ensureIndexesLoaded() try { return await this.metadataIndex.getIdsForFilter(filter, opts) } catch (err) { @@ -11895,6 +11870,10 @@ export class Brainy implements BrainyInterface { async getIndexStatus(): Promise<{ initialized: boolean + /** `true` once open()'s index-build-if-needed step has run. Named for API + * compatibility with the retired first-query lazy-build path; a needed + * rebuild now always runs at open, never deferred to a read, so this is + * simply `initialized`'s index-build counterpart. */ lazyRebuildCompleted: boolean /** Deferred embeds not yet landed (MT5) — the eventual-vector-index backlog. */ pendingEmbeds: number @@ -14373,8 +14352,9 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. Fast path after the latch is one boolean. - await this.ensureIndexesLoaded() + // provider for 15 minutes. A CHECK only — it never builds; throws a typed + // NotReady error if a provider's health report says it isn't serving. + this.ensureIndexesLoaded() // 8.0 BigInt boundary: unmapped node → no relations. const nodeInt = this.graphEntityInt(nodeId) if (nodeInt === undefined) return [] @@ -14983,16 +14963,13 @@ export class Brainy implements BrainyInterface { // Cold-load guard: an empty connected set is suspicious. The native adjacency can report // size()>0 (or isReady()===false) on a cold open yet have loaded NO source→target edges — so - // traversal silently returns []. Re-verify against the honest isReady() signal (or, for older - // providers, a GLOBAL known-edge sample — NOT the queried anchor, which may be genuinely - // edgeless). If the adjacency was dead and a rebuild healed it, re-collect; if it stays dead, - // verifyGraphAdjacencyLive() throws GraphIndexNotReadyError. A genuinely edgeless anchor - // verifies 'live' and the empty result stands — no spurious rebuild/throw. + // traversal would silently return [] as if it were truth. Re-verify against the health-report/ + // isReady() authority (or, for older providers, a READ-ONLY GLOBAL known-edge sample — NOT the + // queried anchor, which may be genuinely edgeless): a dead adjacency throws + // GraphIndexNotReadyError here rather than serving the empty set as fact — verifyGraphAdjacencyLive + // never rebuilds, so a genuinely edgeless anchor simply verifies 'live' and the empty result stands. if (connectedIds.size === 0) { - const verdict = await this.verifyGraphAdjacencyLive() - if (verdict === 'rebuilt') { - await populate() - } + await this.verifyGraphAdjacencyLive() } // Filter existing results to only connected entities @@ -16228,111 +16205,59 @@ export class Brainy implements BrainyInterface { } /** - * Ensure indexes are loaded (Production-scale lazy loading) + * @description THE READ GATE. Every read choke point (getNeighborUuids, + * find, filterIdsBelted, getTypedNeighbors) calls this before touching a + * derived index. It is a CHECK, never a build: it asks each of the three + * providers (vector, metadata, graph) for its named health verdict via + * {@link assessProviderHealth} — the provider's own sync, O(1) + * `healthReport()` when exposed, else the `isReady()` / size-heuristic + * fallback — and either lets the read proceed or throws the matching typed + * `*NotReadyError` naming the provider and its failing reasons. It NEVER + * triggers a rebuild and NEVER walks the store: a needed rebuild is + * entirely open()'s job (see {@link rebuildIndexesIfNeeded}), which runs to + * completion before `init()` returns — so by the time any read reaches + * this gate, a healthy provider is already built. A migrating provider is + * deferred to exactly as before (it owns its own in-place rebuild). * - * Called by query methods (find, search, get, etc.) when disableAutoRebuild is true. - * Handles concurrent queries safely - multiple calls wait for same rebuild. - * - * Performance: - * - First query: Triggers rebuild (~50-200ms for 1K-10K entities) - * - Concurrent queries: Wait for same rebuild (no duplicate work) - * - Subsequent queries: Instant (0ms check, indexes already loaded) - * - * Production scale: - * - 1K entities: ~50ms - * - 10K entities: ~200ms - * - 100K entities: ~2s (streaming pagination) - * - 1M+ entities: Uses chunked lazy loading (per-type on demand) + * A report with something worth telling an operator (a failing invariant, + * whether serving or not, or a named `unledgered` family) narrates via + * `prodLog.warn` ONCE per (provider, `report.generation`) — never once per + * read — before any throw decision is made. */ - private async ensureIndexesLoaded(): Promise { - // Fast path: If rebuild already completed, return immediately (0ms) - if (this.lazyRebuildCompleted) { - return + private ensureIndexesLoaded(): void { + const providers: ReadonlyArray BrainyError]> = [ + ['vector', this.index, VectorIndexNotReadyError], + ['metadata', this.metadataIndex, MetadataIndexNotReadyError], + ['graph', this.graphIndex, GraphIndexNotReadyError] + ] + + for (const [name, provider, ErrorClass] of providers) { + // Migration LOCK (#18) deference: a migrating provider owns its own + // in-place rebuild — brainy must not judge (or race) it here. + if (this.providerIsMigrating(provider)) continue + + const assessment = assessProviderHealth(provider) + + if (assessment.reasons.length > 0 && assessment.report != null) { + const generation = assessment.report.generation + if (this._lastNarratedHealthGeneration.get(provider) !== generation) { + this._lastNarratedHealthGeneration.set(provider, generation) + prodLog.warn( + `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + + assessment.reasons.join('; ') + ) + } + } + + if (assessment.readiness === 'not-ready') { + throw new ErrorClass( + `${name} index is not serving (via ${assessment.via}): ` + + `${assessment.reasons.join('; ') || 'not ready'}. Reads refuse rather than serve an ` + + `empty result — open() builds the derived indexes; a read never does. Rebuild via ` + + `repairIndex({ rebuild: ['${name}'] }) or reopen the brain.` + ) + } } - - // If indexes already populated AND honestly serving, mark complete and skip. - // Honest gate: when a provider exposes isReady(), that REPLACES the size()>0 - // proxy (a native index can report a non-zero size while its serving structure - // is not loaded — the silent-empty cold-load class). A not-ready provider falls - // through so the rebuild path can load it; verifyVectorLive() is the query-time - // backstop either way. Providers without isReady() keep the size() heuristic - // (the JS index's size()>0 genuinely means loaded). - // - // ALL THREE providers vote (fleet-adoption find, SELF-ENGINE-PAIR-STANDARD): - // this gate used to assess ONLY the vector index, so a not-ready native - // METADATA provider (its strand report) never blocked the completion latch - // — under disableAutoRebuild the promised lazy first-query rebuild never - // fired and every find() silently returned [] on a populated store. A - // not-ready report from ANY provider now falls through to the rebuild. - const vectorReadiness = assessIndexReadiness(this.index) - const metadataReadiness = assessIndexReadiness(this.metadataIndex) - const graphReadiness = assessIndexReadiness(this.graphIndex) - const anyProviderNotReady = - vectorReadiness === 'not-ready' || - metadataReadiness === 'not-ready' || - graphReadiness === 'not-ready' - if ( - !anyProviderNotReady && - (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) - ) { - this.lazyRebuildCompleted = true - return - } - - // Migration LOCK (#18) deference: while the vector provider runs its one-time - // 7.x → 8.0 rebuild-from-canonical, a first query must NOT trigger brainy's - // force-rebuild — the provider owns that index. Normally unreachable here: the - // data-plane lock (awaitMigrationLock) makes the caller wait upstream, so a - // query only reaches this point once the migration has cleared. Defensive - // (no `lazyRebuildCompleted` latch) so the check re-runs: once the provider - // clears the lock, `index.size() > 0` above ends the lazy path normally. - if (this.providerIsMigrating(this.index)) { - return - } - - // Concurrency control: If rebuild is in progress, wait for it - if (this.lazyRebuildInProgress && this.lazyRebuildPromise) { - await this.lazyRebuildPromise - return - } - - // Check if lazy rebuild is needed - // Only needed if: disableAutoRebuild=true AND indexes are empty AND storage has data - if (!this.config.disableAutoRebuild) { - // Auto-rebuild is enabled, indexes should already be loaded - return - } - - // Check if storage has data (fast check with limit=1) - const entities = await this.storage.getNouns({ pagination: { limit: 1 } }) - const hasData = (entities.totalCount && entities.totalCount > 0) || entities.items.length > 0 - - if (!hasData) { - // Storage is empty, no rebuild needed - this.lazyRebuildCompleted = true - return - } - - // Start lazy rebuild (with mutex to prevent concurrent rebuilds). - // ALWAYS narrated (prodLog, never the silent-suppressible console): a - // read that triggers an index build must be visible to the operator — - // fifteen silent minutes of a production blackout taught this line. - prodLog.warn( - `[Brainy] first read on this instance is building the derived indexes ` + - `(deferred at open by disableAutoRebuild) — reads WAIT and then serve; ` + - `nothing serves empty. Bounded by store size; progress under [MetadataIndex]/[GraphIndex].` - ) - this.lazyRebuildInProgress = true - this.lazyRebuildPromise = this.rebuildIndexesIfNeeded(true) - .then(() => { - this.lazyRebuildCompleted = true - }) - .finally(() => { - this.lazyRebuildInProgress = false - this.lazyRebuildPromise = null - }) - - await this.lazyRebuildPromise } /** @@ -16392,7 +16317,13 @@ export class Brainy implements BrainyInterface { } /** - * Rebuild indexes from persisted data if needed (LAZY LOADING) + * @description Rebuild indexes from persisted data if needed — THE OPEN-TIME + * BUILD. Called once per open (init calls it; `repairIndex()`'s + * write-quarantine lift calls it forced). Runs to completion BEFORE `init()` + * returns: a needed rebuild is NEVER deferred to a read (there is no more + * first-query lazy path — see {@link ensureIndexesLoaded}, which is a + * read-time CHECK only). `disableAutoRebuild` no longer defers index + * construction to the first query; see its JSDoc in `brainy.types.ts`. * * FIXES FOR CRITICAL BUGS: * - Bug #1: GraphAdjacencyIndex rebuild never called ✅ FIXED @@ -16402,34 +16333,24 @@ export class Brainy implements BrainyInterface { * * Production-grade rebuild with: * - Handles BILLIONS of entities via streaming pagination - * - Smart threshold-based decisions (auto-rebuild < 1000 items) - * - Lazy loading on first query (when disableAutoRebuild: true) + * - A provider's named {@link HealthReport} (when it exposes one) decides + * per-leg need; `isReady()` / a size heuristic decides otherwise — no + * dataset-size threshold gates whether the rebuild runs at open. * - Progress reporting for large datasets * - Parallel index rebuilds for performance * - Robust error recovery (continues on partial failures) - * - Concurrency-safe (multiple queries wait for same rebuild) * - * @param force - Force rebuild even if disableAutoRebuild is true (for lazy loading) + * @param force - Force the rebuild path to run even when no leg reports a need (used by `repairIndex()`'s write-quarantine lift). */ private async rebuildIndexesIfNeeded(force = false): Promise { try { - // Check if auto-rebuild is explicitly disabled (ONLY during init, not for lazy loading) - // force=true means this is a lazy rebuild triggered by first query - if (this.config.disableAutoRebuild === true && !force) { - if (!this.config.silent) { - console.log('⚡ Auto-rebuild explicitly disabled via config') - console.log('💡 Indexes will build automatically on first query (lazy loading)') - } - return - } - // No instant fast-path here: the honest per-leg readiness checks below - // are all O(1) (one bounded storage sample + each provider's size()/ - // isReady()), and this method runs exactly once per open (init calls it; - // the lazy path passes force=true). The removed shortcut keyed off - // `this.index.size() > 0`, a dishonest proxy — it skipped the metadata - // and graph checks whenever the vector happened to be warm, and it never - // fired on a real cold process (the JS vector size is 0 until it loads). + // are all O(1) (one bounded storage sample + each provider's health + // report / size()/isReady()), and this method runs exactly once per + // open. The removed shortcut keyed off `this.index.size() > 0`, a + // dishonest proxy — it skipped the metadata and graph checks whenever + // the vector happened to be warm, and it never fired on a real cold + // process (the JS vector size is 0 until it loads). // BUG #2 FIX: Don't trust counts - check actual storage instead // Counts can be lost/corrupted in container restarts @@ -16448,30 +16369,23 @@ export class Brainy implements BrainyInterface { return } - // Intelligent decision: Auto-rebuild based on dataset size - // Production scale: Handles billions via streaming pagination - const AUTO_REBUILD_THRESHOLD = 10000 // Auto-rebuild if < 10K items (increased from 1K) - // Check if indexes need rebuilding const metadataStats = await this.metadataIndex.getStats() const hnswIndexSize = this.index.size() - // Readiness contract: when a provider exposes isReady(), that honest - // signal REPLACES the size/count heuristic below — an mmap/disk-native - // index legitimately reports 0 resident entries while fully durable on - // disk, and rebuilding it from canonical re-reads every entity file on - // every boot (the 48-seconds-per-restart class a production deployment - // hit). The signal is honest in BOTH directions: a provider whose - // durable state failed to load returns false and gets its rebuild even - // when size() > 0 (the silent-empty cold-load failure). Providers - // without isReady() keep the exact prior empty-heuristics. - const providerReady = (leg: unknown): boolean | undefined => { - const candidate = leg as { isReady?: () => boolean } - return typeof candidate.isReady === 'function' ? candidate.isReady() : undefined + // Readiness contract: a provider's named {@link HealthReport} (when + // exposed) is the authority — `serving === false` needs the rebuild, + // full stop. Absent a health report, fall back to `isReady()` (an + // mmap/disk-native index legitimately reports 0 resident entries while + // fully durable on disk, so rebuilding it from canonical on every boot + // would be the 48-seconds-per-restart class a production deployment + // hit); absent BOTH, keep the per-leg empty-heuristic passed in. + const legNeedsRebuild = (provider: unknown, emptyFallback: boolean): boolean => { + const assessment = assessProviderHealth(provider) + if (assessment.via === 'health-report') return assessment.readiness !== 'ready' + if (assessment.via === 'is-ready') return assessment.readiness === 'not-ready' + return emptyFallback } - const metadataReady = providerReady(this.metadataIndex) - const vectorReady = providerReady(this.index) - const graphReady = providerReady(this.graphIndex) // Epoch-drift trigger: a format-version change makes EVERY derived index // suspect even when each is non-empty, so it forces a rebuild of all @@ -16491,9 +16405,9 @@ export class Brainy implements BrainyInterface { const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating // Per-leg decision, in precedence order: a migrating provider owns its - // index (skip) → epoch drift forces a rebuild → an exposed isReady() - // decides → otherwise a per-leg fallback. The fallbacks differ by leg - // because "empty" means different things: + // index (skip) → epoch drift forces a rebuild → the health-report/ + // isReady() authority decides → otherwise a per-leg fallback. The + // fallbacks differ by leg because "empty" means different things: // - METADATA: past the empty-store early-return, entities exist, so the // id-mapper SHOULD have loaded entries — totalEntries===0 is a real // load-failure signal, so rebuild (self-heal from canonical). @@ -16504,62 +16418,49 @@ export class Brainy implements BrainyInterface { // against canonical) inside storage.getGraphIndex() BEFORE this gate, // so it is already authoritative here; re-deriving would be spurious // (a full O(E) verb scan on every open of an edgeless brain). It - // therefore rebuilds only on epoch drift or a native !isReady(). - // (verifyGraphAdjacencyLive is the query-time backstop.) + // therefore rebuilds only on epoch drift or a native !isReady()/ + // not-serving report. (verifyGraphAdjacencyLive is the query-time + // backstop — it refuses loudly, it never rebuilds.) const shouldRebuildMetadata = !metadataMigrating && - (epochStale || - (metadataReady !== undefined ? !metadataReady : metadataStats.totalEntries === 0)) + (epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0)) const shouldRebuildVector = !vectorMigrating && - (epochStale || (vectorReady !== undefined ? !vectorReady : hnswIndexSize === 0)) + (epochStale || legNeedsRebuild(this.index, hnswIndexSize === 0)) const shouldRebuildGraph = !graphMigrating && - (epochStale || (graphReady !== undefined ? !graphReady : false)) + (epochStale || legNeedsRebuild(this.graphIndex, false)) const needsRebuild = shouldRebuildMetadata || shouldRebuildVector || shouldRebuildGraph if (!needsRebuild && !force) { - // All indexes report current — durably loaded (isReady/size), or owned - // by a background migration. No rebuild needed. + // All indexes report current — durably loaded (health-report/isReady/ + // size), or owned by a background migration. No rebuild needed. return } - // Determine rebuild strategy - const isLazyRebuild = force && this.config.disableAutoRebuild === true - const isSmallDataset = totalCount < AUTO_REBUILD_THRESHOLD - const shouldRebuild = isLazyRebuild || isSmallDataset || this.config.disableAutoRebuild === false + // Name exactly which legs rebuild — "all indexes" was a lie whenever + // the durable legs were skipped (e.g. only the JS vector index loads + // here on a warm reopen), and it misread as a whole-brain rebuild in + // consumer boot logs. + const rebuildingLegs = [ + shouldRebuildMetadata && 'metadata', + shouldRebuildVector && 'vector', + shouldRebuildGraph && 'graph' + ] + .filter(Boolean) + .join(' + ') - if (!shouldRebuild) { - // Large dataset with auto-rebuild disabled: Wait for lazy loading - if (!this.config.silent) { - console.log(`⚡ Large dataset (${totalCount.toLocaleString()} items) - using lazy loading for optimal startup`) - console.log('💡 Indexes will build automatically on first query') - } - return - } - - // REBUILD: Either small dataset, forced rebuild, or explicit enable - const rebuildReason = isLazyRebuild - ? '🔄 Lazy loading triggered by first query' - : isSmallDataset - ? `🔄 Small dataset (${totalCount.toLocaleString()} items)` - : '🔄 Auto-rebuild explicitly enabled' - - if (!this.config.silent) { - // Name exactly which legs rebuild — "all indexes" was a lie whenever - // the durable legs were skipped (e.g. only the JS vector index loads - // here on a warm reopen), and it misread as a whole-brain rebuild in - // consumer boot logs. - const rebuildingLegs = [ - shouldRebuildMetadata && 'metadata', - shouldRebuildVector && 'vector', - shouldRebuildGraph && 'graph' - ] - .filter(Boolean) - .join(' + ') - console.log(`${rebuildReason} - loading/rebuilding ${rebuildingLegs || 'no'} index(es) from persisted data...`) - } + // ALWAYS narrated (prodLog, never the silent-suppressible console): there + // is no more first-query lazy path — a rebuild that runs here BLOCKS + // open() regardless of dataset size or `disableAutoRebuild`, so an + // operator must see it in the boot log, not discover it as an + // unexplained slow open. + prodLog.warn( + `[Brainy] open() is building/rebuilding the ${rebuildingLegs || 'no'} index(es) from ` + + `${totalCount.toLocaleString()} stored entities — open blocks until the derived ` + + `indexes serve; reads never build.` + ) // Before the graph rebuild, hydrate the entity id-mapper from the persisted // snapshot. A native int-keyed adjacency resolves every verb endpoint through @@ -16586,13 +16487,23 @@ export class Brainy implements BrainyInterface { const rebuildDuration = Date.now() - rebuildStartTime const metadataCountAfter = (await this.metadataIndex.getStats()).totalEntries + const graphSizeAfter = await this.graphIndex.size() + + // Completion narration — ALWAYS via prodLog (see the pre-rebuild narration + // above for why): the operator who saw "open() is building…" needs the + // matching "…and it's done" line, with the numbers to confirm it worked. + prodLog.warn( + `[Brainy] open() finished building derived indexes in ${rebuildDuration}ms: ` + + `metadata=${metadataCountAfter} entries, vector=${this.index.size()} nodes, ` + + `graph=${graphSizeAfter} relationships.` + ) if (!this.config.silent) { console.log( `All indexes rebuilt in ${rebuildDuration}ms:\n` + ` - Metadata: ${metadataCountAfter} entries\n` + ` - HNSW Vector: ${this.index.size()} nodes\n` + - ` - Graph Adjacency: ${await this.graphIndex.size()} relationships` + ` - Graph Adjacency: ${graphSizeAfter} relationships` ) } @@ -16998,49 +16909,6 @@ export class Brainy implements BrainyInterface { return result } - /** - * Run the optional metadata cold-open consistency probe at most once per brain. - * When the active provider exposes `probeConsistency()` (the native cross-bucket - * O(1) sampler), a `false` result triggers `detectAndRepairCorruption()` so an - * already-poisoned index self-heals on first read — the metadata counterpart of - * the 7.33.2 graph cold-load guard. Best-effort: a probe failure never breaks the - * read (the guard is reset so a transient failure retries). No-op for the JS index - * (it exposes no probe), and the full-scan `validateConsistency` stays the explicit - * deep diagnostic via `validateIndexConsistency()`. - */ - private async ensureMetadataConsistencyProbed(): Promise { - if (this._metadataConsistencyProbed) return - // Defer while the metadata provider runs its one-time in-place migration: - // probing (and self-healing via rebuild) an index the provider is mid-rebuild - // would collide with the provider that owns it. Mirrors the vector deference - // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next - // read runs the probe. (The family-scoped find() gate waits on the metadata - // family separately before any actual filter read.) - if (this.providerIsMigrating(this.metadataIndex)) return - this._metadataConsistencyProbed = true - const provider = this.metadataIndex as { - probeConsistency?: () => Promise - detectAndRepairCorruption?: () => Promise - } - if (typeof provider.probeConsistency !== 'function') return - try { - const healthy = await provider.probeConsistency() - if (!healthy && typeof provider.detectAndRepairCorruption === 'function') { - if (!this.config.silent) { - console.warn('[Brainy] metadata index failed the cold-open consistency probe — self-healing via rebuild.') - } - await provider.detectAndRepairCorruption() - } - } catch (error) { - // The self-heal is best-effort and must never break a read. Reset the guard - // so a transient probe failure is retried on the next read. - this._metadataConsistencyProbed = false - if (!this.config.silent) { - console.warn('[Brainy] metadata cold-open consistency probe failed (continuing):', error) - } - } - } - /** * Detect and repair corrupted metadata indexes. * @@ -17166,7 +17034,28 @@ export class Brainy implements BrainyInterface { ) } - async repairIndex(): Promise { + /** + * @description The ceremony door for index repair. Bare `repairIndex()` is + * REPORT-DRIVEN, exactly as before: it prunes orphans, recomputes count + * rollups, reconciles VFS containment, and — for the three derived-index + * providers — consults each one's `validateInvariants()` and rebuilds only + * a family whose failing invariant asks for it (`heal: 'rebuild'`). + * + * `options.rebuild` is the EXPLICIT operator override: name one or more + * families (or `'all'`) to rebuild them UNCONDITIONALLY — no invariant is + * consulted, JS or native provider alike. Use it when an operator has + * independent reason to believe a family needs reconciling regardless of + * what its own self-report says (a report can only be as honest as the + * provider that produced it). A family named here is recorded as its own + * `provider:` row with `rebuilt: true` and + * `reason: 'explicit rebuild requested'`, and is SKIPPED by the normal + * invariant-driven pass (it was already rebuilt unconditionally — a second, + * report-driven pass over the same family would be redundant at best). + * + * @param options.rebuild - Family name(s) to unconditionally rebuild, or `'all'` for all three (`'metadata' | 'graph' | 'vector'`). + * @returns The full per-family receipt (see {@link RepairReport}); also narrated via `prodLog.warn`. + */ + async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { await this.ensureInitialized() const startedAt = Date.now() const families: RepairFamilyReport[] = [] @@ -17271,17 +17160,52 @@ export class Brainy implements BrainyInterface { `Writes are re-enabled.` ) } + // THE CEREMONY DOOR: an explicit `options.rebuild` names a family (or + // 'all') to rebuild UNCONDITIONALLY — no invariant consulted. Resolved + // here so the loop below can skip a family's normal report-driven pass + // once its unconditional rebuild has already run. + const explicitRebuildFamilies: ReadonlySet<'metadata' | 'vector' | 'graph'> = + options?.rebuild === 'all' + ? new Set<'metadata' | 'vector' | 'graph'>(['metadata', 'vector', 'graph']) + : new Set(options?.rebuild ?? []) + // Cross-layer repair: repairIndex must reconcile NATIVE derived // state from canonical, not just the JS metadata index. Consult each provider's // own validateInvariants() and rebuild any whose failing invariant asks for it // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption(). - for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { + const providerFamilies: ReadonlyArray = [ + ['metadata', this.metadataIndex], + ['vector', this.index], + ['graph', this.graphIndex] + ] + for (const [familyName, provider] of providerFamilies) { + if (explicitRebuildFamilies.has(familyName)) { + const p = provider as { rebuild?: () => Promise } | null + if (!p || typeof p.rebuild !== 'function') { + record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no rebuild() contract' }) + continue + } + prodLog.warn( + `[Brainy] repairIndex(): explicit rebuild requested for '${familyName}' — ` + + `rebuilding unconditionally (no invariant consulted).` + ) + await p.rebuild() + record(`provider:${familyName}`, { + checked: true, + healed: 1, + rebuilt: true, + reason: 'explicit rebuild requested' + }) + prodLog.warn(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) + continue + } + const p = provider as { validateInvariants?: () => Promise rebuild?: () => Promise } | null if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') { - record(`provider:${(provider as { constructor?: { name?: string } })?.constructor?.name ?? 'unknown'}`, { + record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract' }) continue @@ -17290,7 +17214,7 @@ export class Brainy implements BrainyInterface { try { report = await p.validateInvariants() } catch (err) { - record(`provider:unknown`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` }) + record(`provider:${familyName}`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` }) continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here } if (report.healthy) { diff --git a/src/index.ts b/src/index.ts index 765c20b4..07f318c0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -269,6 +269,10 @@ export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.j export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js' +// The named, synchronous, O(1) health-report contract (the read gate's ONLY +// source of truth for "can I serve right now") — see HealthReport's +// derivation laws in plugin.ts. +export type { HealthReport, LedgerInvariantResult, InvariantSource } from './plugin.js' // Optional provider self-report of outstanding background maintenance work // (compaction, deferred writes, etc.) — the payload type for // brain.maintenanceDebt(). See the measure-only-what-you-track contract on diff --git a/src/plugin.ts b/src/plugin.ts index 947c86a5..47a559b6 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -171,6 +171,66 @@ export interface ProviderInvariantReport { durationMs: number } +/** + * @description Where a {@link LedgerInvariantResult} verdict came from: + * - `'ledger'` — decided from an exact, durable ledger (a real count, not a sample). + * - `'deep'` — decided by a full/expensive scan (the `validateInvariants()` diagnostic path only). + * - `'unledgered'` — this family has no ledger yet; the verdict is UNKNOWN, never healthy and never broken. + */ +export type InvariantSource = 'ledger' | 'deep' | 'unledgered' + +/** + * @description One invariant verdict inside a {@link HealthReport}. Extends + * {@link InvariantResult} with the provenance of the verdict ({@link InvariantSource}) + * and, for a failing set-membership invariant, an exact count plus a capped sample + * of the diverging ids — a VERDICT, never a dump. `sample` MUST be capped at 16 ids; + * `count` is the exact number even when `sample` is truncated. + */ +export interface LedgerInvariantResult extends InvariantResult { + /** Provenance of this verdict — see {@link InvariantSource}. */ + source: InvariantSource + /** Exact count of diverging/missing items plus a capped (≤16 ids) sample. Present only on a failing set-membership invariant. */ + missing?: { count: number; sample: string[] } +} + +/** + * @description The NAMED, SYNCHRONOUS, O(1) health report a provider exposes via + * {@link MetadataIndexProvider.healthReport} / {@link GraphIndexProvider.healthReport} / + * {@link VectorIndexProvider.healthReport}. This is the read gate's ONLY source of + * truth for "can I serve right now" — it replaces sampled self-probes and the + * unnamed `isReady()` latch with an exact, ledger-derived verdict. + * + * Derivation laws (a provider MUST honor these; brainy's read gate assumes them): + * - `healthy` = every VERIFIED invariant in {@link invariants} holds. An invariant + * whose family is named in {@link unledgered} is NEVER counted toward `healthy` + * either way — it is unknown, not passing. + * - `serving` = no verified invariant in {@link invariants} FAILS with `heal: 'rebuild'`. + * A failure with `heal: 'repair'` or `heal: 'none'` is degraded-but-serving — + * `serving` stays `true`. Only a `'rebuild'`-grade failure makes `serving` `false`. + * - `validateInvariants()` remains the async DEEP diagnostic (full scans allowed, + * `source: 'deep'` results); `healthReport()` MUST be synchronous, O(1) from + * exact ledgers/counters, and MUST NOT throw for a well-formed provider — a + * provider that cannot produce a safe verdict reports it as a failing invariant, + * it does not throw (a throw is read by the gate as a CONTRACT VIOLATION, not as + * "unknown"). + */ +export interface HealthReport extends ProviderInvariantReport { + /** + * Monotonic per provider: bumps on every ledger mutation and every rebuild + * boundary. Consumers (the read gate's narration dedup, external callers) may + * cache a verdict per generation. + */ + generation: number + /** Each checked invariant, with provenance — see {@link LedgerInvariantResult}. */ + invariants: LedgerInvariantResult[] + /** + * Families with no ledger yet. NAMED here so an operator can see what is not + * yet tracked — NEVER counted as healthy (they are not verified) and NEVER + * counted as broken (there is nothing to fail). + */ + unledgered: string[] +} + /** * @description A provider's self-report of its own outstanding background * maintenance work (compaction, deferred writes, a build-new→verify→swap in @@ -266,6 +326,20 @@ export interface MetadataIndexProvider { */ validateInvariants?(): Promise + /** + * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this + * provider derives from its own exact ledgers — see {@link HealthReport} for + * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a + * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never + * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) + * reads THIS instead of `isReady()` / size heuristics: `serving` decides + * whether reads may proceed; a `false` refuses the read loudly rather than + * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the + * size heuristic (this train's JS built-in providers stay on that interim + * path). + */ + healthReport?(): HealthReport + /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background @@ -462,6 +536,20 @@ export interface GraphIndexProvider { */ validateInvariants?(): Promise + /** + * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this + * provider derives from its own exact ledgers — see {@link HealthReport} for + * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a + * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never + * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) + * reads THIS instead of `isReady()` / size heuristics: `serving` decides + * whether reads may proceed; a `false` refuses the read loudly rather than + * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the + * size heuristic (this train's JS built-in providers stay on that interim + * path). + */ + healthReport?(): HealthReport + /** * @description OPTIONAL eager cold-load. Called once during brain init — AFTER * the metadata provider's `init()` (so the id-mapper is hydrated; a native int @@ -1225,6 +1313,20 @@ export interface VectorIndexProvider { */ validateInvariants?(): Promise + /** + * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this + * provider derives from its own exact ledgers — see {@link HealthReport} for + * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a + * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never + * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) + * reads THIS instead of `isReady()` / size heuristics: `serving` decides + * whether reads may proceed; a `false` refuses the read loudly rather than + * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the + * size heuristic (this train's JS built-in providers stay on that interim + * path). + */ + healthReport?(): HealthReport + /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 78e310f3..38bc4aa6 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1816,10 +1816,16 @@ export interface BrainyConfig { | StorageAdapter /** - * Disable the automatic index rebuild check during `init()`. By default - * Brainy auto-decides from dataset size: small datasets rebuild missing - * indexes inline, large datasets rebuild lazily on first query. Set `true` - * only when an operator wants full manual control via `repairIndex()`. + * RE-MEANT (the health-gate contract): `init()` (open) always verifies the + * durable generation of every derived index, and a needed rebuild ALWAYS + * runs at open — it is never deferred to the first read, regardless of + * dataset size or this flag. There is no first-query lazy-build path + * anymore: a read that finds a provider not serving throws a typed + * `*NotReadyError` rather than building anything (see + * `assessProviderHealth` / the read gate in `brainy.ts`). Setting this + * `true` no longer defers index construction to the first query — it has + * no effect on WHEN a needed rebuild runs. Full manual control over + * rebuilds remains available via `repairIndex({ rebuild: [...] })`. */ disableAutoRebuild?: boolean diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts index 16266bec..498f2003 100644 --- a/src/utils/indexReadiness.ts +++ b/src/utils/indexReadiness.ts @@ -13,13 +13,28 @@ * `size()` or `isInitialized`. When `isReady()` is absent, callers must fall back * to a KNOWN-ITEM PROBE (a real search/lookup that must return a known-present * datum) before trusting an empty result — never a `size()` proxy. + * + * {@link assessProviderHealth} is the NEWER, PREFERRED authority: it reads a + * provider's NAMED, synchronous, O(1) {@link import('../plugin.js').HealthReport} + * when one is exposed, and falls back to this file's `isReady()` classifier only + * when the provider does not (yet) expose a health report. Read paths in + * `brainy.ts` call `assessProviderHealth` exclusively — `assessIndexReadiness` + * stays exported for the other call sites (`storage/baseStorage.ts`) and for the + * fallback branch inside `assessProviderHealth` itself. */ +import type { HealthReport } from '../plugin.js' + /** A provider that MAY expose the honest cold-load readiness signal. */ export interface MaybeReadyProvider { isReady?: () => boolean } +/** A provider that MAY expose the named, synchronous, O(1) health report. */ +export interface MaybeHealthReportingProvider { + healthReport?: () => HealthReport +} + /** Three-valued honest-readiness verdict. */ export type IndexReadiness = 'ready' | 'not-ready' | 'unknown' @@ -36,3 +51,105 @@ export function assessIndexReadiness(provider: unknown): IndexReadiness { if (p == null || typeof p.isReady !== 'function') return 'unknown' return p.isReady() ? 'ready' : 'not-ready' } + +/** + * @description Which signal {@link assessProviderHealth} actually consulted to + * produce its verdict — surfaced so callers can narrate (and tests can pin) how + * a provider was judged, not just what the judgment was. + * - `'health-report'` — the provider's `healthReport()` was called (the authority). + * - `'is-ready'` — no `healthReport()`; fell back to the provider's `isReady()`. + * - `'size-heuristic'` — no `healthReport()` and no `isReady()`; caller must keep its own size-based heuristic. + * - `'none'` — there was no provider to assess (`null`/`undefined`). + */ +export type ProviderHealthVia = 'health-report' | 'is-ready' | 'size-heuristic' | 'none' + +/** The result of {@link assessProviderHealth}. */ +export interface ProviderHealthAssessment { + /** The honest readiness verdict — see {@link IndexReadiness}. */ + readiness: IndexReadiness + /** The provider's raw {@link HealthReport}, when one was obtained; `null` otherwise. */ + report: HealthReport | null + /** Which signal produced the verdict — see {@link ProviderHealthVia}. */ + via: ProviderHealthVia + /** Human-readable reasons: named failing invariants (with `heal`), unledgered families, or the fallback-path explanation. Empty when the provider is healthy and ready. */ + reasons: string[] +} + +/** + * @description THE read-gate authority. Prefers a provider's NAMED, + * synchronous, O(1) {@link HealthReport} over the older `isReady()` / size + * heuristics; falls back to {@link assessIndexReadiness}'s semantics only when + * a provider does not (yet) expose `healthReport()`. + * + * Derivation: + * - `healthReport()` present → call it (wrapped in try/catch). A THROW is a + * CONTRACT VIOLATION, not "unknown": returns `readiness: 'not-ready'`, + * `via: 'health-report'`, and a reason naming the throw — never swallowed + * into `'unknown'`. + * - Otherwise → `readiness = report.serving ? 'ready' : 'not-ready'`; `reasons` + * names every invariant with `holds: false` (with its `heal`), plus an + * `unledgered: [...]` line when {@link HealthReport.unledgered} is non-empty. + * UNLEDGERED IS UNKNOWN: an unledgered family never flips a serving provider + * to not-ready, and never flips a not-serving provider to ready — `serving` + * is always the provider's own verdict, verbatim. + * - No `healthReport()` → fall back to {@link assessIndexReadiness}'s semantics: + * `via: 'is-ready'` when `isReady()` exists, `via: 'size-heuristic'` when + * neither hook exists (caller must keep its own size-based heuristic), + * `via: 'none'` when there is no provider at all. + * @param provider - Any index provider (vector / graph / metadata) or `null`/`undefined`. + */ +export function assessProviderHealth(provider: unknown): ProviderHealthAssessment { + const p = provider as (MaybeHealthReportingProvider & MaybeReadyProvider) | null | undefined + + if (p == null) { + return { readiness: 'unknown', report: null, via: 'none', reasons: ['no provider to assess'] } + } + + if (typeof p.healthReport === 'function') { + let report: HealthReport + try { + report = p.healthReport() + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + return { + readiness: 'not-ready', + report: null, + via: 'health-report', + reasons: [`healthReport() threw: ${message} — a health-report throw is a contract violation, never read as healthy`] + } + } + + const reasons: string[] = [] + for (const invariant of report.invariants) { + if (!invariant.holds) { + reasons.push(`${invariant.name} (heal:${invariant.heal}): ${invariant.detail}`) + } + } + if (report.unledgered.length > 0) { + reasons.push(`unledgered: ${report.unledgered.join(', ')}`) + } + + return { + readiness: report.serving ? 'ready' : 'not-ready', + report, + via: 'health-report', + reasons + } + } + + const readiness = assessIndexReadiness(p) + if (readiness === 'unknown') { + return { + readiness, + report: null, + via: 'size-heuristic', + reasons: ['provider exposes neither healthReport() nor isReady() — falling back to the size heuristic'] + } + } + return { + readiness, + report: null, + via: 'is-ready', + reasons: readiness === 'not-ready' ? ['isReady() returned false'] : [] + } +} diff --git a/tests/configs/vitest.integration.config.ts b/tests/configs/vitest.integration.config.ts index 3d3a3721..af86097d 100644 --- a/tests/configs/vitest.integration.config.ts +++ b/tests/configs/vitest.integration.config.ts @@ -20,6 +20,9 @@ export default defineConfig({ // Include only integration tests include: [ 'tests/integration/**/*.test.ts', + // The lifecycle biography lane (day-in-the-life scenarios; see + // tests/lifecycle/README.md) runs in the integration gate. + 'tests/lifecycle/**/*.test.ts', 'tests/**/*.integration.test.ts' ], diff --git a/tests/integration/cold-graph-connected-8.0.test.ts b/tests/integration/cold-graph-connected-8.0.test.ts index 71ae345d..7ee725f5 100644 --- a/tests/integration/cold-graph-connected-8.0.test.ts +++ b/tests/integration/cold-graph-connected-8.0.test.ts @@ -1,29 +1,29 @@ /** * @module tests/integration/cold-graph-connected-8.0 * @description BRAINY-COLD-GRAPH-CONNECTED (8.0) — regression coverage for the silent-empty - * graph-traversal bug, gated on the converged 8.0 contract: a sync `graphIndex.isReady()` that - * is true ONLY when the source→target EDGES are loaded (NOT the membership/manifest count). + * graph-traversal bug, gated on the honest readiness signal: a sync `graphIndex.isReady()` + * that is true ONLY when the source→target EDGES are loaded (NOT the membership/manifest count). * - * On the FIRST `find({ connected })` after a cold process start of a LARGE brain (≥10k nouns, - * which skips the eager index rebuild), a native graph adjacency can reload its relationship - * COUNT (so `size() > 0`) but NOT its edges — so `getNeighbors()` returns `[]` for EVERY source - * and brainy would serve that `[]` as if the anchor were genuinely edgeless. + * On the FIRST `find({ connected })` after a cold process start, a native graph adjacency can + * reload its relationship COUNT (so `size() > 0`) but NOT its edges — so `getNeighbors()` returns + * `[]` for EVERY source and brainy would serve that `[]` as if the anchor were genuinely edgeless. * - * The 8.0 guard (`verifyGraphAdjacencyLive`) prefers the honest `isReady()` signal: - * - `isReady() === false` → hydrate the id-mapper, rebuild from storage, re-check; a still-false - * `isReady()` throws {@link GraphIndexNotReadyError} instead of returning `[]` ('rebuilt' when - * the rebuild heals it); + * RE-POINTED to the health-gate law: `verifyGraphAdjacencyLive` NEVER rebuilds and NEVER walks the + * store from a read — a read-path rebuild is exactly the dark-rebuild failure mode the law retires + * (open() alone owns building). The guard now: + * - `isReady() === false` → THROWS {@link GraphIndexNotReadyError} immediately — no rebuild attempt; * - a genuinely edgeless anchor with `isReady() === true` verifies 'live' and the empty result - * stands — no spurious rebuild, no throw; - * - a provider WITHOUT `isReady()` falls back to the shipped 7.x known-edge-sample probe. + * stands — no spurious throw; + * - a provider WITHOUT `isReady()` falls back to the shipped known-edge-sample probe, which is + * now READ-ONLY: it refuses loudly (throws) rather than self-healing via rebuild. * * These exercise REAL `find({ connected })` against an in-memory brain whose graph index is - * instrumented with a test-double `isReady()` (and, for the fallback case, an empty-then-healed + * instrumented with a test-double `isReady()` (and, for the fallback case, an always-empty * `getNeighbors`). Only the readiness/edge surface is wrapped; the underlying real adjacency - * (built by `relate()`) is unmasked once a rebuild "heals" it. + * (built by `relate()`) is what a healthy provider actually serves. */ -import { describe, it, expect, afterEach } from 'vitest' +import { describe, it, expect, afterEach, vi } from 'vitest' import { Brainy } from '../../src/index.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' import { GraphIndexNotReadyError } from '../../src/errors/brainyError.js' @@ -63,17 +63,17 @@ async function buildBrain( } /** - * Instrument the brain's real graph index with a test-double `isReady()` (the 8.0 contract) plus - * an edge surface that goes empty while NOT ready. `getNeighbors` returns `[]` while `!ready` - * (modelling the cold-unloaded adjacency) and delegates to the REAL index once a rebuild flips - * `ready` on. `rebuild` is counted; it heals (`ready = true`) only when `healsOnRebuild` is set. - * Pass `failFirstRebuild` to make the FIRST rebuild throw a transient error (without healing) so - * the empty-result re-collect path in executeGraphSearch is exercised. + * Instrument the brain's real graph index with a test-double `isReady()` (the honest-readiness + * contract) plus an edge surface that goes empty while NOT ready. `getNeighbors` returns `[]` + * while `!ready` (modelling the cold-unloaded adjacency) and delegates to the REAL index once + * `ready` flips true (used only by the "healthy" control cases — the guard itself never flips + * this anymore, since it never rebuilds). `rebuild` is counted so tests can assert it is NEVER + * called by a read. */ function instrumentIsReady( brain: any, - opts: { ready: boolean; healsOnRebuild: boolean; failFirstRebuild?: boolean } -): { rebuildCalls: number } { + opts: { ready: boolean } +): { rebuildCalls: number; ready: boolean } { const gi = brain.graphIndex const origGetNeighbors = gi.getNeighbors.bind(gi) const state = { ready: opts.ready, rebuildCalls: 0 } @@ -85,10 +85,6 @@ function instrumentIsReady( gi.rebuild = async (): Promise => { state.rebuildCalls++ - if (opts.failFirstRebuild && state.rebuildCalls === 1) { - throw new Error('transient rebuild hiccup') - } - if (opts.healsOnRebuild) state.ready = true // unmask the real (already-populated) adjacency } return state @@ -96,12 +92,12 @@ function instrumentIsReady( /** * Fallback instrumentation — a provider WITHOUT `isReady()` (older cortex / JS baseline). Wraps - * `getNeighbors` to return `[]` while `broken` and delegates to the REAL index once a rebuild - * heals it. This is the shipped 7.x known-edge-sample probe path on 8.0. + * `getNeighbors` to always return `[]` while `broken`. This is the shipped known-edge-sample + * probe path — now READ-ONLY: it refuses loudly rather than self-healing. */ function instrumentNoIsReady( brain: any, - opts: { broken: boolean; healsOnRebuild: boolean } + opts: { broken: boolean } ): { rebuildCalls: number } { const gi = brain.graphIndex // Ensure the provider does NOT expose isReady() — the default JS provider doesn't. @@ -114,13 +110,12 @@ function instrumentNoIsReady( gi.rebuild = async (): Promise => { state.rebuildCalls++ - if (opts.healsOnRebuild) state.broken = false } return state } -describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a silent []', () => { +describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a silent [], never rebuilds from a read', () => { let brains: any[] = [] afterEach(async () => { for (const b of brains) { @@ -131,35 +126,37 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si } } brains = [] + vi.restoreAllMocks() }) - it('(a) isReady() false → rebuild heals it true → find({ connected }) returns correct N (rebuilt)', async () => { - const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) - brains.push(brain) - const state = instrumentIsReady(brain, { ready: false, healsOnRebuild: true }) - - const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) - - expect(state.rebuildCalls).toBeGreaterThanOrEqual(1) // detected not-ready + healed it - const ids = results.map((r: any) => r.id).sort() - expect(ids).toEqual(targetIds.sort()) // B, C, D — the real edges, served after the heal - }) - - it('(b) isReady() stays false after rebuild → throws GraphIndexNotReadyError (NOT a silent [])', async () => { + it('(a) isReady() false → THROWS GraphIndexNotReadyError immediately, no rebuild attempt', async () => { const { brain, anchorId } = await buildBrain({ anchorEdges: true }) brains.push(brain) - instrumentIsReady(brain, { ready: false, healsOnRebuild: false }) // rebuild never makes it ready + const state = instrumentIsReady(brain, { ready: false }) await expect( brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) ).rejects.toBeInstanceOf(GraphIndexNotReadyError) + + expect(state.rebuildCalls).toBe(0) // a read never rebuilds — it refuses loudly instead + }) + + it('(b) isReady() stays false → throws GraphIndexNotReadyError (NOT a silent [])', async () => { + const { brain, anchorId } = await buildBrain({ anchorEdges: true }) + brains.push(brain) + const state = instrumentIsReady(brain, { ready: false }) + + await expect( + brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + ).rejects.toBeInstanceOf(GraphIndexNotReadyError) + expect(state.rebuildCalls).toBe(0) }) it('(c) edgeless anchor + isReady() true → returns [] with NO rebuild and NO throw', async () => { // The anchor has no edges, but E -> F does — the adjacency is genuinely loaded (ready). const { brain, anchorId } = await buildBrain({ anchorEdges: false }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: true, healsOnRebuild: false }) + const state = instrumentIsReady(brain, { ready: true }) const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) @@ -170,7 +167,7 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si it('(d) healthy isReady() true → correct results, NO rebuild', async () => { const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: true, healsOnRebuild: false }) + const state = instrumentIsReady(brain, { ready: true }) const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) @@ -179,30 +176,30 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si expect(ids).toEqual(targetIds.sort()) }) - it('(e) provider WITHOUT isReady() → falls back to the known-edge-sample probe (self-heals)', async () => { - const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + it('(e) provider WITHOUT isReady() → the known-edge-sample probe REFUSES LOUDLY (never self-heals)', async () => { + const { brain, anchorId } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentNoIsReady(brain, { broken: true, healsOnRebuild: true }) + const state = instrumentNoIsReady(brain, { broken: true }) - const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + await expect( + brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + ).rejects.toBeInstanceOf(GraphIndexNotReadyError) - expect(state.rebuildCalls).toBeGreaterThanOrEqual(1) // detected the empty adjacency + healed it - const ids = results.map((r: any) => r.id).sort() - expect(ids).toEqual(targetIds.sort()) // B, C, D — served after the heal + expect(state.rebuildCalls).toBe(0) // the fallback probe is READ-ONLY — it never calls rebuild() }) - it('(f) executeGraphSearch re-collect: a transient first rebuild leaves connectedIds empty; the empty-result guard then heals + re-collects', async () => { - // First verify (inside neighbors()) hits a transient rebuild failure → returns 'live' without - // healing, so getNeighbors stays empty and connectedIds is empty. The empty connectedIds set - // then drives executeGraphSearch's own verify, whose rebuild now heals → 'rebuilt' → re-collect. - const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + it('(f) an empty connectedIds set re-verifies against a not-serving adjacency and throws, rather than serving [] as truth', async () => { + // executeGraphSearch's cold-load guard (connectedIds.size === 0 → re-verify) used to + // interpret a healed rebuild as "re-collect and serve." That rebuild-and-heal path is + // retired: the re-verify now either confirms a genuinely edgeless anchor ('live', case (c)) + // or — as here — discovers the adjacency itself is not serving, and throws. + const { brain, anchorId } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: false, healsOnRebuild: true, failFirstRebuild: true }) + const state = instrumentIsReady(brain, { ready: false }) - const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) - - expect(state.rebuildCalls).toBeGreaterThanOrEqual(2) // first transient, second heals - const ids = results.map((r: any) => r.id).sort() - expect(ids).toEqual(targetIds.sort()) // re-collected after the heal + await expect( + brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + ).rejects.toBeInstanceOf(GraphIndexNotReadyError) + expect(state.rebuildCalls).toBe(0) }) }) diff --git a/tests/integration/health-gate.test.ts b/tests/integration/health-gate.test.ts new file mode 100644 index 00000000..4c4fb454 --- /dev/null +++ b/tests/integration/health-gate.test.ts @@ -0,0 +1,352 @@ +/** + * @module tests/integration/health-gate + * @description Pins for the health-by-accounting read gate: the read gate stops + * consulting an unnamed `isReady()` boolean and reads a NAMED, sync, O(1) + * {@link HealthReport}; no read path may ever start a store walk; the open path + * brings every provider to serving before it returns; an explicit operator door + * (`repairIndex({ rebuild: [...] })`) rebuilds a named leg unconditionally. + * + * Providers here are white-box test doubles: a `healthReport()` (or, for the + * interim-path pins, an `isReady()`) function assigned directly onto the LIVE + * JS provider object, the same pattern `tests/unit/validate-invariants-delegation.test.ts` + * uses for `validateInvariants`. This exercises brainy's real gate/verify code + * against a controlled provider self-report — no engine mocks. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + Brainy, + NounType, + VerbType, + GraphIndexNotReadyError, + MetadataIndexNotReadyError, + VectorIndexNotReadyError +} from '../../src/index.js' +import type { HealthReport, LedgerInvariantResult } from '../../src/plugin.js' +import { prodLog } from '../../src/utils/logger.js' +import { createTestConfig } from '../helpers/test-factory.js' + +/** The white-box surface these pins drive on a live brain instance. */ +interface BrainInternals { + storage: { + getNoun(id: string): Promise + getNounMetadata(id: string): Promise + getNouns(options?: unknown): Promise + getVerbs(options?: unknown): Promise + } + index: { healthReport?: () => HealthReport; isReady?: () => boolean; rebuild(): Promise } + metadataIndex: { + healthReport?: () => HealthReport + isReady?: () => boolean + rebuild(): Promise + validateInvariants?: () => Promise + } + graphIndex: { + healthReport?: () => HealthReport + isReady?: () => boolean + rebuild(): Promise + validateInvariants?: () => Promise + } + rebuildIndexesIfNeeded(force?: boolean): Promise +} + +function internalsOf(brain: Brainy): BrainInternals { + return brain as unknown as BrainInternals +} + +function invariant(overrides: Partial = {}): LedgerInvariantResult { + return { + name: 'manifest-residency', + holds: true, + detail: 'ok', + heal: 'none', + source: 'ledger', + ...overrides + } +} + +function healthReport(overrides: Partial = {}): HealthReport { + return { + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 1, + generation: 1, + unledgered: [], + ...overrides + } +} + +const brains: Brainy[] = [] +const dirs: string[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) + vi.restoreAllMocks() +}) + +describe('health gate (a) — not-serving refuses loudly, ZERO canonical reads during the refusal', () => { + it('metadata not-serving: find() throws MetadataIndexNotReadyError naming the failing invariant', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + internals.metadataIndex.healthReport = () => + healthReport({ + provider: 'metadata', + serving: false, + healthy: false, + invariants: [invariant({ name: 'posted-count-floor', holds: false, heal: 'rebuild', detail: 'posted 2 < canonical 5' })] + }) + + const getNounSpy = vi.spyOn(internals.storage, 'getNoun') + const getNounMetadataSpy = vi.spyOn(internals.storage, 'getNounMetadata') + const getNounsSpy = vi.spyOn(internals.storage, 'getNouns') + + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError) + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/posted-count-floor/) + + expect(getNounSpy).not.toHaveBeenCalled() + expect(getNounMetadataSpy).not.toHaveBeenCalled() + expect(getNounsSpy).not.toHaveBeenCalled() + + delete internals.metadataIndex.healthReport + }) + + it('graph not-serving: related() throws GraphIndexNotReadyError naming the failing invariant, no canonical reads', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + const a = await brain.add({ data: 'a', type: NounType.Person }) + const b = await brain.add({ data: 'b', type: NounType.Person }) + await brain.relate({ from: a, to: b, type: VerbType.Knows }) + await brain.flush() + + const internals = internalsOf(brain) + internals.graphIndex.healthReport = () => + healthReport({ + provider: 'graph', + serving: false, + healthy: false, + invariants: [invariant({ name: 'adjacency-residency', holds: false, heal: 'rebuild', detail: 'edges not loaded' })] + }) + + const getNounSpy = vi.spyOn(internals.storage, 'getNoun') + const getVerbsSpy = vi.spyOn(internals.storage, 'getVerbs') + + await expect(brain.related({ from: a })).rejects.toBeInstanceOf(GraphIndexNotReadyError) + await expect(brain.related({ from: a })).rejects.toThrow(/adjacency-residency/) + + expect(getNounSpy).not.toHaveBeenCalled() + expect(getVerbsSpy).not.toHaveBeenCalled() + + delete internals.graphIndex.healthReport + }) +}) + +describe('health gate (b) — unledgered is unknown: never blocks a serving provider', () => { + it('serving:true with an unledgered family and no failing invariant serves normally; at most one narration', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + internals.metadataIndex.healthReport = () => + healthReport({ + provider: 'metadata', + serving: true, + healthy: true, + invariants: [], + unledgered: ['canonical-verb-coverage'] + }) + + const warnSpy = vi.spyOn(prodLog, 'warn') + + const r1 = await brain.find({ where: { team: 'atlas' } }) + const r2 = await brain.find({ where: { team: 'atlas' } }) + expect(r1.length).toBe(1) + expect(r2.length).toBe(1) + + const narrations = warnSpy.mock.calls.filter( + ([msg]) => typeof msg === 'string' && msg.includes('canonical-verb-coverage') + ) + expect(narrations.length).toBe(1) // one narration at most across both reads (same generation) + + delete internals.metadataIndex.healthReport + }) +}) + +describe('health gate (c) — degraded-but-serving narrates once per generation', () => { + it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + let generation = 1 + internals.index.healthReport = () => + healthReport({ + provider: 'vector', + serving: true, + healthy: false, + invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail: 'counter drift' })], + generation + }) + + const warnSpy = vi.spyOn(prodLog, 'warn') + const countNarrations = () => + warnSpy.mock.calls.filter(([msg]) => typeof msg === 'string' && msg.includes('stale-vector-counter')).length + + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + expect(countNarrations()).toBe(1) // same generation both times — one narration + + generation = 2 + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + expect(countNarrations()).toBe(2) // generation bumped — a second narration + + delete internals.index.healthReport + }) +}) + +describe('health gate (d) — interim isReady()-only path (no healthReport) is unchanged', () => { + it('isReady() === true serves; isReady() === false refuses via the typed NotReady error', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + internals.metadataIndex.isReady = () => true + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + + internals.metadataIndex.isReady = () => false + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError) + + delete internals.metadataIndex.isReady + }) +}) + +describe('health gate (e) — open builds; the first read never does', () => { + it('disableAutoRebuild:true on a populated store: open narrates + builds; the first find() triggers zero rebuilds', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-healthgate-open-')) + dirs.push(dir) + + const writer = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false, + silent: true, + disableAutoRebuild: true + }) + await writer.init() + brains.push(writer) + await writer.add({ data: 'row one', type: NounType.Document, metadata: { team: 'atlas' } }) + await writer.flush() + await brains.pop()!.close() + + const warnSpy = vi.spyOn(prodLog, 'warn') + const reader = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false, + silent: true, + disableAutoRebuild: true + }) + const internals = internalsOf(reader) + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded') + + await reader.init() + brains.push(reader) + + expect(rebuildSpy).toHaveBeenCalledTimes(1) // open() built it, exactly once + expect( + warnSpy.mock.calls.some( + ([msg]) => typeof msg === 'string' && msg.includes('open() is building') + ) + ).toBe(true) + + rebuildSpy.mockClear() + const rows = await reader.find({ where: { team: 'atlas' } }) + expect(rebuildSpy).toHaveBeenCalledTimes(0) // the read never builds + expect(rows.length).toBe(1) + }, 30000) +}) + +describe('health gate (f) — the ceremony door: explicit rebuild bypasses invariant consultation', () => { + it("repairIndex({ rebuild: ['graph'] }) rebuilds unconditionally without consulting validateInvariants", async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'x', type: NounType.Concept }) + await brain.flush() + + const internals = internalsOf(brain) + let validateCalls = 0 + internals.graphIndex.validateInvariants = async () => { + validateCalls++ + return healthReport({ provider: 'graph' }) + } + const rebuildSpy = vi.spyOn(internals.graphIndex, 'rebuild') + + const report = await brain.repairIndex({ rebuild: ['graph'] }) + + expect(rebuildSpy).toHaveBeenCalledTimes(1) + expect(validateCalls).toBe(0) // the door never consults validateInvariants to decide + + const graphFamily = report.families.find((f) => f.family === 'provider:graph') + expect(graphFamily?.rebuilt).toBe(true) + expect(graphFamily?.checked).toBe(true) + expect(graphFamily?.reason).toBe('explicit rebuild requested') + + delete internals.graphIndex.validateInvariants + }) + + it('bare repairIndex() on a healthy provider calls no rebuild()', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'x', type: NounType.Concept }) + await brain.flush() + + const internals = internalsOf(brain) + internals.graphIndex.validateInvariants = async () => healthReport({ provider: 'graph', healthy: true, serving: true }) + const rebuildSpy = vi.spyOn(internals.graphIndex, 'rebuild') + + await brain.repairIndex() + + expect(rebuildSpy).not.toHaveBeenCalled() + + delete internals.graphIndex.validateInvariants + }) +}) + +describe('health gate (g) — a throwing healthReport() is a contract violation, never read as healthy', () => { + it('healthReport() that throws refuses loudly with the typed NotReady error naming the throw', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + internals.index.healthReport = () => { + throw new Error('accelerator: mmap window busy') + } + + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(VectorIndexNotReadyError) + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/mmap window busy/) + + delete internals.index.healthReport + }) +}) diff --git a/tests/lifecycle/README.md b/tests/lifecycle/README.md new file mode 100644 index 00000000..e6456f19 --- /dev/null +++ b/tests/lifecycle/README.md @@ -0,0 +1,16 @@ +# The Lifecycle Lane + +One brain, driven through founding, a working day, a clean restart, a +crash, a repair, and a second life, checked chapter by chapter against an +independent shadow-model referee (`biographyHarness.ts`). It catches +COMPOSITION regressions unit tests miss — a store fine in one process but +broken across a restart/crash/repair. Runs on the plain JS engine, so it +gates every commit. + +Run it: `npx vitest run tests/lifecycle --pool=forks` + +A red names the chapter label, the id, and expected-vs-actual — diagnosable +from the message alone. `biography.test.ts` is split into two `it` blocks +(Ch1-3, then Ch4-6) purely for reporting; it is still ONE fixed-order story. +Chapters must never be reordered, skipped, or made conditional, and a +failing chapter's assertion must never be weakened to force green. diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts new file mode 100644 index 00000000..305d7a99 --- /dev/null +++ b/tests/lifecycle/biography.test.ts @@ -0,0 +1,429 @@ +/** + * @module tests/lifecycle/biography + * @description THE LIFECYCLE LANE — see `tests/lifecycle/README.md` for what + * this proves and how to run it. One scenario, "the working store": a single + * brain driven through founding, a working day, a clean restart, a crash, a + * repair, and a second life, verified chapter by chapter against an + * independent shadow-model referee (`biographyHarness.ts`). + * + * Split into two `it` blocks so a currently-failing later chapter (see the + * second block's header comment — a live engine finding, not a defect in + * this lane) never hides the earlier chapters' passing coverage. The two + * blocks share one brain's directory and one shadow model, run in the SAME + * fixed order the single scenario always has (`describe.sequential` below + * exists to say so explicitly, though vitest's own default is sequential + * within a file) — this is a split for REPORTING clarity, not a reordering + * or conditional skip of any chapter. + */ +import { describe, it, expect } from 'vitest' +import * as fs from 'node:fs' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import type { Brainy } from '../../src/brainy.js' +import type { AddParams, RelateParams, UpdateParams, UpdateRelationParams } from '../../src/index.js' +import { abandonAsCrashed, makeTempDir, openBrain, uid } from '../helpers/durabilityKillMatrix.js' +import { + createModel, + getCanonicalCountsFor, + modelAdd, + modelDelete, + modelRelate, + modelUpdate, + modelUpdateRelation, + recordVfsFileWrite, + snapshotVfsBaseline, + verifyChapter, + type HubCheck, + type ShadowModel +} from './biographyHarness.js' + +const STATUSES = ['active', 'pending', 'closed', 'archived'] as const + +/** Cycle a status value to the next one in the fixed rotation — used so + * Ch2's 40 updates provably MOVE entities across find() buckets rather than + * risking a no-op reassignment of the same value. */ +function nextStatus(current: unknown): (typeof STATUSES)[number] { + const currentStr = typeof current === 'string' ? current : STATUSES[0] + const idx = STATUSES.indexOf(currentStr as (typeof STATUSES)[number]) + return STATUSES[(idx < 0 ? 0 : idx + 1) % STATUSES.length] +} + +// --------------------------------------------------------------------------- +// Shared biography state — set up by the first `it`, consumed by the second. +// The two blocks are one continuous story told in two named pieces; nothing +// here resets or diverges between them. +// --------------------------------------------------------------------------- +let dir: string +let model: ShadowModel +let brain: Brainy +let hubs: HubCheck[] +let employees: string[] +let customers: string[] +let invoices: string[] +let tasks: string[] +let projects: string[] +let nonHub: string[] + +// ---- Wrappers: every call to the real brain updates the shadow model in +// the same statement, so the two can never drift apart by construction. +// Defined once, closing over the `let` bindings above so both `it` blocks +// (and any future reopen inside them) operate on the current brain/model. +async function doAdd(label: string, params: Omit): Promise { + const id = uid(label) + await brain.add({ ...params, id }) + modelAdd(model, id, { + type: params.type, + subtype: params.subtype, + metadata: params.metadata ?? {}, + visibility: params.visibility + }) + return id +} + +async function doUpdate(id: string, patch: Omit): Promise { + await brain.update({ ...patch, id }) + modelUpdate(model, id, { metadata: patch.metadata, merge: patch.merge, visibility: patch.visibility }) +} + +async function doRemove(id: string): Promise { + await brain.remove(id) + modelDelete(model, id) +} + +async function doRelate(params: RelateParams): Promise { + const id = await brain.relate(params) + modelRelate(model, id, { + from: params.from, + to: params.to, + type: params.type, + subtype: params.subtype, + metadata: params.metadata + }) + return id +} + +async function doUpdateRelation(id: string, patch: Omit): Promise { + await brain.updateRelation({ ...patch, id }) + modelUpdateRelation(model, id, { metadata: patch.metadata, merge: patch.merge }) +} + +async function doVfsWrite(path: string, content: string): Promise { + await brain.vfs.writeFile(path, content) + recordVfsFileWrite(model) +} + +describe.sequential('lifecycle — the working store', () => { + it( + 'Ch1 FOUNDING -> Ch2 A WORKING DAY -> Ch3 CLEAN RESTART: every read serves truth', + async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = makeTempDir() + model = createModel() + + // logAuthority: 'adopt' from the first open, mirrored across every + // reopen — see write-flow-production-shape.test.ts, which the later + // crash chapter's at-ack law is pinned against. + brain = await openBrain(dir, { logAuthority: 'adopt' }) + + // ================================================================= + // CHAPTER 1 — FOUNDING + // ================================================================= + // Baseline MUST be snapshotted before any biography act — it is the + // VFS root's own system-tier footprint, measured, never hardcoded. + await snapshotVfsBaseline(brain, model) + + employees = [] + for (let i = 0; i < 20; i++) { + employees.push( + await doAdd(`emp-${i}`, { + data: `employee record ${i}`, + type: NounType.Person, + subtype: 'employee', + metadata: { status: STATUSES[i % STATUSES.length], department: ['engineering', 'sales', 'support'][i % 3] } + }) + ) + } + customers = [] + for (let i = 0; i < 20; i++) { + customers.push( + await doAdd(`cust-${i}`, { + data: `customer record ${i}`, + type: NounType.Person, + subtype: 'customer', + metadata: { status: STATUSES[i % STATUSES.length], tier: i % 2 === 0 ? 'gold' : 'standard' } + }) + ) + } + invoices = [] + for (let i = 0; i < 30; i++) { + invoices.push( + await doAdd(`inv-${i}`, { + data: `invoice record ${i}`, + type: NounType.Document, + subtype: 'invoice', + metadata: { status: STATUSES[i % STATUSES.length], amount: 100 + i * 17 } + }) + ) + } + tasks = [] + for (let i = 0; i < 25; i++) { + tasks.push( + await doAdd(`task-${i}`, { + data: `task record ${i}`, + type: NounType.Task, + subtype: 'milestone', + metadata: { status: STATUSES[i % STATUSES.length], priority: (i % 5) + 1 } + }) + ) + } + projects = [] + for (let i = 0; i < 25; i++) { + projects.push( + await doAdd(`proj-${i}`, { + data: `project record ${i}`, + type: NounType.Project, + metadata: { status: STATUSES[i % STATUSES.length], budget: 1000 * (i + 1) } + }) + ) + } + expect(employees.length + customers.length + invoices.length + tasks.length + projects.length).toBe(120) + + // Five hubs (proj-0..proj-4) fan out to tasks (Contains) and employees + // (WorksWith); a residual band of invoice->customer RelatedTo edges is + // unrelated to any hub. Hubs are never touched again for the rest of + // the biography, so they stay valid adjacency samples in every chapter. + const hubIds = projects.slice(0, 5) + for (let h = 0; h < 5; h++) { + for (let k = 0; k < 15; k++) { + const taskIdx = (h * 5 + k) % tasks.length + await doRelate({ from: hubIds[h], to: tasks[taskIdx], type: VerbType.Contains, subtype: 'delivers' }) + } + for (let k = 0; k < 10; k++) { + const empIdx = (h * 4 + k) % employees.length + await doRelate({ from: hubIds[h], to: employees[empIdx], type: VerbType.WorksWith }) + } + } + for (let j = 0; j < 25; j++) { + await doRelate({ from: invoices[j], to: customers[j % customers.length], type: VerbType.RelatedTo, subtype: 'billed-to' }) + } + expect(model.relations.size).toBe(150) + + // A handful of VFS files. + for (let i = 0; i < 5; i++) { + await doVfsWrite(`/report-${i}.txt`, `founding report ${i}`) + } + + await brain.flush() + + hubs = hubIds.map((id) => ({ id, typeFilters: [VerbType.Contains, VerbType.WorksWith] })) + await verifyChapter(brain, model, 'Ch1 FOUNDING', { hubs, bucketField: 'status' }) + + // ================================================================= + // CHAPTER 2 — A WORKING DAY + // ================================================================= + // Non-hub pool for every mutation below. + nonHub = [...employees, ...customers, ...invoices, ...tasks, ...projects.slice(5)] + + // 40 updates that provably MOVE entities across find() status buckets. + const updateTargets = nonHub.slice(0, 40) + for (const id of updateTargets) { + const current = model.entities.get(id)!.metadata.status + await doUpdate(id, { metadata: { status: nextStatus(current) } }) + } + + // 10 visibility flips (public -> internal). + const visibilityTargets = nonHub.slice(40, 50) + for (const id of visibilityTargets) { + await doUpdate(id, { visibility: 'internal' }) + } + + // 15 deletes — some hub members (their edges cascade away), 3 of them + // earmarked for Ch6's resurrection. + const resurrectIds = [tasks[0], tasks[1], employees[0]] + const otherDeletes = [ + tasks[2], tasks[3], tasks[4], tasks[5], tasks[6], + employees[1], employees[2], employees[3], + customers[0], customers[1], customers[2], customers[3] + ] + const ch2DeleteTargets = [...resurrectIds, ...otherDeletes] + expect(ch2DeleteTargets.length).toBe(15) + for (const id of ch2DeleteTargets) { + await doRemove(id) + } + + // 20 new adds. + const ch2NewTypes = [NounType.Person, NounType.Document, NounType.Task] + for (let i = 0; i < 20; i++) { + await doAdd(`ch2-new-${i}`, { + data: `working-day addition ${i}`, + type: ch2NewTypes[i % ch2NewTypes.length], + subtype: 'ad-hoc', + metadata: { status: STATUSES[i % STATUSES.length] } + }) + } + + // 10 updateRelation metadata patches — read AFTER the deletes above, + // so only relations the cascade left alive are ever targeted. + const survivingRelationIds = [...model.relations.keys()].slice(0, 10) + expect(survivingRelationIds.length).toBe(10) + for (const relId of survivingRelationIds) { + await doUpdateRelation(relId, { metadata: { reviewed: true } }) + } + + await brain.flush() + await verifyChapter(brain, model, 'Ch2 A WORKING DAY', { hubs, bucketField: 'status' }) + + // ================================================================= + // CHAPTER 3 — CLEAN RESTART + // ================================================================= + await brain.close() + brain = await openBrain(dir, { logAuthority: 'adopt' }) + await verifyChapter(brain, model, 'Ch3 CLEAN RESTART', { hubs, bucketField: 'status' }) + + // Leave the brain closed and the directory intact for the next `it` + // (the biography continues there) — do NOT remove `dir` here. + await brain.close() + }, + 300000 + ) + + /** + * Ch4 CRASH is a LIVE ENGINE FINDING, not a defect in this lane (see + * README.md and the project report this lane's build produced): after a + * crash (writes acked at commit but never flushed, the process abandoned + * exactly as `abandonAsCrashed` models, then reopened), canonical storage + * (`get()`), the vector index, and `getNounCount()`/`getCanonicalCounts()` + * all correctly recover every acked write — but the METADATA INDEX behind + * `find({ where })` recovers NONE of the crash-window's acked writes + * (neither new adds nor metadata updates to pre-existing entities), even + * though `getIndexStatus()` reports `projections.metadata.synchronous: + * true`. `repairIndex()` cannot close the gap either: its own report names + * `provider:metadata` as `checked: false, skipped: "no + * validateInvariants/rebuild contract"`. The assertion below states the + * TRUE contract (find() must agree with get()) and is expected to fail + * against the current engine — it must never be loosened to paper over + * this. Ch5/Ch6 are written in full below it and will start running the + * moment this gap is closed; they are not dead code, they are blocked code. + */ + // RELEASE-BLOCKING FINDING (the kill-matrix convention: assert the CONTRACT, + // mark `.fails`, never weaken): after a crash + adopt reopen, the JS metadata + // index computes its watermark verdict but nothing consumes 'catchup' + // (metadataIndex.ts loadWatermarkVerdict) — find() serves the pre-crash + // index while get()/counts recover. The catchup wiring is the cure; when it + // lands this `.fails` marker MUST be removed (vitest will force it: a + // passing `.fails` test is itself a failure). + it.fails( + 'Ch4 CRASH -> Ch5 REPAIR -> Ch6 SECOND LIFE: continues the Ch3 store', + async () => { + try { + brain = await openBrain(dir, { logAuthority: 'adopt' }) + + // =============================================================== + // CHAPTER 4 — CRASH + // =============================================================== + const ch4Types = [NounType.Person, NounType.Document, NounType.Task, NounType.Project] + for (let i = 0; i < 10; i++) { + await doAdd(`ch4-new-${i}`, { + data: `crash-window addition ${i}`, + type: ch4Types[i % ch4Types.length], + metadata: { status: STATUSES[i % STATUSES.length] } + }) + } + const ch4UpdateTargets = nonHub.slice(50, 55) // invoices[10..14] — untouched so far + for (const id of ch4UpdateTargets) { + await doUpdate(id, { metadata: { status: 'active' } }) + } + // NO flush — abandon exactly the way process death would (the + // at-ack law: every write already awaited above must survive). + await abandonAsCrashed(brain) + brain = await openBrain(dir, { logAuthority: 'adopt' }) + await verifyChapter(brain, model, 'Ch4 CRASH', { hubs, bucketField: 'status' }) + + // =============================================================== + // CHAPTER 5 — REPAIR + // =============================================================== + const report = await brain.repairIndex() + for (const family of report.families) { + const accounted = + family.checked === true || (family.checked === false && typeof family.skipped === 'string' && family.skipped.length > 0) + expect( + accounted, + `[Ch5 REPAIR] family '${family.family}' must be checked or explicitly skipped with a reason; got ${JSON.stringify(family)}` + ).toBe(true) + } + // A healthy store: repair must change nothing the model doesn't + // already expect — verifyChapter against the UNCHANGED model proves it. + await verifyChapter(brain, model, 'Ch5 REPAIR', { hubs, bucketField: 'status' }) + + // =============================================================== + // CHAPTER 6 — SECOND LIFE + // =============================================================== + const ch6Types = [NounType.Person, NounType.Document, NounType.Task, NounType.Project] + for (let i = 0; i < 10; i++) { + await doAdd(`ch6-new-${i}`, { + data: `second-life addition ${i}`, + type: ch6Types[i % ch6Types.length], + metadata: { status: STATUSES[i % STATUSES.length] } + }) + } + const ch6UpdateTargets = nonHub.slice(55, 65) // invoices[15..24] — untouched so far + expect(ch6UpdateTargets.every((id) => model.entities.get(id)!.alive)).toBe(true) + for (const id of ch6UpdateTargets) { + await doUpdate(id, { metadata: { status: 'closed' } }) + } + const ch6DeleteTargets = nonHub + .slice(65, 90) // invoices[25..29] + tasks[0..19] (some already dead — filtered below) + .filter((id) => model.entities.get(id)!.alive) + .slice(0, 7) + expect(ch6DeleteTargets.length).toBe(7) + for (const id of ch6DeleteTargets) { + await doRemove(id) + } + + // Resurrection: the SAME three ids Ch2 deleted, reinserted with + // BRAND-NEW metadata — the model expects the new metadata only. + await doAdd('task-0', { data: 'resurrected task 0', type: NounType.Task, subtype: 'milestone', metadata: { status: 'active', resurrected: true } }) + await doAdd('task-1', { data: 'resurrected task 1', type: NounType.Task, subtype: 'milestone', metadata: { status: 'pending', resurrected: true } }) + await doAdd('emp-0', { data: 'resurrected employee 0', type: NounType.Person, subtype: 'employee', metadata: { status: 'active', resurrected: true } }) + expect(tasks[0]).toBe(uid('task-0')) // same id as Ch1/Ch2 — the resurrection-adjacent shape + + await brain.close() + brain = await openBrain(dir, { logAuthority: 'adopt' }) + await verifyChapter(brain, model, 'Ch6 SECOND LIFE', { hubs, bucketField: 'status' }) + + // Final, standalone getCanonicalCounts() exactness check (beyond + // verifyChapter's own (f) leg) — the whole ledger, in one shot. + const finalCounts = await getCanonicalCountsFor(brain) + const aliveEntities = [...model.entities.values()].filter((e) => e.alive) + const alivePublicEntities = aliveEntities.filter((e) => (e.visibility ?? 'public') === 'public') + const aliveVerbs = model.relations.size + expect(finalCounts, 'final getCanonicalCounts() exactness — Ch6 SECOND LIFE').toEqual({ + nouns: { + counted: alivePublicEntities.length + model.vfsFileNouns, + all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns + }, + verbs: { + counted: aliveVerbs + model.vfsContainsVerbs, + all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs + }, + suspect: false + }) + } finally { + await brain.close().catch(() => {}) + // Best-effort, retried: a still-draining background persistence + // write (e.g. count/index write-through) can race a single rmSync + // and leave a partial directory behind — retry a couple of times + // rather than let this temp dir leak. + for (let attempt = 0; attempt < 3; attempt++) { + try { + fs.rmSync(dir, { recursive: true, force: true }) + if (!fs.existsSync(dir)) break + } catch { + // ignore and retry + } + await new Promise((resolve) => setTimeout(resolve, 100)) + } + } + }, + 300000 + ) +}) diff --git a/tests/lifecycle/biographyHarness.ts b/tests/lifecycle/biographyHarness.ts new file mode 100644 index 00000000..ca15b36a --- /dev/null +++ b/tests/lifecycle/biographyHarness.ts @@ -0,0 +1,389 @@ +/** + * @module tests/lifecycle/biographyHarness + * @description The referee for the LIFECYCLE LANE (see `biography.test.ts`): + * a plain in-memory SHADOW MODEL of a brain's contents, updated by every act + * the biography performs (add/update/remove/relate/updateRelation/vfs writes), + * plus `verifyChapter()`, which asserts the live brain agrees with the model + * after every chapter. No engine code runs inside the model — it is an + * independent ledger, not a mirror of the implementation under test. + * + * COUNT SEMANTICS this harness encodes (verified against the live engine, + * not assumed — see the module-level comments below for how each was + * confirmed): + * + * - `getNounCount()` / `getVerbCount()` count PUBLIC-tier alive records only + * (visibility absent or `'public'`) — `'internal'` and `'system'` are both + * excluded. `storage.getCanonicalCounts()` mirrors that same PUBLIC-only + * scalar as `counted`, and additionally reports `all` — every tier, + * unfiltered — as the coverage-ledger denominator (see + * tests/integration/canonical-count-ledger.test.ts). + * - `brain.vfs.writeFile()` for a brand-new file at a path directly under the + * VFS root creates exactly ONE new File noun plus ONE new `Contains` verb + * (root -> file), and BOTH are ordinary PUBLIC records (no visibility + * field is set) — so they count toward `getNounCount()`/`getVerbCount()` + * as well as the canonical `all` scalars. Only the VFS ROOT entity itself + * is `'system'`-tier (created once, at `init()`, before any biography + * chapter runs) — that lone record is the only hidden-tier footprint the + * model does not construct explicitly, so it is captured empirically via + * `snapshotVfsBaseline()` immediately after `init()` rather than hardcoded. + * - `related()` filters edges by the RELATION's own visibility tier, not by + * the visibility of the entities the edge connects — flipping an entity to + * `'internal'` does not hide its edges from `related()`. This lane never + * sets relation visibility, so every relation the model tracks is exactly + * as reachable as its presence in `model.relations` implies. + * - `remove()` cascades: every relation touching the removed entity (as + * `from` or `to`) is hard-deleted along with it. The model mirrors this by + * deleting the relation entirely from `model.relations` (no relation + * "alive" flag — presence in the map IS aliveness). + */ +import { expect } from 'vitest' +import type { Brainy } from '../../src/brainy.js' +import type { NounType, VerbType } from '../../src/types/graphTypes.js' +import type { EntityVisibility, StorageAdapter } from '../../src/coreTypes.js' + +/** + * One entity's complete lifecycle-relevant state, as the biography's acts + * leave it. `alive: false` means the model believes the id has been removed + * — the entry is KEPT (never deleted from the map) so `verifyChapter` can + * assert the negative half of the contract: a dead id must read as `null`. + */ +export interface ShadowEntity { + type: NounType + subtype?: string + metadata: Record + visibility?: EntityVisibility + alive: boolean +} + +/** + * One relation's complete lifecycle-relevant state. There is no `alive` + * flag here — presence in {@link ShadowModel.relations} IS aliveness, + * mirroring the engine's hard delete of the canonical verb record on + * cascade (see the module header). + */ +export interface ShadowRelation { + from: string + to: string + type: VerbType + subtype?: string + metadata: Record +} + +/** + * The independent truth ledger the biography updates on every act it + * performs. `verifyChapter` checks the live brain against this — never the + * other way around. + */ +export interface ShadowModel { + entities: Map + relations: Map + /** + * `getCanonicalCounts()` nouns.all / verbs.all captured right after + * `init()`, before chapter 1 — the VFS root's own system-tier footprint. + * Set once via {@link snapshotVfsBaseline}; never hardcoded. + */ + vfsBaselineNouns: number + vfsBaselineVerbs: number + /** + * Public nouns/verbs created by `vfs.writeFile()` for a brand-new file at + * a flat top-level path: exactly one File noun + one Contains verb per + * call (see the module header). Bumped by {@link recordVfsFileWrite}. + */ + vfsFileNouns: number + vfsContainsVerbs: number +} + +/** A fresh, empty shadow model — call once before chapter 1. */ +export function createModel(): ShadowModel { + return { + entities: new Map(), + relations: new Map(), + vfsBaselineNouns: 0, + vfsBaselineVerbs: 0, + vfsFileNouns: 0, + vfsContainsVerbs: 0 + } +} + +/** Narrow, documented private-storage access (the same style already used by + * `tests/helpers/durabilityKillMatrix.ts`'s `storeOf()`), needed because + * `getCanonicalCounts()` lives on the storage adapter, not on `Brainy`. */ +function storageOf(brain: Brainy): StorageAdapter { + return (brain as unknown as { storage: StorageAdapter }).storage +} + +/** Public wrapper around the private-storage `getCanonicalCounts()` read, so + * callers never need their own private-access cast — used internally by + * {@link snapshotVfsBaseline} and {@link verifyChapter}, and by + * `biography.test.ts` for its final standalone exactness check. */ +export async function getCanonicalCountsFor(brain: Brainy): ReturnType> { + const storage = storageOf(brain) + if (!storage.getCanonicalCounts) { + throw new Error( + 'lifecycle lane: the storage adapter under test has no getCanonicalCounts() — the canonical-count-exactness leg of this lane is unrepresentable without it.' + ) + } + return storage.getCanonicalCounts() +} + +/** + * Snapshot the VFS root's own hidden-tier footprint. Call exactly once, + * immediately after `init()` and before chapter 1 does anything — this is + * the ONE baseline offset the model does not construct by hand (see the + * module header for why: the root is `'system'`-tier plumbing the biography + * never explicitly creates). + */ +export async function snapshotVfsBaseline(brain: Brainy, model: ShadowModel): Promise { + const counts = await getCanonicalCountsFor(brain) + model.vfsBaselineNouns = counts.nouns.all + model.vfsBaselineVerbs = counts.verbs.all +} + +/** + * Record one `brain.vfs.writeFile()` call for a brand-new file at a flat + * top-level path (no intermediate directories). Bumps both the noun and verb + * VFS counters by one, matching the engine's actual write path exactly (see + * the module header) — never call this for an overwrite of an existing path, + * a nested path (which would also vivify intermediate directory nouns/edges, + * a different, unmodeled shape), or the biography loses its exactness. + */ +export function recordVfsFileWrite(model: ShadowModel): void { + model.vfsFileNouns += 1 + model.vfsContainsVerbs += 1 +} + +/** Record a fresh `add()` (or a Ch6 resurrection — `Map.set` fully replaces + * whatever a prior dead entry held, which is exactly the "new metadata only" + * contract a resurrection must honor). */ +export function modelAdd( + model: ShadowModel, + id: string, + entity: { type: NounType; subtype?: string; metadata: Record; visibility?: EntityVisibility } +): void { + model.entities.set(id, { + type: entity.type, + subtype: entity.subtype, + metadata: { ...entity.metadata }, + visibility: entity.visibility, + alive: true + }) +} + +/** Record an `update()` — merges metadata by default, matching the engine's + * `merge: true` default; pass `merge: false` to mirror a full replace. */ +export function modelUpdate( + model: ShadowModel, + id: string, + patch: { metadata?: Record; merge?: boolean; visibility?: EntityVisibility } +): void { + const existing = model.entities.get(id) + if (!existing || !existing.alive) { + throw new Error(`shadow model: update() targeted ${id}, which the model does not have alive — biography sequencing bug`) + } + if (patch.metadata) { + existing.metadata = patch.merge === false ? { ...patch.metadata } : { ...existing.metadata, ...patch.metadata } + } + if (patch.visibility !== undefined) { + existing.visibility = patch.visibility + } +} + +/** Record a `remove()` — marks the entity dead (entry retained, per + * {@link ShadowEntity}) and cascades: every relation touching it, in either + * direction, is hard-deleted from the model too (matching the engine). */ +export function modelDelete(model: ShadowModel, id: string): void { + const existing = model.entities.get(id) + if (!existing || !existing.alive) { + throw new Error(`shadow model: remove() targeted ${id}, which the model does not have alive — biography sequencing bug`) + } + existing.alive = false + for (const [relId, rel] of model.relations) { + if (rel.from === id || rel.to === id) model.relations.delete(relId) + } +} + +/** Record a `relate()` — `id` is the relation id the real call returned. */ +export function modelRelate( + model: ShadowModel, + id: string, + relation: { from: string; to: string; type: VerbType; subtype?: string; metadata?: Record } +): void { + model.relations.set(id, { + from: relation.from, + to: relation.to, + type: relation.type, + subtype: relation.subtype, + metadata: { ...(relation.metadata ?? {}) } + }) +} + +/** Record an `updateRelation()` metadata patch — merges by default. */ +export function modelUpdateRelation( + model: ShadowModel, + id: string, + patch: { metadata?: Record; merge?: boolean } +): void { + const existing = model.relations.get(id) + if (!existing) { + throw new Error(`shadow model: updateRelation() targeted ${id}, which the model does not have — biography sequencing bug`) + } + if (patch.metadata) { + existing.metadata = patch.merge === false ? { ...patch.metadata } : { ...existing.metadata, ...patch.metadata } + } +} + +/** Order-independent structural equality for plain JSON-shaped metadata. */ +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true + if (typeof a !== typeof b) return false + if (a === null || b === null) return a === b + if (typeof a !== 'object') return false + const aKeys = Object.keys(a as Record) + const bKeys = Object.keys(b as Record) + if (aKeys.length !== bKeys.length) return false + for (const k of aKeys) { + if (!deepEqual((a as Record)[k], (b as Record)[k])) return false + } + return true +} + +/** One hub entity to sample for the `related()` adjacency check, plus the + * verb type(s) it is known (by biography construction) to have OUT-edges + * of, so the type-filtered variant is exercised too. */ +export interface HubCheck { + id: string + typeFilters: VerbType[] +} + +/** Options steering one `verifyChapter()` call. */ +export interface VerifyOptions { + /** Hub entities to sample for the `related()` adjacency check. */ + hubs: HubCheck[] + /** The metadata field `find()` bucket-checks against (a bare string field + * every alive entity may or may not carry — distinct values present among + * ALIVE model entities are discovered automatically each call, so a + * chapter that moves entities across buckets is re-checked exactly). */ + bucketField: string +} + +/** + * Assert the live brain agrees with the model, in full, after one chapter. + * Every failure message names the chapter `label`, the id (where + * applicable), and expected-vs-actual — a red here must be diagnosable from + * the assertion message alone, with no need to re-read this file. + */ +export async function verifyChapter(brain: Brainy, model: ShadowModel, label: string, opts: VerifyOptions): Promise { + // (a) + (b): every alive entity reads back exactly as modeled; every dead + // entity reads as null. + for (const [id, entity] of model.entities) { + const live = await brain.get(id) + if (entity.alive) { + expect(live, `[${label}] alive entity ${id} (type=${entity.type}) must be readable via get(), got null`).not.toBeNull() + const e = live! + expect(e.type, `[${label}] entity ${id} .type mismatch: expected ${entity.type}, got ${e.type}`).toBe(entity.type) + expect(e.subtype, `[${label}] entity ${id} .subtype mismatch: expected ${JSON.stringify(entity.subtype)}, got ${JSON.stringify(e.subtype)}`).toBe(entity.subtype) + expect( + e.visibility, + `[${label}] entity ${id} .visibility mismatch: expected ${JSON.stringify(entity.visibility)}, got ${JSON.stringify(e.visibility)}` + ).toBe(entity.visibility) + const metaMatches = deepEqual(e.metadata ?? {}, entity.metadata) + expect( + metaMatches, + `[${label}] entity ${id} .metadata mismatch: expected ${JSON.stringify(entity.metadata)}, got ${JSON.stringify(e.metadata)}` + ).toBe(true) + } else { + expect(live, `[${label}] dead entity ${id} (type=${entity.type}) must read as null, got ${JSON.stringify(live)}`).toBeNull() + } + } + + // (c) find({ where: { : value } }) returns exactly the + // model's matching alive set, per distinct value currently present. + const bucketValues = new Set() + for (const entity of model.entities.values()) { + if (!entity.alive) continue + const v = entity.metadata[opts.bucketField] + if (typeof v === 'string') bucketValues.add(v) + } + for (const value of bucketValues) { + const expectedIds = [...model.entities.entries()] + .filter(([, e]) => e.alive && e.metadata[opts.bucketField] === value) + .map(([id]) => id) + .sort() + const results = await brain.find({ + where: { [opts.bucketField]: value } as Record, + includeInternal: true, + limit: 100000 + }) + const actualIds = results.map((r) => r.id).sort() + expect( + actualIds, + `[${label}] find({ where: { ${opts.bucketField}: ${JSON.stringify(value)} } }) mismatch: expected ${expectedIds.length} ids ${JSON.stringify(expectedIds)}, got ${actualIds.length} ids ${JSON.stringify(actualIds)}` + ).toEqual(expectedIds) + } + + // (d) related(id) / related(id, { type }) for the hub sample matches the + // model's adjacency exactly (out-edges — related(id) is shorthand for + // { from: id }). + for (const hub of opts.hubs) { + const expectedAll = [...model.relations.entries()] + .filter(([, r]) => r.from === hub.id) + .map(([id]) => id) + .sort() + const liveAll = await brain.related({ from: hub.id, limit: 100000 }) + const actualAllIds = liveAll.map((r) => r.id).sort() + expect( + actualAllIds, + `[${label}] related(${hub.id}) mismatch: expected ${expectedAll.length} ids ${JSON.stringify(expectedAll)}, got ${actualAllIds.length} ids ${JSON.stringify(actualAllIds)}` + ).toEqual(expectedAll) + + for (const typeFilter of hub.typeFilters) { + const expectedTyped = [...model.relations.entries()] + .filter(([, r]) => r.from === hub.id && r.type === typeFilter) + .map(([id]) => id) + .sort() + const liveTyped = await brain.related({ from: hub.id, type: typeFilter, limit: 100000 }) + const actualTypedIds = liveTyped.map((r) => r.id).sort() + expect( + actualTypedIds, + `[${label}] related(${hub.id}, { type: '${typeFilter}' }) mismatch: expected ${expectedTyped.length} ids ${JSON.stringify(expectedTyped)}, got ${actualTypedIds.length} ids ${JSON.stringify(actualTypedIds)}` + ).toEqual(expectedTyped) + } + } + + // (e) getNounCount() / getVerbCount(): PUBLIC-tier alive records + // (visibility absent/'public'; 'internal' and 'system' both excluded — see + // the module header) plus the VFS's own public contributions. + const alivePublicNouns = [...model.entities.values()].filter((e) => e.alive && (e.visibility ?? 'public') === 'public').length + const aliveVerbs = model.relations.size + const expectedNounCount = alivePublicNouns + model.vfsFileNouns + const expectedVerbCount = aliveVerbs + model.vfsContainsVerbs + expect( + await brain.getNounCount(), + `[${label}] getNounCount() mismatch: expected ${expectedNounCount} (alive public entities ${alivePublicNouns} + vfs file nouns ${model.vfsFileNouns})` + ).toBe(expectedNounCount) + expect( + await brain.getVerbCount(), + `[${label}] getVerbCount() mismatch: expected ${expectedVerbCount} (alive relations ${aliveVerbs} + vfs contains verbs ${model.vfsContainsVerbs})` + ).toBe(expectedVerbCount) + + // (f) getCanonicalCounts(): ALL-visibility scalars (every tier) equal the + // model's alive totals including hidden tiers, plus the VFS's own + // contributions (both file nouns/verbs AND the once-measured root + // baseline). suspect must be false — every delete in this biography goes + // through brain.remove(), which always proves the record it decrements. + const ledger = await getCanonicalCountsFor(brain) + const aliveAllNouns = [...model.entities.values()].filter((e) => e.alive).length + const expectedNounsAll = aliveAllNouns + model.vfsFileNouns + model.vfsBaselineNouns + const expectedVerbsAll = aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs + expect( + ledger.nouns.all, + `[${label}] getCanonicalCounts().nouns.all mismatch: expected ${expectedNounsAll} (alive incl. internal ${aliveAllNouns} + vfs file nouns ${model.vfsFileNouns} + vfs root baseline ${model.vfsBaselineNouns})` + ).toBe(expectedNounsAll) + expect( + ledger.verbs.all, + `[${label}] getCanonicalCounts().verbs.all mismatch: expected ${expectedVerbsAll} (alive relations ${aliveVerbs} + vfs contains verbs ${model.vfsContainsVerbs} + vfs root baseline ${model.vfsBaselineVerbs})` + ).toBe(expectedVerbsAll) + expect(ledger.nouns.counted, `[${label}] getCanonicalCounts().nouns.counted mismatch (should mirror getNounCount())`).toBe(expectedNounCount) + expect(ledger.verbs.counted, `[${label}] getCanonicalCounts().verbs.counted mismatch (should mirror getVerbCount())`).toBe(expectedVerbCount) + expect(ledger.suspect, `[${label}] getCanonicalCounts().suspect must be false — every delete in this biography proves its record`).toBe(false) +} diff --git a/tests/unit/brainy/lazy-notready-honor.test.ts b/tests/unit/brainy/lazy-notready-honor.test.ts index 4cfc6857..e53be4a6 100644 --- a/tests/unit/brainy/lazy-notready-honor.test.ts +++ b/tests/unit/brainy/lazy-notready-honor.test.ts @@ -1,38 +1,49 @@ /** * @module tests/unit/brainy/lazy-notready-honor * @description THE SILENT-EMPTY TRAP pin (found during a fleet adoption, - * SELF-ENGINE-PAIR-STANDARD): under `disableAutoRebuild: true`, the lazy + * SELF-ENGINE-PAIR-STANDARD): under `disableAutoRebuild: true`, the OLD lazy * first-query path (`ensureIndexesLoaded`) assessed ONLY the vector index's * readiness — a native METADATA provider reporting not-ready (its strand * report) never blocked the completion latch, so the promised lazy rebuild - * never fired and every `find()` silently returned `[]` on a populated - * store (measured: 52 entities durable-but-unqueryable, first query - * 0ms/0 rows). The law: a not-ready report from ANY provider falls through - * to the rebuild — never a silent empty. + * never fired and every `find()` silently returned `[]` on a populated store + * (measured: 52 entities durable-but-unqueryable, first query 0ms/0 rows). + * + * RE-POINTED to the health-gate law (a read never builds; a rebuild runs + * entirely at open): `ensureIndexesLoaded()` is now a pure CHECK. A not-ready + * report from ANY provider — metadata, vector, or graph — makes it THROW the + * matching typed `*NotReadyError` rather than silently letting the read + * proceed, and it NEVER calls `rebuildIndexesIfNeeded` (that is entirely + * open()'s job now — see the second describe block below). The spirit is + * unchanged: a not-ready report from any single provider can never be + * shadowed into a silent empty result. * * White-box provider-double pattern per tests/unit/brainy/migration-deference. */ import { describe, it, expect, afterEach, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy, MetadataIndexNotReadyError } from '../../../src/index.js' import { NounType } from '../../../src/types/graphTypes.js' import { createTestConfig } from '../../helpers/test-factory.js' interface BrainInternals { index: { size(): number } metadataIndex: { isReady?: () => boolean } - lazyRebuildCompleted: boolean - ensureIndexesLoaded(): Promise + ensureIndexesLoaded(): void rebuildIndexesIfNeeded(force?: boolean): Promise } const brains: Brainy[] = [] +const dirs: string[] = [] afterEach(async () => { for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) vi.restoreAllMocks() }) -async function warmLazyBrain(): Promise<{ brain: Brainy; internals: BrainInternals }> { +async function warmBrain(): Promise<{ brain: Brainy; internals: BrainInternals }> { const brain = new Brainy(createTestConfig({ disableAutoRebuild: true })) await brain.init() brains.push(brain) @@ -40,36 +51,59 @@ async function warmLazyBrain(): Promise<{ brain: Brainy; internals: BrainInterna await brain.add({ data: `row ${i}`, type: NounType.Document, metadata: { i } }) } const internals = brain as unknown as BrainInternals - internals.lazyRebuildCompleted = false // simulate the cold first query return { brain, internals } } -describe('lazy path honors EVERY provider’s not-ready report', () => { - it('a not-ready METADATA provider blocks the completion latch and fires the rebuild', async () => { - const { internals } = await warmLazyBrain() +describe('the read gate honors EVERY provider’s not-ready report', () => { + it('a not-ready METADATA provider refuses loudly — it never lets a read proceed, and it never rebuilds', async () => { + const { internals } = await warmBrain() // The trap's shape: vector side looks fine (populated), metadata - // provider says NOT ready — the old gate latched complete here. - ;(internals.metadataIndex as { isReady?: () => boolean }).isReady = () => false - const rebuildSpy = vi - .spyOn(internals, 'rebuildIndexesIfNeeded') - .mockResolvedValue(undefined) + // provider says NOT ready — the OLD gate silently latched complete here. + // The new gate refuses loudly instead; a read never triggers a rebuild. + internals.metadataIndex.isReady = () => false + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) - await internals.ensureIndexesLoaded() - - expect(rebuildSpy, 'not-ready metadata provider must fire the lazy rebuild').toHaveBeenCalledWith(true) + expect(() => internals.ensureIndexesLoaded()).toThrow(MetadataIndexNotReadyError) + expect(rebuildSpy, 'a read NEVER triggers a rebuild — building is entirely open()\'s job now').not.toHaveBeenCalled() }) - it('control: all providers ready/unknown+populated → latch completes, no rebuild', async () => { - const { internals } = await warmLazyBrain() - ;(internals.metadataIndex as { isReady?: () => boolean }).isReady = () => true - const rebuildSpy = vi - .spyOn(internals, 'rebuildIndexesIfNeeded') - .mockResolvedValue(undefined) - - await internals.ensureIndexesLoaded() + it('control: all providers ready/unknown+populated → the gate lets the read through, no rebuild', async () => { + const { internals } = await warmBrain() + internals.metadataIndex.isReady = () => true + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) + expect(() => internals.ensureIndexesLoaded()).not.toThrow() expect(rebuildSpy).not.toHaveBeenCalled() - expect(internals.lazyRebuildCompleted).toBe(true) }) }) + +describe('the open-time build honors the same law: a needed rebuild runs at open, never deferred to a read', () => { + it('disableAutoRebuild:true does not defer a needed rebuild past open() on a reopened, populated store', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-lazy-notready-honor-')) + dirs.push(dir) + + const writer = new Brainy(createTestConfig({ disableAutoRebuild: true, storage: { type: 'filesystem', path: dir } })) + await writer.init() + for (let i = 0; i < 3; i++) { + await writer.add({ data: `row ${i}`, type: NounType.Document, metadata: { i } }) + } + await writer.flush() + await writer.close() + + // Fresh instance over the same store: its derived indexes start empty in + // memory, so open()'s rebuildIndexesIfNeeded MUST fire (and complete) + // before init() returns — even though disableAutoRebuild is true, there + // is no first-query lazy path left to defer to. + const reader = new Brainy(createTestConfig({ disableAutoRebuild: true, storage: { type: 'filesystem', path: dir } })) + const internals = reader as unknown as { rebuildIndexesIfNeeded(force?: boolean): Promise } + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded') + + await reader.init() + brains.push(reader) + + expect(rebuildSpy).toHaveBeenCalledTimes(1) + const rows = await reader.find({ where: { i: 1 } }) + expect(rows.length).toBe(1) + }, 30000) +}) diff --git a/tests/unit/brainy/metadata-provider-contract.test.ts b/tests/unit/brainy/metadata-provider-contract.test.ts index 7e690978..945c0670 100644 --- a/tests/unit/brainy/metadata-provider-contract.test.ts +++ b/tests/unit/brainy/metadata-provider-contract.test.ts @@ -1,15 +1,18 @@ /** * @module tests/unit/brainy/metadata-provider-contract - * @description Brainy-side wiring of the two metadata-provider contract additions - * confirmed with cor for the lockstep: + * @description Brainy-side wiring of the metadata-provider contract. * - * 1. `probeConsistency()` — an OPTIONAL O(1) cold-open consistency sampler. On the - * first read, brainy calls it once; on `false` it self-heals via - * `detectAndRepairCorruption()` (the metadata counterpart of the graph cold-load - * guard). The native provider implements it; the JS index omits it (no-op). - * 2. `getIdsForFilter(filter, opts?)` — brainy passes a page bound on the UNSORTED - * `find({ type, where, limit })` path so a native provider can early-stop. The JS - * index ignores `opts`. + * `getIdsForFilter(filter, opts?)` — brainy passes a page bound on the UNSORTED + * `find({ type, where, limit })` path so a native provider can early-stop. The JS + * index ignores `opts`. + * + * RETIRED (health-gate law): `probeConsistency()` / `ensureMetadataConsistencyProbed()` + * — a read-time consistency probe that launches `detectAndRepairCorruption()` on + * `false` was exactly the read-triggered dark rebuild the law forbids (a read must + * never start a store walk or a rebuild). The probe's diagnostic value lives on in + * `validateIndexConsistency()` / `repairIndex()`, which remain explicit, operator-invoked + * calls. The pin below confirms the retirement: `probeConsistency()` is never called by + * a read, even when a provider exposes it. * * These are unit tests of brainy's CALL behaviour (the real end-to-end honoring is * exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS @@ -19,7 +22,7 @@ import { describe, it, expect, beforeEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' -describe('metadata-provider contract wiring (probeConsistency + getIdsForFilter opts)', () => { +describe('metadata-provider contract wiring (getIdsForFilter opts)', () => { let brain: Brainy let mi: any @@ -29,47 +32,23 @@ describe('metadata-provider contract wiring (probeConsistency + getIdsForFilter await brain.add({ data: 'a', type: NounType.Thing, metadata: { kind: 'x' } }) await brain.add({ data: 'b', type: NounType.Thing, metadata: { kind: 'y' } }) mi = (brain as any).metadataIndex - ;(brain as any)._metadataConsistencyProbed = false // reset the one-shot guard }) - it('calls probeConsistency once on cold open and self-heals via detectAndRepairCorruption on false', async () => { + it('RETIRED: a read never calls probeConsistency() / self-heals via detectAndRepairCorruption — that is the read-triggered dark rebuild the health-gate law forbids', async () => { let probes = 0 let repairs = 0 - mi.probeConsistency = async () => { probes++; return false } // corrupt → must repair + mi.probeConsistency = async () => { probes++; return false } // would-be corrupt signal const origRepair = mi.detectAndRepairCorruption.bind(mi) mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } await brain.find({ where: { kind: 'x' } }) - expect(probes).toBe(1) - expect(repairs).toBe(1) - - // Second read must NOT re-probe (once per brain). await brain.find({ where: { kind: 'y' } }) - expect(probes).toBe(1) - expect(repairs).toBe(1) - }) - it('does NOT repair when the probe reports healthy', async () => { - let repairs = 0 - mi.probeConsistency = async () => true // clean - const origRepair = mi.detectAndRepairCorruption.bind(mi) - mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } + expect(probes).toBe(0) // no read-time probe exists anymore + expect(repairs).toBe(0) // and therefore no read-triggered self-heal either - await brain.find({ where: { kind: 'x' } }) - expect(repairs).toBe(0) - }) - - it('a probe failure never breaks the read (best-effort, retried next time)', async () => { - let probes = 0 - mi.probeConsistency = async () => { probes++; throw new Error('probe boom') } - - // The read still succeeds despite the throwing probe. - const rows = await brain.find({ where: { kind: 'x' } }) - expect(rows.length).toBe(1) - expect(probes).toBe(1) - // Guard reset on failure → the next read retries the probe. - await brain.find({ where: { kind: 'y' } }) - expect(probes).toBe(2) + delete mi.probeConsistency + mi.detectAndRepairCorruption = origRepair }) it('passes a page bound to getIdsForFilter on the unsorted find path (offset 0, brainy re-windows)', async () => { diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index 5968c620..31b9b216 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -25,7 +25,7 @@ */ import { describe, it, expect, afterEach, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' +import { Brainy, VectorIndexNotReadyError } from '../../../src/index.js' import { NounType } from '../../../src/types/graphTypes.js' import { createTestConfig } from '../../helpers/test-factory.js' import { BaseStorage } from '../../../src/storage/baseStorage.js' @@ -43,9 +43,8 @@ interface BrainInternals { metadataIndex: { rebuild(...a: unknown[]): Promise } graphIndex: { size(): number; rebuild(...a: unknown[]): Promise } _indexEpochStale: boolean - lazyRebuildCompleted: boolean rebuildIndexesIfNeeded(force?: boolean): Promise - ensureIndexesLoaded(): Promise + ensureIndexesLoaded(): void storage: { readRawObject(p: string): Promise } } @@ -181,40 +180,40 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b expect(idxSpy).toHaveBeenCalledTimes(1) }) - // --- Hook 1: large-path first-query lazy force-rebuild deference ---------- + // --- Hook 1: read-gate deference (RE-POINTED — the health-gate law retired + // the first-query lazy force-rebuild entirely: ensureIndexesLoaded() is now + // a pure CHECK that never calls rebuildIndexesIfNeeded, migrating or not. + // What survives from the original law is the DEFERENCE itself: a migrating + // provider's report is never judged by the gate — it neither throws nor + // rebuilds — while the exact same not-ready report on a NON-migrating + // provider throws the typed error instead of ever rebuilding.) ------------ - it('lazy first-query force-rebuild is SKIPPED when the vector provider isMigrating()', async () => { - // disableAutoRebuild routes first queries through ensureIndexesLoaded() (the - // large-brain lazy path that would otherwise force a blocking rebuild). + it('the read gate defers to a migrating vector provider — a not-ready report neither throws nor rebuilds', async () => { const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) const internals = internalsOf(brain) const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) - // Simulate a cold/empty live vector index (cor is mid-swap, serving canonical). - vi.spyOn(internals.index, 'size').mockReturnValue(0) - internals.lazyRebuildCompleted = false + // Simulate a not-ready live vector index (cor is mid-swap, serving canonical). + ;(internals.index as unknown as { isReady?: () => boolean }).isReady = () => false setMigrating(internals.index, true) - await internals.ensureIndexesLoaded() - - // A query during cor's background swap must not trigger brainy's blocking rebuild. + expect(() => internals.ensureIndexesLoaded()).not.toThrow() + // A query during cor's background swap must not trigger brainy's own + // rebuild — reads never rebuild in any case, migrating or not. expect(rebuildSpy).toHaveBeenCalledTimes(0) }) - it('lazy first-query force-rebuild STILL fires when the vector provider is not migrating (control)', async () => { + it('the read gate THROWS for the same not-ready vector provider once migration clears (control)', async () => { const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) const internals = internalsOf(brain) const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) - vi.spyOn(internals.index, 'size').mockReturnValue(0) - internals.lazyRebuildCompleted = false + ;(internals.index as unknown as { isReady?: () => boolean }).isReady = () => false // No isMigrating → not deferring. - await internals.ensureIndexesLoaded() - - // Without deference, the cold empty index drives the lazy force-rebuild. - expect(rebuildSpy).toHaveBeenCalledTimes(1) - expect(rebuildSpy).toHaveBeenCalledWith(true) + expect(() => internals.ensureIndexesLoaded()).toThrow(VectorIndexNotReadyError) + // Still never rebuilds — the gate refuses loudly instead. + expect(rebuildSpy).toHaveBeenCalledTimes(0) }) // --- Hook 2: public stampBrainFormat() ----------------------------------- diff --git a/tests/unit/metadata-cold-read-guard.test.ts b/tests/unit/metadata-cold-read-guard.test.ts index 40d37de6..b4f82f15 100644 --- a/tests/unit/metadata-cold-read-guard.test.ts +++ b/tests/unit/metadata-cold-read-guard.test.ts @@ -3,10 +3,14 @@ * reported cold `find({ where })` returning a silent `[]` on a freshly-opened * brain (a native metadata index that reports data but has not loaded its field * postings). This guard, the field-index counterpart of verifyGraphAdjacencyLive, - * probes a known persisted value on the first filtered find(): if the index does - * not serve it, brainy rebuilds and re-probes, and raises a loud - * MetadataIndexNotReadyError only if the rebuild still can't serve — never a - * silent empty result that misrepresents existing data. + * probes a known persisted value on the first filtered find(). + * + * RE-POINTED to the health-gate law: the guard NEVER rebuilds and NEVER walks + * the store from a read — a read-path rebuild is exactly the dark-rebuild + * failure mode the law retires (open() alone owns building). When the probe + * cannot serve the known value it raises a loud MetadataIndexNotReadyError + * IMMEDIATELY, with no rebuild attempt in between — never a silent empty + * result that misrepresents existing data. * * The 8.0 JS index cold-loads correctly, so we simulate the cold native failure * mode by intercepting the provider's getIdsForFilter/rebuild. @@ -42,37 +46,19 @@ describe('Metadata cold-read guard (#venue silent-[])', () => { mi.rebuild = origRebuild }) - it('cold index: verifyMetadataLive self-heals via rebuild — find({where}) is correct, NOT silent []', async () => { + it('cold index: verifyMetadataLive REFUSES immediately — find({where}) throws MetadataIndexNotReadyError, NEVER a silent [], and NEVER a rebuild attempt', async () => { const mi = brain.metadataIndex const origGetIds = mi.getIdsForFilter.bind(mi) + let rebuilds = 0 const origRebuild = mi.rebuild.bind(mi) - let cold = true brain._metadataVerified = false // re-arm the one-shot for this scenario - mi.getIdsForFilter = async (...a: any[]) => (cold ? [] : origGetIds(...a)) - mi.rebuild = async () => { - await origRebuild() - cold = false // the rebuild warms the postings - } - try { - const res = await brain.find({ where: { status: 'active' }, limit: 100 }) - expect(res.length).toBe(1) // self-healed — the known entity is returned - } finally { - mi.getIdsForFilter = origGetIds - mi.rebuild = origRebuild - } - }) - - it('unrecoverably cold index: find({where}) throws MetadataIndexNotReadyError — never a silent []', async () => { - const mi = brain.metadataIndex - const origGetIds = mi.getIdsForFilter.bind(mi) - const origRebuild = mi.rebuild.bind(mi) - brain._metadataVerified = false - mi.getIdsForFilter = async () => [] // always cold; rebuild can't fix it - mi.rebuild = async () => {} + mi.getIdsForFilter = async () => [] // cold: the known value never resolves + mi.rebuild = async () => { rebuilds++; return origRebuild() } try { await expect(brain.find({ where: { status: 'active' }, limit: 100 })).rejects.toBeInstanceOf( MetadataIndexNotReadyError ) + expect(rebuilds).toBe(0) // the guard never rebuilds from a read — it refuses loudly instead } finally { mi.getIdsForFilter = origGetIds mi.rebuild = origRebuild diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index c43b2cf2..d4d268ac 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -63,6 +63,9 @@ function inGate(rel: string): boolean { return ( rel.startsWith('tests/unit/') || rel.startsWith('tests/integration/') || + // The lifecycle biography lane — included by the integration config + // ('tests/lifecycle/**/*.test.ts'; see tests/lifecycle/README.md). + rel.startsWith('tests/lifecycle/') || rel.endsWith('.unit.test.ts') || rel.endsWith('.integration.test.ts') ) diff --git a/tests/unit/utils/indexReadiness.test.ts b/tests/unit/utils/indexReadiness.test.ts new file mode 100644 index 00000000..e00d1ff0 --- /dev/null +++ b/tests/unit/utils/indexReadiness.test.ts @@ -0,0 +1,153 @@ +/** + * @module tests/unit/utils/indexReadiness + * @description Pins for the read-gate authority, {@link assessProviderHealth}, and + * its older sibling {@link assessIndexReadiness}. The health-gate law: a provider's + * NAMED, synchronous, O(1) health report — when exposed — REPLACES the `isReady()`/ + * size-heuristic fallback as the read gate's source of truth. A throw from + * `healthReport()` is a CONTRACT VIOLATION (never read as healthy, never swallowed + * into "unknown"); an `unledgered` family is UNKNOWN (never healthy, never broken — + * `serving` is always the provider's own verdict, verbatim). + */ +import { describe, it, expect } from 'vitest' +import { assessIndexReadiness, assessProviderHealth } from '../../../src/utils/indexReadiness.js' +import type { HealthReport, LedgerInvariantResult } from '../../../src/plugin.js' + +function invariant(overrides: Partial = {}): LedgerInvariantResult { + return { + name: 'manifest-residency', + holds: true, + detail: 'ok', + heal: 'none', + source: 'ledger', + ...overrides + } +} + +function report(overrides: Partial = {}): HealthReport { + return { + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 1, + generation: 1, + unledgered: [], + ...overrides + } +} + +describe('assessIndexReadiness (legacy isReady() classifier)', () => { + it('unknown when the provider is null/undefined', () => { + expect(assessIndexReadiness(null)).toBe('unknown') + expect(assessIndexReadiness(undefined)).toBe('unknown') + }) + + it('unknown when isReady() is absent', () => { + expect(assessIndexReadiness({})).toBe('unknown') + }) + + it('ready / not-ready mirror isReady()', () => { + expect(assessIndexReadiness({ isReady: () => true })).toBe('ready') + expect(assessIndexReadiness({ isReady: () => false })).toBe('not-ready') + }) +}) + +describe('assessProviderHealth — the read-gate authority', () => { + it('via "none": no provider at all', () => { + const a = assessProviderHealth(null) + expect(a.via).toBe('none') + expect(a.readiness).toBe('unknown') + expect(a.report).toBeNull() + expect(a.reasons.length).toBeGreaterThan(0) + }) + + it('via "size-heuristic": provider exposes neither healthReport() nor isReady()', () => { + const a = assessProviderHealth({}) + expect(a.via).toBe('size-heuristic') + expect(a.readiness).toBe('unknown') + expect(a.report).toBeNull() + }) + + it('via "is-ready": provider exposes isReady() but no healthReport() — ready', () => { + const a = assessProviderHealth({ isReady: () => true }) + expect(a.via).toBe('is-ready') + expect(a.readiness).toBe('ready') + expect(a.reasons).toEqual([]) + }) + + it('via "is-ready": isReady() === false — not-ready with a reason', () => { + const a = assessProviderHealth({ isReady: () => false }) + expect(a.via).toBe('is-ready') + expect(a.readiness).toBe('not-ready') + expect(a.reasons.length).toBeGreaterThan(0) + }) + + it('healthReport() present REPLACES isReady() — serving:true wins even if isReady() lies false', () => { + const p = { isReady: () => false, healthReport: () => report({ serving: true }) } + const a = assessProviderHealth(p) + expect(a.via).toBe('health-report') + expect(a.readiness).toBe('ready') + }) + + it('serving:true, healthy:true, no invariants failing → ready, no reasons', () => { + const p = { healthReport: () => report({ serving: true, healthy: true }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('ready') + expect(a.reasons).toEqual([]) + expect(a.report).toEqual(report({ serving: true, healthy: true })) + }) + + it('serving:false with a named heal:"rebuild" failing invariant → not-ready, reason names it', () => { + const failing = invariant({ name: 'posted-count-floor', holds: false, heal: 'rebuild', detail: 'posted 10 < canonical 20' }) + const p = { healthReport: () => report({ serving: false, healthy: false, invariants: [failing] }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('not-ready') + expect(a.reasons.some((r) => r.includes('posted-count-floor') && r.includes('heal:rebuild') && r.includes('posted 10 < canonical 20'))).toBe(true) + }) + + it('unledgered-only report (serving:true, no failing invariant) → ready, reason names the unledgered family', () => { + const p = { healthReport: () => report({ serving: true, healthy: true, unledgered: ['canonical-verb-coverage'] }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('ready') + expect(a.reasons.some((r) => r.includes('unledgered') && r.includes('canonical-verb-coverage'))).toBe(true) + }) + + it('UNLEDGERED IS UNKNOWN: an unledgered family never flips a NOT-serving provider to ready', () => { + const failing = invariant({ holds: false, heal: 'rebuild', name: 'x' }) + const p = { healthReport: () => report({ serving: false, healthy: false, invariants: [failing], unledgered: ['some-family'] }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('not-ready') + }) + + it('serving:true, healthy:false with a heal:"repair" failure → still ready (degraded-but-serving)', () => { + const failing = invariant({ name: 'stale-counter', holds: false, heal: 'repair', detail: 'counter drift' }) + const p = { healthReport: () => report({ serving: true, healthy: false, invariants: [failing] }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('ready') + expect(a.reasons.some((r) => r.includes('stale-counter') && r.includes('heal:repair'))).toBe(true) + }) + + it('healthReport() that THROWS is a CONTRACT VIOLATION: not-ready, via health-report, reason names the throw — never "unknown"', () => { + const p = { healthReport: () => { throw new Error('mmap window busy') } } + const a = assessProviderHealth(p) + expect(a.via).toBe('health-report') + expect(a.readiness).toBe('not-ready') + expect(a.report).toBeNull() + expect(a.reasons.some((r) => r.includes('mmap window busy'))).toBe(true) + expect(a.readiness).not.toBe('unknown') + }) + + it('healthReport() that throws a non-Error value still produces a named reason (String(err))', () => { + const p = { healthReport: () => { throw 'boom' } } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('not-ready') + expect(a.reasons.some((r) => r.includes('boom'))).toBe(true) + }) + + it('the returned report carries the generation for narration dedup', () => { + const p = { healthReport: () => report({ generation: 42 }) } + const a = assessProviderHealth(p) + expect(a.report?.generation).toBe(42) + }) +}) diff --git a/tests/unit/vector-cold-read-guard.test.ts b/tests/unit/vector-cold-read-guard.test.ts index 49ca6426..0905f298 100644 --- a/tests/unit/vector-cold-read-guard.test.ts +++ b/tests/unit/vector-cold-read-guard.test.ts @@ -3,9 +3,14 @@ * @description Pattern-A / Finding 1: a pure semantic find({ query }) has no * filter, so verifyMetadataLive never fires — nothing guarded the vector index. * A cold native vector index that loaded its COUNT but not its serving structure - * returned a silent []. verifyVectorLive() closes that: honest isReady() first, - * else a known-vector self-match probe; self-heal (rebuild) or throw - * VectorIndexNotReadyError — never a silent empty result. + * returned a silent []. verifyVectorLive() closes that: the health-report/isReady() + * authority first, else a known-vector self-match probe. + * + * RE-POINTED to the health-gate law: the guard NEVER rebuilds and NEVER walks + * the store from a read — a read-path rebuild is exactly the dark-rebuild + * failure mode the law retires (open() alone owns building). A not-serving + * signal (from either strategy) THROWS VectorIndexNotReadyError immediately, + * with no rebuild attempt in between — never a silent empty result. */ import { describe, it, expect, beforeEach } from 'vitest' import { Brainy, NounType, VectorIndexNotReadyError } from '../../src/index.js' @@ -34,50 +39,37 @@ describe('Vector cold-read guard (verifyVectorLive) — silent-[] on cold semant vi.rebuild = origRebuild }) - it('cold index: verifyVectorLive self-heals via rebuild — semantic find is correct, NOT silent []', async () => { - const vi = brain.index - const origSearch = vi.search.bind(vi) - const origRebuild = vi.rebuild.bind(vi) - let cold = true - brain._vectorVerified = false - // size()>0 (count present) but search returns nothing until a rebuild warms it. - vi.search = async (...a: any[]) => (cold ? [] : origSearch(...a)) - vi.rebuild = async (...a: any[]) => { await origRebuild(...a); cold = false } - try { - const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) - expect(res.length).toBeGreaterThan(0) // self-healed - } finally { - vi.search = origSearch; vi.rebuild = origRebuild - } - }) - - it('unrecoverably cold index: semantic find throws VectorIndexNotReadyError', async () => { + it('cold index (no isReady()): verifyVectorLive REFUSES immediately — throws VectorIndexNotReadyError, NEVER rebuilds', async () => { const vi = brain.index const origSearch = vi.search.bind(vi) + let rebuilds = 0 const origRebuild = vi.rebuild.bind(vi) brain._vectorVerified = false - vi.search = async () => [] // always cold; rebuild can't fix it - vi.rebuild = async () => {} + // size()>0 (count present) but search never returns a hit for the known vector. + vi.search = async () => [] + vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } try { await expect( brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) ).rejects.toBeInstanceOf(VectorIndexNotReadyError) + expect(rebuilds).toBe(0) // the guard never rebuilds from a read — it refuses loudly instead } finally { vi.search = origSearch; vi.rebuild = origRebuild } }) - it('native provider reporting isReady()===false rebuilds, then serves', async () => { + it('native provider reporting isReady()===false THROWS immediately — never rebuilds', async () => { const vi = brain.index + let rebuilds = 0 const origRebuild = vi.rebuild.bind(vi) - let ready = false brain._vectorVerified = false - vi.isReady = () => ready - vi.rebuild = async (...a: any[]) => { await origRebuild(...a); ready = true } + vi.isReady = () => false + vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } try { - const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) - expect(ready).toBe(true) // rebuild ran because isReady() was false - expect(res).toBeDefined() + await expect( + brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) + ).rejects.toBeInstanceOf(VectorIndexNotReadyError) + expect(rebuilds).toBe(0) // a not-ready report throws immediately — it is never a rebuild trigger } finally { delete vi.isReady; vi.rebuild = origRebuild }