From ba958d97b5ca9c17c7a979a6ebc073cf752f67d8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 13 Jul 2026 09:49:38 -0700 Subject: [PATCH] fix: surface a degraded derived index on reads instead of serving it silently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two known-degraded states were recorded but never consulted by the read paths, so a partial result looked authoritative: - commitSingleOp returns `degraded` ids on an adopt-forward failed-rollback recovery (the canonical record is durable but its derived-index entry may be incomplete). persistSingleOp dropped that list on the floor — no health flag, no read signal. It now records them in a queryable degraded set. - A non-fatal index-rebuild failure at init (_indexRebuildFailed) was folded into checkHealth()/validateIndexConsistency() but no read consulted it. find() and get() now emit ONE loud warning per degraded window (reads still return — canonical is the source of truth — but the caller is told results may be partial and to run repairIndex()). Both degraded sources fold into the two health surfaces, and repairIndex() reconciles from canonical and clears them. --- src/brainy.ts | 108 +++++++++++++++++- .../brainy/degraded-reads-surfaced.test.ts | 79 +++++++++++++ 2 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 tests/unit/brainy/degraded-reads-surfaced.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index da44c252..32b8757b 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -478,6 +478,18 @@ export class Brainy implements BrainyInterface { * during rebuild is NOT recorded here — it re-throws and aborts init() loudly. */ private _indexRebuildFailed: Error | null = null + /** + * Ids of records that committed via an adopt-forward failed-rollback recovery + * ({@link GenerationStore.commitSingleOp} `degraded`): the canonical record is + * durable but its derived index entry may be incomplete until + * {@link repairIndex}. Non-empty = a queryable degraded state, folded into + * {@link checkHealth}/{@link validateIndexConsistency} and warned on the read + * paths. Cleared by {@link repairIndex}. + */ + private _indexDegradedIds: Set = new Set() + /** One-shot guard so the degraded-reads warning fires once per degraded window + * (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */ + private _degradedReadWarned = false /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */ private _metadataConsistencyProbed = false /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */ @@ -1608,7 +1620,7 @@ export class Brainy implements BrainyInterface { run: TransactionFunction, precommit?: (before: CommitBeforeImages) => void, pendingEvents?: PendingChangeEvent[] - ): Promise<{ generation?: number; timestamp: number }> { + ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> { // Change-feed capture: when this write will emit, hold a reference to the // commit's before-images so `remove` events can carry the record's last // committed state (free — the commit reads them anyway for Model B). @@ -1667,6 +1679,21 @@ export class Brainy implements BrainyInterface { // that did not become durable. (A degraded adopt-forward write DID commit — // it carries a generation and emits normally.) this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp) + // An adopt-forward failed-rollback recovery committed the record but may have + // left its derived index incomplete (generationStore already warned once). + // Record the ids so reads and checkHealth() surface the incompleteness rather + // than silently returning partial data; repairIndex() clears them. + if (receipt.degraded && receipt.degraded.length > 0) { + for (const degradedId of receipt.degraded) this._indexDegradedIds.add(degradedId) + this._degradedReadWarned = false // re-arm the read-path warning + if (!this.config.silent) { + prodLog.warn( + `[Brainy] A single-op write committed in a DEGRADED state — the derived ` + + `index may be incomplete for id(s) ${receipt.degraded.join(', ')}. ` + + `Reads may return partial results until repairIndex() reconciles them.` + ) + } + } return receipt } @@ -2157,6 +2184,7 @@ export class Brainy implements BrainyInterface { */ async get(id: string, options?: GetOptions): Promise | null> { await this.ensureInitialized() + this.warnIfReadsDegraded('get') // Id normalization (8.0): a caller may read by their natural key — resolve // it to the same canonical UUID add() stored. A real UUID passes through. @@ -5562,8 +5590,17 @@ export class Brainy implements BrainyInterface { this._aggregationIndex = new AggregationIndex(this.storage, nativeProvider) // Note: init() is async but definitions can be registered synchronously. // State loading happens lazily on first query. - this._aggregationIndex.init().catch(() => { - // Non-fatal — aggregation state will be empty but definitions still work + this._aggregationIndex.init().catch((err) => { + // Non-fatal — definitions still work and state backfills on first query — + // but a failed state load means aggregates read empty/stale until then, so + // surface it loudly rather than swallow it. + if (!this.config.silent) { + prodLog.warn( + `[Brainy] Aggregation index state failed to load at init ` + + `(${(err as Error).message}). Aggregates may read empty until a ` + + `backfill-on-query repopulates them.` + ) + } }) } @@ -5643,6 +5680,12 @@ export class Brainy implements BrainyInterface { // metadata counterpart of the graph cold-load guard. No-op for the JS index. await this.ensureMetadataConsistencyProbed() + // Loudly flag a degraded derived index (failed init rebuild, or an + // adopt-forward degraded commit) so a partial result is never mistaken for + // authoritative. The streaming search() generator delegates to find(), so it + // is covered here. + this.warnIfReadsDegraded('find') + // Parse natural language queries let params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query @@ -12318,6 +12361,17 @@ export class Brainy implements BrainyInterface { (result.recommendation ? ` Also: ${result.recommendation}` : '') } } + if (this._indexDegradedIds.size > 0) { + return { + ...result, + healthy: false, + recommendation: + `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + + `failed-rollback recovery — the derived index may be incomplete for them. ` + + `Run repairIndex() to reconcile.` + + (result.recommendation ? ` Also: ${result.recommendation}` : '') + } + } return result } @@ -14746,6 +14800,17 @@ export class Brainy implements BrainyInterface { (result.recommendation ? ` Also: ${result.recommendation}` : '') } } + if (this._indexDegradedIds.size > 0) { + return { + ...result, + healthy: false, + recommendation: + `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + + `failed-rollback recovery — the derived index may be incomplete for them. ` + + `Run repairIndex() to reconcile.` + + (result.recommendation ? ` Also: ${result.recommendation}` : '') + } + } return result } @@ -14800,6 +14865,34 @@ export class Brainy implements BrainyInterface { * genuinely lost record cannot be resurrected here (restore from a snapshot for * that), but the store is made internally consistent and writes re-enabled. */ + /** + * Emit ONE loud warning per degraded window when a read is served while the + * derived index is known-incomplete — either a non-fatal init rebuild failure + * ({@link _indexRebuildFailed}) or an adopt-forward degraded commit + * ({@link _indexDegradedIds}). Reads still return (canonical is the source of + * truth), but the caller is told the result may be partial and how to heal it. + * No-op when healthy or when the brain is configured `silent`. + * @param op - The read method name for the message (e.g. `'find'`, `'get'`). + */ + private warnIfReadsDegraded(op: string): void { + const degraded = + this._indexRebuildFailed !== null || this._indexDegradedIds.size > 0 + if (!degraded) { + this._degradedReadWarned = false + return + } + if (this._degradedReadWarned || this.config.silent) return + this._degradedReadWarned = true + const reason = this._indexRebuildFailed + ? `index rebuild failed at init() (${this._indexRebuildFailed.message})` + : `${this._indexDegradedIds.size} record(s) committed in a degraded state` + prodLog.warn( + `[Brainy] ${op}() is serving reads while the derived index is INCOMPLETE ` + + `(${reason}). Results may be missing entries — run repairIndex() to ` + + `reconcile the derived indexes against canonical storage.` + ) + } + async repairIndex(): Promise { await this.ensureInitialized() await this.metadataIndex.detectAndRepairCorruption() @@ -14816,6 +14909,15 @@ export class Brainy implements BrainyInterface { `Writes are re-enabled.` ) } + // detectAndRepairCorruption() above rebuilt the derived indexes from + // canonical, so any adopt-forward degraded ids and a non-fatal init + // rebuild failure are now reconciled — clear the queryable degraded state + // and re-arm the read-path warning. + if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) { + this._indexDegradedIds.clear() + this._indexRebuildFailed = null + this._degradedReadWarned = false + } } /** diff --git a/tests/unit/brainy/degraded-reads-surfaced.test.ts b/tests/unit/brainy/degraded-reads-surfaced.test.ts new file mode 100644 index 00000000..29a8a77c --- /dev/null +++ b/tests/unit/brainy/degraded-reads-surfaced.test.ts @@ -0,0 +1,79 @@ +/** + * @module tests/unit/brainy/degraded-reads-surfaced + * @description Finding 10: a known-degraded derived index must be SURFACED, not + * silently served as authoritative. Two degraded sources: + * (a) a non-fatal index rebuild failure at init() (`_indexRebuildFailed`), and + * (b) an adopt-forward failed-rollback recovery (`commitSingleOp`'s `degraded` + * ids, previously dropped on the floor by persistSingleOp). + * Both now fold into checkHealth()/validateIndexConsistency() and emit ONE loud + * read-path warning per degraded window; repairIndex() reconciles + clears them. + * + * The private fields are the observable contract of the fix, so the test drives + * them directly (a real failed rollback is exercised elsewhere). + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { prodLog } from '../../../src/utils/logger.js' + +const UUID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}` + +describe('Finding 10 — degraded derived-index state is surfaced on reads', () => { + beforeEach(() => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + }) + afterEach(() => vi.restoreAllMocks()) + + it('checkHealth() reports adopt-forward degraded ids as unhealthy', async () => { + const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + await brain.init() + ;(brain as any)._indexDegradedIds.add(UUID('de')) + + const health = await brain.checkHealth() + expect(health.healthy).toBe(false) + expect(health.recommendation).toMatch(/repairIndex\(\)/) + }) + + it('find()/get() warn loudly while degraded, ONCE, then repairIndex() clears it', async () => { + const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + await brain.init() + await brain.add({ id: UUID('a1'), data: 'x', type: NounType.Document }) + ;(brain as any)._indexRebuildFailed = new Error('rebuild boom') + + await brain.find({ type: NounType.Document }) + await brain.get(UUID('a1')) // second read: must NOT double-warn + const degradedWarns = warn.mock.calls.filter((c) => + String(c[0]).includes('derived index is INCOMPLETE') + ) + expect(degradedWarns.length).toBe(1) + + await brain.repairIndex() + expect((brain as any)._indexRebuildFailed).toBeNull() + expect((brain as any)._indexDegradedIds.size).toBe(0) + + warn.mockClear() + await brain.find({ type: NounType.Document }) + expect(warn.mock.calls.filter((c) => String(c[0]).includes('INCOMPLETE')).length).toBe(0) + }) + + it('persistSingleOp records receipt.degraded (widened return type, not dropped)', async () => { + const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + await brain.init() + // Simulate a degraded receipt by wrapping the generation store's commitSingleOp. + const gs: any = (brain as any).generationStore + const realCommit = gs.commitSingleOp.bind(gs) + vi.spyOn(gs, 'commitSingleOp').mockImplementation(async (args: any) => { + const r = await realCommit(args) + return { ...r, degraded: [...(args.touched.nouns ?? [])] } + }) + + await brain.add({ id: UUID('b2'), data: 'y', type: NounType.Document }) + expect((brain as any)._indexDegradedIds.size).toBeGreaterThan(0) + expect((brain as any)._indexDegradedIds.has(UUID('b2'))).toBe(true) + + const health = await brain.checkHealth() + expect(health.healthy).toBe(false) + expect(health.recommendation).toMatch(/repairIndex\(\)/) + }) +})