fix: surface a degraded derived index on reads instead of serving it silently
Two known-degraded states were recorded but never consulted by the read paths, so a partial result looked authoritative: - commitSingleOp returns `degraded` ids on an adopt-forward failed-rollback recovery (the canonical record is durable but its derived-index entry may be incomplete). persistSingleOp dropped that list on the floor — no health flag, no read signal. It now records them in a queryable degraded set. - A non-fatal index-rebuild failure at init (_indexRebuildFailed) was folded into checkHealth()/validateIndexConsistency() but no read consulted it. find() and get() now emit ONE loud warning per degraded window (reads still return — canonical is the source of truth — but the caller is told results may be partial and to run repairIndex()). Both degraded sources fold into the two health surfaces, and repairIndex() reconciles from canonical and clears them.
This commit is contained in:
parent
7feba49d94
commit
ba958d97b5
2 changed files with 184 additions and 3 deletions
108
src/brainy.ts
108
src/brainy.ts
|
|
@ -478,6 +478,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* during rebuild is NOT recorded here — it re-throws and aborts init() loudly.
|
||||
*/
|
||||
private _indexRebuildFailed: Error | null = null
|
||||
/**
|
||||
* Ids of records that committed via an adopt-forward failed-rollback recovery
|
||||
* ({@link GenerationStore.commitSingleOp} `degraded`): the canonical record is
|
||||
* durable but its derived index entry may be incomplete until
|
||||
* {@link repairIndex}. Non-empty = a queryable degraded state, folded into
|
||||
* {@link checkHealth}/{@link validateIndexConsistency} and warned on the read
|
||||
* paths. Cleared by {@link repairIndex}.
|
||||
*/
|
||||
private _indexDegradedIds: Set<string> = new Set()
|
||||
/** One-shot guard so the degraded-reads warning fires once per degraded window
|
||||
* (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */
|
||||
private _degradedReadWarned = false
|
||||
/** One-shot guard so the metadata cold-open consistency probe runs once per brain. */
|
||||
private _metadataConsistencyProbed = false
|
||||
/** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */
|
||||
|
|
@ -1608,7 +1620,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
run: TransactionFunction<void>,
|
||||
precommit?: (before: CommitBeforeImages) => void,
|
||||
pendingEvents?: PendingChangeEvent[]
|
||||
): Promise<{ generation?: number; timestamp: number }> {
|
||||
): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> {
|
||||
// Change-feed capture: when this write will emit, hold a reference to the
|
||||
// commit's before-images so `remove` events can carry the record's last
|
||||
// committed state (free — the commit reads them anyway for Model B).
|
||||
|
|
@ -1667,6 +1679,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// that did not become durable. (A degraded adopt-forward write DID commit —
|
||||
// it carries a generation and emits normally.)
|
||||
this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp)
|
||||
// An adopt-forward failed-rollback recovery committed the record but may have
|
||||
// left its derived index incomplete (generationStore already warned once).
|
||||
// Record the ids so reads and checkHealth() surface the incompleteness rather
|
||||
// than silently returning partial data; repairIndex() clears them.
|
||||
if (receipt.degraded && receipt.degraded.length > 0) {
|
||||
for (const degradedId of receipt.degraded) this._indexDegradedIds.add(degradedId)
|
||||
this._degradedReadWarned = false // re-arm the read-path warning
|
||||
if (!this.config.silent) {
|
||||
prodLog.warn(
|
||||
`[Brainy] A single-op write committed in a DEGRADED state — the derived ` +
|
||||
`index may be incomplete for id(s) ${receipt.degraded.join(', ')}. ` +
|
||||
`Reads may return partial results until repairIndex() reconciles them.`
|
||||
)
|
||||
}
|
||||
}
|
||||
return receipt
|
||||
}
|
||||
|
||||
|
|
@ -2157,6 +2184,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*/
|
||||
async get(id: string, options?: GetOptions): Promise<Entity<T> | null> {
|
||||
await this.ensureInitialized()
|
||||
this.warnIfReadsDegraded('get')
|
||||
|
||||
// Id normalization (8.0): a caller may read by their natural key — resolve
|
||||
// it to the same canonical UUID add() stored. A real UUID passes through.
|
||||
|
|
@ -5562,8 +5590,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this._aggregationIndex = new AggregationIndex(this.storage, nativeProvider)
|
||||
// Note: init() is async but definitions can be registered synchronously.
|
||||
// State loading happens lazily on first query.
|
||||
this._aggregationIndex.init().catch(() => {
|
||||
// Non-fatal — aggregation state will be empty but definitions still work
|
||||
this._aggregationIndex.init().catch((err) => {
|
||||
// Non-fatal — definitions still work and state backfills on first query —
|
||||
// but a failed state load means aggregates read empty/stale until then, so
|
||||
// surface it loudly rather than swallow it.
|
||||
if (!this.config.silent) {
|
||||
prodLog.warn(
|
||||
`[Brainy] Aggregation index state failed to load at init ` +
|
||||
`(${(err as Error).message}). Aggregates may read empty until a ` +
|
||||
`backfill-on-query repopulates them.`
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -5643,6 +5680,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// metadata counterpart of the graph cold-load guard. No-op for the JS index.
|
||||
await this.ensureMetadataConsistencyProbed()
|
||||
|
||||
// Loudly flag a degraded derived index (failed init rebuild, or an
|
||||
// adopt-forward degraded commit) so a partial result is never mistaken for
|
||||
// authoritative. The streaming search() generator delegates to find(), so it
|
||||
// is covered here.
|
||||
this.warnIfReadsDegraded('find')
|
||||
|
||||
// Parse natural language queries
|
||||
let params: FindParams<T> =
|
||||
typeof query === 'string' ? await this.parseNaturalQuery(query) : query
|
||||
|
|
@ -12318,6 +12361,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
(result.recommendation ? ` Also: ${result.recommendation}` : '')
|
||||
}
|
||||
}
|
||||
if (this._indexDegradedIds.size > 0) {
|
||||
return {
|
||||
...result,
|
||||
healthy: false,
|
||||
recommendation:
|
||||
`${this._indexDegradedIds.size} record(s) committed via adopt-forward ` +
|
||||
`failed-rollback recovery — the derived index may be incomplete for them. ` +
|
||||
`Run repairIndex() to reconcile.` +
|
||||
(result.recommendation ? ` Also: ${result.recommendation}` : '')
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
|
|
@ -14746,6 +14800,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
(result.recommendation ? ` Also: ${result.recommendation}` : '')
|
||||
}
|
||||
}
|
||||
if (this._indexDegradedIds.size > 0) {
|
||||
return {
|
||||
...result,
|
||||
healthy: false,
|
||||
recommendation:
|
||||
`${this._indexDegradedIds.size} record(s) committed via adopt-forward ` +
|
||||
`failed-rollback recovery — the derived index may be incomplete for them. ` +
|
||||
`Run repairIndex() to reconcile.` +
|
||||
(result.recommendation ? ` Also: ${result.recommendation}` : '')
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
|
|
@ -14800,6 +14865,34 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* genuinely lost record cannot be resurrected here (restore from a snapshot for
|
||||
* that), but the store is made internally consistent and writes re-enabled.
|
||||
*/
|
||||
/**
|
||||
* Emit ONE loud warning per degraded window when a read is served while the
|
||||
* derived index is known-incomplete — either a non-fatal init rebuild failure
|
||||
* ({@link _indexRebuildFailed}) or an adopt-forward degraded commit
|
||||
* ({@link _indexDegradedIds}). Reads still return (canonical is the source of
|
||||
* truth), but the caller is told the result may be partial and how to heal it.
|
||||
* No-op when healthy or when the brain is configured `silent`.
|
||||
* @param op - The read method name for the message (e.g. `'find'`, `'get'`).
|
||||
*/
|
||||
private warnIfReadsDegraded(op: string): void {
|
||||
const degraded =
|
||||
this._indexRebuildFailed !== null || this._indexDegradedIds.size > 0
|
||||
if (!degraded) {
|
||||
this._degradedReadWarned = false
|
||||
return
|
||||
}
|
||||
if (this._degradedReadWarned || this.config.silent) return
|
||||
this._degradedReadWarned = true
|
||||
const reason = this._indexRebuildFailed
|
||||
? `index rebuild failed at init() (${this._indexRebuildFailed.message})`
|
||||
: `${this._indexDegradedIds.size} record(s) committed in a degraded state`
|
||||
prodLog.warn(
|
||||
`[Brainy] ${op}() is serving reads while the derived index is INCOMPLETE ` +
|
||||
`(${reason}). Results may be missing entries — run repairIndex() to ` +
|
||||
`reconcile the derived indexes against canonical storage.`
|
||||
)
|
||||
}
|
||||
|
||||
async repairIndex(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
await this.metadataIndex.detectAndRepairCorruption()
|
||||
|
|
@ -14816,6 +14909,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
`Writes are re-enabled.`
|
||||
)
|
||||
}
|
||||
// detectAndRepairCorruption() above rebuilt the derived indexes from
|
||||
// canonical, so any adopt-forward degraded ids and a non-fatal init
|
||||
// rebuild failure are now reconciled — clear the queryable degraded state
|
||||
// and re-arm the read-path warning.
|
||||
if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) {
|
||||
this._indexDegradedIds.clear()
|
||||
this._indexRebuildFailed = null
|
||||
this._degradedReadWarned = false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue