fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal

- AggregationIndex: defineAggregate before init no longer forces a backfill.
  init reconciles instead of clobbering: the app definition wins, persisted
  state is adopted on hash match, a write landing pre-adoption forces an exact
  rescan, and a successful state load clears the backfill flag. New ready()
  settles every adoption decision before query paths consult backfill state.
- brainy: backfills are single-flight and batched. Concurrent queries share one
  store walk and every pending aggregate fills from that same walk; the old
  behavior let each concurrent query wipe the others' partial state and start
  its own full walk, so a store under steady aggregate traffic never converged.
  queryAggregate also waits for persisted definitions before deciding an
  aggregate does not exist.
- paramValidation: recordQuery is telemetry-only. The duration-based cap
  ratchet (x0.8 per recorded query while lifetime-average exceeded 1s, floored
  at 1000 - below the documented 10000 auto floor, reported under a stale
  basis label) is removed; the cap comes from its construction-time basis or
  explicit overrides alone.
- storage: __aggregation_* and singleton system keys (brainy:entityIdMapper)
  are recognized before the unknown-key warning fires; routing is unchanged.
- docs: find-limits cap-immutability note, aggregation reopen/adoption
  semantics, RELEASES.md 8.5.1 entry.
This commit is contained in:
David Snelling 2026-07-17 09:20:09 -07:00
parent 593bb8b0f9
commit da55be7520
10 changed files with 584 additions and 44 deletions

View file

@ -598,6 +598,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
private _hub?: IntegrationHub // Integration Hub for external tools
private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets
private _aggregationIndex?: AggregationIndex // Incremental aggregation engine
private _aggregationBackfillFlight: Promise<void> | null = null // Single-flight backfill walk
private _materializer?: AggregateMaterializer // Debounced materialization of aggregate results
/**
* Fields registered via `brain.trackField()` drives optional value validation on
@ -5783,6 +5784,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
): Promise<AggregateResult[]> {
await this.ensureInitialized()
this.ensureAggregationIndex()
// Persisted definitions load asynchronously — wait for them before deciding
// the aggregate doesn't exist (an app that relies on persisted definitions
// without re-defining at boot would otherwise race a spurious throw here).
await this._aggregationIndex!.ready()
if (!this._aggregationIndex!.hasAggregate(name)) {
throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`)
}
@ -15810,9 +15815,42 @@ export class Brainy<T = any> implements BrainyInterface<T> {
*/
private async backfillAggregateIfNeeded(name: string): Promise<void> {
const index = this._aggregationIndex
if (!index || !index.getPendingBackfills().includes(name)) return
if (!index) return
index.beginBackfill(name)
// Persisted-state adoption happens inside ready(); after it resolves the
// pending-backfill set is authoritative (an unchanged definition with valid
// persisted state is NOT listed — no walk at all on a clean reopen).
await index.ready()
// Single-flight: concurrent queries share ONE walk instead of each wiping
// the others' partial state and starting their own (the stampede that kept
// a busy store from ever converging). The loop covers the rare case where
// the in-flight walk snapshotted its batch before `name` became pending —
// the next iteration starts a fresh walk that includes it.
while (index.getPendingBackfills().includes(name)) {
if (!this._aggregationBackfillFlight) {
this._aggregationBackfillFlight = this.runAggregationBackfillWalk()
.finally(() => {
this._aggregationBackfillFlight = null
})
}
await this._aggregationBackfillFlight
}
}
/**
* One store walk fills EVERY aggregate currently pending backfill M pending
* aggregates cost one enumeration, not M. Only reached when an aggregate
* genuinely needs a rescan (new definition over a populated store, changed
* definition, or failed state load); a clean reopen adopts persisted state
* and never walks.
*/
private async runAggregationBackfillWalk(): Promise<void> {
const index = this._aggregationIndex!
const names = index.getPendingBackfills()
if (names.length === 0) return
for (const n of names) index.beginBackfill(n)
const PAGE = 500
let offset = 0
@ -15822,7 +15860,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
})
for (const noun of page.items) {
index.backfillEntity(name, noun as unknown as Record<string, unknown>)
const record = noun as unknown as Record<string, unknown>
for (const n of names) {
index.backfillEntity(n, record)
}
}
if (!page.hasMore || page.items.length === 0) break
if (page.nextCursor) {
@ -15832,7 +15873,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
index.finishBackfill(name)
for (const n of names) index.finishBackfill(n)
}
/**