fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal
- AggregationIndex: defineAggregate before init no longer forces a backfill. init reconciles instead of clobbering: the app definition wins, persisted state is adopted on hash match, a write landing pre-adoption forces an exact rescan, and a successful state load clears the backfill flag. New ready() settles every adoption decision before query paths consult backfill state. - brainy: backfills are single-flight and batched. Concurrent queries share one store walk and every pending aggregate fills from that same walk; the old behavior let each concurrent query wipe the others' partial state and start its own full walk, so a store under steady aggregate traffic never converged. queryAggregate also waits for persisted definitions before deciding an aggregate does not exist. - paramValidation: recordQuery is telemetry-only. The duration-based cap ratchet (x0.8 per recorded query while lifetime-average exceeded 1s, floored at 1000 - below the documented 10000 auto floor, reported under a stale basis label) is removed; the cap comes from its construction-time basis or explicit overrides alone. - storage: __aggregation_* and singleton system keys (brainy:entityIdMapper) are recognized before the unknown-key warning fires; routing is unchanged. - docs: find-limits cap-immutability note, aggregation reopen/adoption semantics, RELEASES.md 8.5.1 entry.
This commit is contained in:
parent
593bb8b0f9
commit
da55be7520
10 changed files with 584 additions and 44 deletions
|
|
@ -598,6 +598,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
private _hub?: IntegrationHub // Integration Hub for external tools
|
||||
private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets
|
||||
private _aggregationIndex?: AggregationIndex // Incremental aggregation engine
|
||||
private _aggregationBackfillFlight: Promise<void> | null = null // Single-flight backfill walk
|
||||
private _materializer?: AggregateMaterializer // Debounced materialization of aggregate results
|
||||
/**
|
||||
* Fields registered via `brain.trackField()` — drives optional value validation on
|
||||
|
|
@ -5783,6 +5784,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
): Promise<AggregateResult[]> {
|
||||
await this.ensureInitialized()
|
||||
this.ensureAggregationIndex()
|
||||
// Persisted definitions load asynchronously — wait for them before deciding
|
||||
// the aggregate doesn't exist (an app that relies on persisted definitions
|
||||
// without re-defining at boot would otherwise race a spurious throw here).
|
||||
await this._aggregationIndex!.ready()
|
||||
if (!this._aggregationIndex!.hasAggregate(name)) {
|
||||
throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`)
|
||||
}
|
||||
|
|
@ -15810,9 +15815,42 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*/
|
||||
private async backfillAggregateIfNeeded(name: string): Promise<void> {
|
||||
const index = this._aggregationIndex
|
||||
if (!index || !index.getPendingBackfills().includes(name)) return
|
||||
if (!index) return
|
||||
|
||||
index.beginBackfill(name)
|
||||
// Persisted-state adoption happens inside ready(); after it resolves the
|
||||
// pending-backfill set is authoritative (an unchanged definition with valid
|
||||
// persisted state is NOT listed — no walk at all on a clean reopen).
|
||||
await index.ready()
|
||||
|
||||
// Single-flight: concurrent queries share ONE walk instead of each wiping
|
||||
// the others' partial state and starting their own (the stampede that kept
|
||||
// a busy store from ever converging). The loop covers the rare case where
|
||||
// the in-flight walk snapshotted its batch before `name` became pending —
|
||||
// the next iteration starts a fresh walk that includes it.
|
||||
while (index.getPendingBackfills().includes(name)) {
|
||||
if (!this._aggregationBackfillFlight) {
|
||||
this._aggregationBackfillFlight = this.runAggregationBackfillWalk()
|
||||
.finally(() => {
|
||||
this._aggregationBackfillFlight = null
|
||||
})
|
||||
}
|
||||
await this._aggregationBackfillFlight
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One store walk fills EVERY aggregate currently pending backfill — M pending
|
||||
* aggregates cost one enumeration, not M. Only reached when an aggregate
|
||||
* genuinely needs a rescan (new definition over a populated store, changed
|
||||
* definition, or failed state load); a clean reopen adopts persisted state
|
||||
* and never walks.
|
||||
*/
|
||||
private async runAggregationBackfillWalk(): Promise<void> {
|
||||
const index = this._aggregationIndex!
|
||||
const names = index.getPendingBackfills()
|
||||
if (names.length === 0) return
|
||||
|
||||
for (const n of names) index.beginBackfill(n)
|
||||
|
||||
const PAGE = 500
|
||||
let offset = 0
|
||||
|
|
@ -15822,7 +15860,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset }
|
||||
})
|
||||
for (const noun of page.items) {
|
||||
index.backfillEntity(name, noun as unknown as Record<string, unknown>)
|
||||
const record = noun as unknown as Record<string, unknown>
|
||||
for (const n of names) {
|
||||
index.backfillEntity(n, record)
|
||||
}
|
||||
}
|
||||
if (!page.hasMore || page.items.length === 0) break
|
||||
if (page.nextCursor) {
|
||||
|
|
@ -15832,7 +15873,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
index.finishBackfill(name)
|
||||
for (const n of names) index.finishBackfill(n)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue