soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal

Browse source 
- AggregationIndex: defineAggregate before init no longer forces a backfill.
  init reconciles instead of clobbering: the app definition wins, persisted
  state is adopted on hash match, a write landing pre-adoption forces an exact
  rescan, and a successful state load clears the backfill flag. New ready()
  settles every adoption decision before query paths consult backfill state.
- brainy: backfills are single-flight and batched. Concurrent queries share one
  store walk and every pending aggregate fills from that same walk; the old
  behavior let each concurrent query wipe the others' partial state and start
  its own full walk, so a store under steady aggregate traffic never converged.
  queryAggregate also waits for persisted definitions before deciding an
  aggregate does not exist.
- paramValidation: recordQuery is telemetry-only. The duration-based cap
  ratchet (x0.8 per recorded query while lifetime-average exceeded 1s, floored
  at 1000 - below the documented 10000 auto floor, reported under a stale
  basis label) is removed; the cap comes from its construction-time basis or
  explicit overrides alone.
- storage: __aggregation_* and singleton system keys (brainy:entityIdMapper)
  are recognized before the unknown-key warning fires; routing is unchanged.
- docs: find-limits cap-immutability note, aggregation reopen/adoption
  semantics, RELEASES.md 8.5.1 entry.
This commit is contained in:
David Snelling 2026-07-17 09:20:09 -07:00
parent 593bb8b0f9
commit da55be7520
10 changed files with 584 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

126
src/aggregation/AggregationIndex.ts
View file

@ -327,6 +327,22 @@ export class AggregationIndex {
/** Track aggregates with stale MIN/MAX (need lazy recompute) */
private staleMinMax = new Map<string, Set<string>>()
/** Resolves when init() has finished loading persisted definitions/state. */
private initPromise: Promise<void> | null = null
/** True once init() has settled (success or failure). */
private initDone = false
/**
* Aggregates registered by the app before init() finished loading persisted
* state, awaiting reconciliation: init() adopts the persisted state when the
* definition hash matches; anything left unadopted when init settles resolves
* to a backfill. Deciding backfill eagerly at define time was the boot-order
* bug that wiped valid persisted state on every restart — the synchronous
* defineAggregate() always beats the async init().
*/
private pendingAdopt = new Set<string>()
constructor(storage: StorageAdapter, nativeProvider?: AggregationProvider) {
this.storage = storage
this.nativeProvider = nativeProvider
@ -336,19 +352,87 @@ export class AggregationIndex {
/**
* Initialize: load persisted definitions and state, detect changes, rebuild stale.
*
* Idempotent — repeated calls return the same promise. Definitions registered
* *before* this completes (the normal boot order: `defineAggregate()` is
* synchronous and always beats this async load) are reconciled rather than
* clobbered: the app's definition wins, and its persisted state is adopted
* when the definition hash matches — backfill happens only on a real change.
*/
async init(): Promise<void> {
init(): Promise<void> {
if (!this.initPromise) {
this.initPromise = this.loadPersisted().finally(() => {
this.resolvePendingAdoptToBackfill()
this.initDone = true
})
}
return this.initPromise
}
/**
* Await the persisted-state load (if one was started) and settle every
* pending adoption decision. After this resolves, `getPendingBackfills()`
* is authoritative: a name is listed iff it genuinely needs a rescan.
* Query paths must await this before consulting backfill state.
*/
async ready(): Promise<void> {
if (this.initPromise) {
try {
await this.initPromise
} catch {
// The owner already surfaced the load failure loudly; backfill covers.
}
}
this.resolvePendingAdoptToBackfill()
}
/**
* Any definition still awaiting state adoption has no persisted state to
* adopt (or init never ran / failed) — it must backfill.
*/
private resolvePendingAdoptToBackfill(): void {
for (const name of this.pendingAdopt) this.needsBackfill.add(name)
this.pendingAdopt.clear()
}
private async loadPersisted(): Promise<void> {
// Load persisted definitions
const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY)
if (savedDefs && typeof savedDefs === 'object' && savedDefs.definitions) {
const defs = savedDefs.definitions as Array<AggregateDefinition & { _hash?: string }>
for (const def of defs) {
this.definitions.set(def.name, def)
const currentHash = hashDefinition(def)
const savedHash = def._hash || ''
// Load persisted state
if (this.definitions.has(def.name)) {
// The app re-registered this aggregate before the load finished.
// The app's definition wins — never clobber it with the persisted
// copy. Adopt the persisted state when the definition is unchanged
// AND no write has landed for it yet (a landed write would be lost
// by adoption; the hook flips such names to backfill).
const appHash = this.definitionHashes.get(def.name) || ''
if (appHash === savedHash && this.pendingAdopt.has(def.name)) {
const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
if (stateData && stateData.groups) {
const groupMap = new Map<string, AggregateGroupState>()
for (const group of stateData.groups as AggregateGroupState[]) {
groupMap.set(serializeGroupKey(group.groupKey), group)
}
this.states.set(def.name, groupMap)
this.pendingAdopt.delete(def.name)
this.needsBackfill.delete(def.name)
}
// No/invalid persisted state: stays in pendingAdopt and resolves
// to backfill when init settles.
}
continue
}
// Not registered this session — restore definition + state from
// persistence.
this.definitions.set(def.name, def)
const currentHash = hashDefinition(def)
const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
if (stateData && stateData.groups && savedHash === currentHash) {
// Definition unchanged — load state
@ -358,6 +442,7 @@ export class AggregationIndex {
groupMap.set(serialized, group)
}
this.states.set(def.name, groupMap)
this.needsBackfill.delete(def.name)
} else {
// Definition changed or no saved state — start fresh and backfill from
// existing entities (the owner drains needsBackfill on first query).
@ -452,10 +537,19 @@ export class AggregationIndex {
this.definitions.set(def.name, def)
this.definitionHashes.set(def.name, newHash)
// First sight this session, before init() settled: defer the backfill
// decision — init() adopts the persisted state on hash match, and anything
// left unadopted resolves to backfill. Deciding eagerly here wiped valid
// persisted state on every restart.
if (!this.states.has(def.name) && !this.initDone) {
this.states.set(def.name, new Map())
this.pendingAdopt.add(def.name)
}
// Reset state if definition changed or doesn't exist yet, and flag it for
// backfill so already-stored entities are counted (write-time hooks only see
// future writes). The owner drains this on the next query via getPendingBackfills().
if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) {
else if (!this.states.has(def.name) || (oldHash && oldHash !== newHash)) {
this.pendingAdopt.delete(def.name)
this.states.set(def.name, new Map())
this.needsBackfill.add(def.name)
}
@ -476,6 +570,8 @@ export class AggregationIndex {
this.definitionHashes.delete(name)
this.states.delete(name)
this.staleMinMax.delete(name)
this.pendingAdopt.delete(name)
this.needsBackfill.delete(name)
// Notify native provider
if (this.nativeProvider?.removeAggregate) {
@ -545,6 +641,20 @@ export class AggregationIndex {
// ============= Write-Time Hooks =============
/**
* A write is landing for an aggregate whose persisted-state adoption is still
* pending — adopting after this write would lose its contribution. Settle the
* decision now: an exact rescan instead of adoption. The window is the few
* milliseconds between a boot-time defineAggregate() and init() completing,
* so this rarely fires; when it does, correctness wins over the walk.
*/
private resolveAdoptOnWrite(name: string): void {
if (this.pendingAdopt.has(name)) {
this.pendingAdopt.delete(name)
this.needsBackfill.add(name)
}
}
/**
* Called when an entity is added. Updates all matching aggregates.
*/
@ -553,6 +663,7 @@ export class AggregationIndex {
for (const [name, def] of this.definitions) {
if (!matchesSource(entity, def.source)) continue
this.resolveAdoptOnWrite(name)
if (this.nativeProvider) {
const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'add')
@ -579,6 +690,10 @@ export class AggregationIndex {
const oldMatches = matchesSource(oldEntity, def.source)
const newMatches = matchesSource(newEntity, def.source)
if (oldMatches || newMatches) {
this.resolveAdoptOnWrite(name)
}
if (this.nativeProvider && (oldMatches || newMatches)) {
const results = this.nativeProvider.incrementalUpdate(name, def, newEntity, 'update', oldEntity)
this.applyNativeResults(name, results)
@ -605,6 +720,7 @@ export class AggregationIndex {
for (const [name, def] of this.definitions) {
if (!matchesSource(entity, def.source)) continue
this.resolveAdoptOnWrite(name)
if (this.nativeProvider) {
const results = this.nativeProvider.incrementalUpdate(name, def, entity, 'delete')