fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal

- AggregationIndex: defineAggregate before init no longer forces a backfill.
  init reconciles instead of clobbering: the app definition wins, persisted
  state is adopted on hash match, a write landing pre-adoption forces an exact
  rescan, and a successful state load clears the backfill flag. New ready()
  settles every adoption decision before query paths consult backfill state.
- brainy: backfills are single-flight and batched. Concurrent queries share one
  store walk and every pending aggregate fills from that same walk; the old
  behavior let each concurrent query wipe the others' partial state and start
  its own full walk, so a store under steady aggregate traffic never converged.
  queryAggregate also waits for persisted definitions before deciding an
  aggregate does not exist.
- paramValidation: recordQuery is telemetry-only. The duration-based cap
  ratchet (x0.8 per recorded query while lifetime-average exceeded 1s, floored
  at 1000 - below the documented 10000 auto floor, reported under a stale
  basis label) is removed; the cap comes from its construction-time basis or
  explicit overrides alone.
- storage: __aggregation_* and singleton system keys (brainy:entityIdMapper)
  are recognized before the unknown-key warning fires; routing is unchanged.
- docs: find-limits cap-immutability note, aggregation reopen/adoption
  semantics, RELEASES.md 8.5.1 entry.
This commit is contained in:
David Snelling 2026-07-17 09:20:09 -07:00
parent 593bb8b0f9
commit da55be7520
10 changed files with 584 additions and 44 deletions

View file

@ -675,4 +675,104 @@ describe('AggregationIndex', () => {
await reloaded.close()
})
})
// ============= Boot-order reconciliation =============
//
// The production boot pattern: defineAggregate() is synchronous and always
// beats the async init() that loads persisted state. The old code flagged a
// backfill at define time and init never cleared it — so the loaded state
// was wiped and the whole store re-walked on EVERY restart.
describe('boot-order reconciliation (define-before-init)', () => {
const DEF: AggregateDefinition = {
name: 'boot_agg',
source: { type: NounType.Event },
groupBy: ['category'],
metrics: { count: { op: 'count' } }
}
const entity = (id: string, category: string): Record<string, unknown> => ({
id,
noun: NounType.Event,
metadata: { category },
createdAt: Date.now(),
updatedAt: Date.now()
})
/** Simulate the previous session: define, contribute, flush. */
async function seedAndFlush(store: MemoryStorage): Promise<void> {
const first = new AggregationIndex(store)
await first.init()
first.defineAggregate(DEF)
first.onEntityAdded('e1', entity('e1', 'food'))
first.onEntityAdded('e2', entity('e2', 'food'))
first.onEntityAdded('e3', entity('e3', 'transport'))
await first.flush()
}
it('adopts persisted state when define beats init with an unchanged definition', async () => {
const store = new MemoryStorage()
await store.init()
await seedAndFlush(store)
const second = new AggregationIndex(store)
second.defineAggregate(DEF) // synchronous define FIRST — the real boot order
await second.init()
await second.ready()
expect(second.getPendingBackfills()).toEqual([])
const rows = second.queryAggregate({ name: 'boot_agg' })
const food = rows.find(r => r.groupKey.category === 'food')!
expect(food.metrics.count).toBe(2)
})
it('a write landing before adoption forces an exact rescan instead', async () => {
const store = new MemoryStorage()
await store.init()
await seedAndFlush(store)
const second = new AggregationIndex(store)
second.defineAggregate(DEF)
second.onEntityAdded('e4', entity('e4', 'food')) // lands before init settles
await second.init()
await second.ready()
// Adoption would lose e4's contribution — the engine must rescan.
expect(second.getPendingBackfills()).toEqual(['boot_agg'])
})
it('init never clobbers a changed app definition registered before it', async () => {
const store = new MemoryStorage()
await store.init()
await seedAndFlush(store)
const CHANGED: AggregateDefinition = {
...DEF,
metrics: { count: { op: 'count' }, total: { op: 'sum', field: 'amount' } }
}
const second = new AggregationIndex(store)
second.defineAggregate(CHANGED)
await second.init()
await second.ready()
const def = second.getDefinitions().find(d => d.name === 'boot_agg')!
expect(Object.keys(def.metrics).sort()).toEqual(['count', 'total'])
expect(second.getPendingBackfills()).toEqual(['boot_agg'])
})
it('init alone restores persisted definitions with adopted state, no backfill', async () => {
const store = new MemoryStorage()
await store.init()
await seedAndFlush(store)
const second = new AggregationIndex(store)
await second.init()
await second.ready()
expect(second.hasAggregate('boot_agg')).toBe(true)
expect(second.getPendingBackfills()).toEqual([])
const rows = second.queryAggregate({ name: 'boot_agg' })
expect(rows.reduce((s, r) => s + (r.metrics.count as number), 0)).toBe(3)
})
})
})

View file

@ -270,27 +270,24 @@ describe('Zero-Config Parameter Validation', () => {
expect(config.availableMemory).toBeGreaterThan(0)
})
it('should adapt limits based on query performance', () => {
const initialConfig = getValidationConfig()
const initialLimit = initialConfig.maxLimit
// Simulate fast queries with large results
it('never mutates the cap from query timing (telemetry only)', () => {
const initialLimit = getValidationConfig().maxLimit
// Fast queries with large results: no silent growth.
for (let i = 0; i < 10; i++) {
recordQueryPerformance(50, initialLimit * 0.9)
}
const updatedConfig = getValidationConfig()
// Limit might increase if performance is good
expect(updatedConfig.maxLimit).toBeGreaterThanOrEqual(initialLimit)
// Simulate slow queries
for (let i = 0; i < 10; i++) {
recordQueryPerformance(2000, 100)
expect(getValidationConfig().maxLimit).toBe(initialLimit)
// A burst of catastrophically slow queries must not strangle the cap.
// The removed "learning" ratchet shrank it 20% per recorded query down
// to a floor of 1000 — below the documented MIN_AUTO_QUERY_LIMIT — and
// the error message blamed "available free memory" (a production
// incident: every find({ limit: 5000 }) failed on an idle 23GB-free box).
for (let i = 0; i < 50; i++) {
recordQueryPerformance(90_000, 100)
}
const finalConfig = getValidationConfig()
// Limit should decrease if performance is poor
expect(finalConfig.maxLimit).toBeLessThanOrEqual(updatedConfig.maxLimit)
expect(getValidationConfig().maxLimit).toBe(initialLimit)
})
})
})